]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: change the unit of GL value macro
[mirror_ubuntu-focal-kernel.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
76ad4f0e
S		 1/*
2 * Copyright (c) 2016~2017 Hisilicon Limited.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/etherdevice.h>
12#include <linux/interrupt.h>
13#include <linux/if_vlan.h>
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/skbuff.h>
19#include <linux/sctp.h>
20#include <linux/vermagic.h>
21#include <net/gre.h>
30d240df 22#include <net/pkt_cls.h>
76ad4f0e
S		 23#include <net/vxlan.h>
24
25#include "hnae3.h"
26#include "hns3_enet.h"
27
1db9b1bf 28static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 29const char hns3_driver_version[] = VERMAGIC_STRING;
30static const char hns3_driver_string[] =
31 "Hisilicon Ethernet Network Driver for Hip08 Family";
32static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33static struct hnae3_client client;
34
35/* hns3_pci_tbl - PCI Device ID Table
36 *
37 * Last entry must be all 0s
38 *
39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40 * Class, Class Mask, private data (not used) }
41 */
42static const struct pci_device_id hns3_pci_tbl[] = {
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
424eb834
SM		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62static irqreturn_t hns3_irq_handle(int irq, void *dev)
63{
64 struct hns3_enet_tqp_vector *tqp_vector = dev;
65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
71static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
72{
73 struct hns3_enet_tqp_vector *tqp_vectors;
74 unsigned int i;
75
76 for (i = 0; i < priv->vector_num; i++) {
77 tqp_vectors = &priv->tqp_vector[i];
78
79 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
80 continue;
81
82 /* release the irq resource */
83 free_irq(tqp_vectors->vector_irq, tqp_vectors);
84 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
85 }
86}
87
88static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 int txrx_int_idx = 0;
92 int rx_int_idx = 0;
93 int tx_int_idx = 0;
94 unsigned int i;
95 int ret;
96
97 for (i = 0; i < priv->vector_num; i++) {
98 tqp_vectors = &priv->tqp_vector[i];
99
100 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
101 continue;
102
103 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
104 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
105 "%s-%s-%d", priv->netdev->name, "TxRx",
106 txrx_int_idx++);
107 txrx_int_idx++;
108 } else if (tqp_vectors->rx_group.ring) {
109 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
110 "%s-%s-%d", priv->netdev->name, "Rx",
111 rx_int_idx++);
112 } else if (tqp_vectors->tx_group.ring) {
113 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
114 "%s-%s-%d", priv->netdev->name, "Tx",
115 tx_int_idx++);
116 } else {
117 /* Skip this unused q_vector */
118 continue;
119 }
120
121 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
122
123 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
124 tqp_vectors->name,
125 tqp_vectors);
126 if (ret) {
127 netdev_err(priv->netdev, "request irq(%d) fail\n",
128 tqp_vectors->vector_irq);
129 return ret;
130 }
131
132 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
133 }
134
135 return 0;
136}
137
138static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
139 u32 mask_en)
140{
141 writel(mask_en, tqp_vector->mask_addr);
142}
143
144static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
145{
146 napi_enable(&tqp_vector->napi);
147
148 /* enable vector */
149 hns3_mask_vector_irq(tqp_vector, 1);
150}
151
152static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
153{
154 /* disable vector */
155 hns3_mask_vector_irq(tqp_vector, 0);
156
157 disable_irq(tqp_vector->vector_irq);
158 napi_disable(&tqp_vector->napi);
159}
160
434776a5
FL		 161void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
162 u32 rl_value)
76ad4f0e 163{
434776a5
FL		 164 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
165
76ad4f0e
S		 166 /* this defines the configuration for RL (Interrupt Rate Limiter).
167 * Rl defines rate of interrupts i.e. number of interrupts-per-second
168 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
169 */
434776a5
FL		 170
171 if (rl_reg > 0 && !tqp_vector->tx_group.gl_adapt_enable &&
172 !tqp_vector->rx_group.gl_adapt_enable)
173 /* According to the hardware, the range of rl_reg is
174 * 0-59 and the unit is 4.
175 */
176 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
177
178 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
179}
180
181void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
182 u32 gl_value)
183{
184 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
185
186 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
187}
188
189void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
190 u32 gl_value)
191{
192 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
193
194 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 195}
196
5fd4789a
FL		 197static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
198 struct hns3_nic_priv *priv)
76ad4f0e 199{
5fd4789a
FL		 200 struct hnae3_handle *h = priv->ae_handle;
201
76ad4f0e
S		 202 /* initialize the configuration for interrupt coalescing.
203 * 1. GL (Interrupt Gap Limiter)
204 * 2. RL (Interrupt Rate Limiter)
205 */
206
5fd4789a
FL		 207 /* Default: enable interrupt coalescing self-adaptive and GL */
208 tqp_vector->tx_group.gl_adapt_enable = 1;
209 tqp_vector->rx_group.gl_adapt_enable = 1;
210
76ad4f0e 211 tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
5fd4789a
FL		 212 tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
213
214 hns3_set_vector_coalesce_tx_gl(tqp_vector,
215 tqp_vector->tx_group.int_gl);
216 hns3_set_vector_coalesce_rx_gl(tqp_vector,
217 tqp_vector->rx_group.int_gl);
218
219 /* Default: disable RL */
220 h->kinfo.int_rl_setting = 0;
221 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
222
76ad4f0e
S		 223 tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
224 tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
225}
226
9df8f79a
YL		 227static int hns3_nic_set_real_num_queue(struct net_device *netdev)
228{
9780cb97 229 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 230 struct hnae3_knic_private_info *kinfo = &h->kinfo;
231 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
232 int ret;
233
234 ret = netif_set_real_num_tx_queues(netdev, queue_size);
235 if (ret) {
236 netdev_err(netdev,
237 "netif_set_real_num_tx_queues fail, ret=%d!\n",
238 ret);
239 return ret;
240 }
241
242 ret = netif_set_real_num_rx_queues(netdev, queue_size);
243 if (ret) {
244 netdev_err(netdev,
245 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
246 return ret;
247 }
248
249 return 0;
250}
251
76ad4f0e
S		 252static int hns3_nic_net_up(struct net_device *netdev)
253{
254 struct hns3_nic_priv *priv = netdev_priv(netdev);
255 struct hnae3_handle *h = priv->ae_handle;
256 int i, j;
257 int ret;
258
259 /* get irq resource for all vectors */
260 ret = hns3_nic_init_irq(priv);
261 if (ret) {
262 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
263 return ret;
264 }
265
266 /* enable the vectors */
267 for (i = 0; i < priv->vector_num; i++)
268 hns3_vector_enable(&priv->tqp_vector[i]);
269
270 /* start the ae_dev */
271 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
272 if (ret)
273 goto out_start_err;
274
b875cc37
JS		 275 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
276
76ad4f0e
S		 277 return 0;
278
279out_start_err:
280 for (j = i - 1; j >= 0; j--)
281 hns3_vector_disable(&priv->tqp_vector[j]);
282
283 hns3_nic_uninit_irq(priv);
284
285 return ret;
286}
287
288static int hns3_nic_net_open(struct net_device *netdev)
289{
f8fa222c 290 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 291 int ret;
292
293 netif_carrier_off(netdev);
294
9df8f79a
YL		 295 ret = hns3_nic_set_real_num_queue(netdev);
296 if (ret)
76ad4f0e 297 return ret;
76ad4f0e
S		 298
299 ret = hns3_nic_net_up(netdev);
300 if (ret) {
301 netdev_err(netdev,
302 "hns net up fail, ret=%d!\n", ret);
303 return ret;
304 }
305
f8fa222c 306 priv->last_reset_time = jiffies;
76ad4f0e
S		 307 return 0;
308}
309
310static void hns3_nic_net_down(struct net_device *netdev)
311{
312 struct hns3_nic_priv *priv = netdev_priv(netdev);
313 const struct hnae3_ae_ops *ops;
314 int i;
315
b875cc37
JS		 316 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
317 return;
318
76ad4f0e
S		 319 /* stop ae_dev */
320 ops = priv->ae_handle->ae_algo->ops;
321 if (ops->stop)
322 ops->stop(priv->ae_handle);
323
324 /* disable vectors */
325 for (i = 0; i < priv->vector_num; i++)
326 hns3_vector_disable(&priv->tqp_vector[i]);
327
328 /* free irq resources */
329 hns3_nic_uninit_irq(priv);
330}
331
332static int hns3_nic_net_stop(struct net_device *netdev)
333{
334 netif_tx_stop_all_queues(netdev);
335 netif_carrier_off(netdev);
336
337 hns3_nic_net_down(netdev);
338
339 return 0;
340}
341
76ad4f0e
S		 342static int hns3_nic_uc_sync(struct net_device *netdev,
343 const unsigned char *addr)
344{
9780cb97 345 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 346
347 if (h->ae_algo->ops->add_uc_addr)
348 return h->ae_algo->ops->add_uc_addr(h, addr);
349
350 return 0;
351}
352
353static int hns3_nic_uc_unsync(struct net_device *netdev,
354 const unsigned char *addr)
355{
9780cb97 356 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 357
358 if (h->ae_algo->ops->rm_uc_addr)
359 return h->ae_algo->ops->rm_uc_addr(h, addr);
360
361 return 0;
362}
363
364static int hns3_nic_mc_sync(struct net_device *netdev,
365 const unsigned char *addr)
366{
9780cb97 367 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 368
720a8478 369 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 370 return h->ae_algo->ops->add_mc_addr(h, addr);
371
372 return 0;
373}
374
375static int hns3_nic_mc_unsync(struct net_device *netdev,
376 const unsigned char *addr)
377{
9780cb97 378 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 379
720a8478 380 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 381 return h->ae_algo->ops->rm_mc_addr(h, addr);
382
383 return 0;
384}
385
1db9b1bf 386static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 387{
9780cb97 388 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 389
390 if (h->ae_algo->ops->set_promisc_mode) {
391 if (netdev->flags & IFF_PROMISC)
392 h->ae_algo->ops->set_promisc_mode(h, 1);
393 else
394 h->ae_algo->ops->set_promisc_mode(h, 0);
395 }
396 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
397 netdev_err(netdev, "sync uc address fail\n");
398 if (netdev->flags & IFF_MULTICAST)
399 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
400 netdev_err(netdev, "sync mc address fail\n");
401}
402
403static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
404 u16 *mss, u32 *type_cs_vlan_tso)
405{
406 u32 l4_offset, hdr_len;
407 union l3_hdr_info l3;
408 union l4_hdr_info l4;
409 u32 l4_paylen;
410 int ret;
411
412 if (!skb_is_gso(skb))
413 return 0;
414
415 ret = skb_cow_head(skb, 0);
416 if (ret)
417 return ret;
418
419 l3.hdr = skb_network_header(skb);
420 l4.hdr = skb_transport_header(skb);
421
422 /* Software should clear the IPv4's checksum field when tso is
423 * needed.
424 */
425 if (l3.v4->version == 4)
426 l3.v4->check = 0;
427
428 /* tunnel packet.*/
429 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
430 SKB_GSO_GRE_CSUM |
431 SKB_GSO_UDP_TUNNEL |
432 SKB_GSO_UDP_TUNNEL_CSUM)) {
433 if ((!(skb_shinfo(skb)->gso_type &
434 SKB_GSO_PARTIAL)) &&
435 (skb_shinfo(skb)->gso_type &
436 SKB_GSO_UDP_TUNNEL_CSUM)) {
437 /* Software should clear the udp's checksum
438 * field when tso is needed.
439 */
440 l4.udp->check = 0;
441 }
442 /* reset l3&l4 pointers from outer to inner headers */
443 l3.hdr = skb_inner_network_header(skb);
444 l4.hdr = skb_inner_transport_header(skb);
445
446 /* Software should clear the IPv4's checksum field when
447 * tso is needed.
448 */
449 if (l3.v4->version == 4)
450 l3.v4->check = 0;
451 }
452
453 /* normal or tunnel packet*/
454 l4_offset = l4.hdr - skb->data;
455 hdr_len = (l4.tcp->doff * 4) + l4_offset;
456
457 /* remove payload length from inner pseudo checksum when tso*/
458 l4_paylen = skb->len - l4_offset;
459 csum_replace_by_diff(&l4.tcp->check,
460 (__force __wsum)htonl(l4_paylen));
461
462 /* find the txbd field values */
463 *paylen = skb->len - hdr_len;
464 hnae_set_bit(*type_cs_vlan_tso,
465 HNS3_TXD_TSO_B, 1);
466
467 /* get MSS for TSO */
468 *mss = skb_shinfo(skb)->gso_size;
469
470 return 0;
471}
472
1898d4e4
S		 473static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
474 u8 *il4_proto)
76ad4f0e
S		 475{
476 union {
477 struct iphdr *v4;
478 struct ipv6hdr *v6;
479 unsigned char *hdr;
480 } l3;
481 unsigned char *l4_hdr;
482 unsigned char *exthdr;
483 u8 l4_proto_tmp;
484 __be16 frag_off;
485
486 /* find outer header point */
487 l3.hdr = skb_network_header(skb);
488 l4_hdr = skb_inner_transport_header(skb);
489
490 if (skb->protocol == htons(ETH_P_IPV6)) {
491 exthdr = l3.hdr + sizeof(*l3.v6);
492 l4_proto_tmp = l3.v6->nexthdr;
493 if (l4_hdr != exthdr)
494 ipv6_skip_exthdr(skb, exthdr - skb->data,
495 &l4_proto_tmp, &frag_off);
496 } else if (skb->protocol == htons(ETH_P_IP)) {
497 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 498 } else {
499 return -EINVAL;
76ad4f0e
S		 500 }
501
502 *ol4_proto = l4_proto_tmp;
503
504 /* tunnel packet */
505 if (!skb->encapsulation) {
506 *il4_proto = 0;
1898d4e4 507 return 0;
76ad4f0e
S		 508 }
509
510 /* find inner header point */
511 l3.hdr = skb_inner_network_header(skb);
512 l4_hdr = skb_inner_transport_header(skb);
513
514 if (l3.v6->version == 6) {
515 exthdr = l3.hdr + sizeof(*l3.v6);
516 l4_proto_tmp = l3.v6->nexthdr;
517 if (l4_hdr != exthdr)
518 ipv6_skip_exthdr(skb, exthdr - skb->data,
519 &l4_proto_tmp, &frag_off);
520 } else if (l3.v4->version == 4) {
521 l4_proto_tmp = l3.v4->protocol;
522 }
523
524 *il4_proto = l4_proto_tmp;
1898d4e4
S		 525
526 return 0;
76ad4f0e
S		 527}
528
529static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
530 u8 il4_proto, u32 *type_cs_vlan_tso,
531 u32 *ol_type_vlan_len_msec)
532{
533 union {
534 struct iphdr *v4;
535 struct ipv6hdr *v6;
536 unsigned char *hdr;
537 } l3;
538 union {
539 struct tcphdr *tcp;
540 struct udphdr *udp;
541 struct gre_base_hdr *gre;
542 unsigned char *hdr;
543 } l4;
544 unsigned char *l2_hdr;
545 u8 l4_proto = ol4_proto;
546 u32 ol2_len;
547 u32 ol3_len;
548 u32 ol4_len;
549 u32 l2_len;
550 u32 l3_len;
551
552 l3.hdr = skb_network_header(skb);
553 l4.hdr = skb_transport_header(skb);
554
555 /* compute L2 header size for normal packet, defined in 2 Bytes */
556 l2_len = l3.hdr - skb->data;
557 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
558 HNS3_TXD_L2LEN_S, l2_len >> 1);
559
560 /* tunnel packet*/
561 if (skb->encapsulation) {
562 /* compute OL2 header size, defined in 2 Bytes */
563 ol2_len = l2_len;
564 hnae_set_field(*ol_type_vlan_len_msec,
565 HNS3_TXD_L2LEN_M,
566 HNS3_TXD_L2LEN_S, ol2_len >> 1);
567
568 /* compute OL3 header size, defined in 4 Bytes */
569 ol3_len = l4.hdr - l3.hdr;
570 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
571 HNS3_TXD_L3LEN_S, ol3_len >> 2);
572
573 /* MAC in UDP, MAC in GRE (0x6558)*/
574 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
575 /* switch MAC header ptr from outer to inner header.*/
576 l2_hdr = skb_inner_mac_header(skb);
577
578 /* compute OL4 header size, defined in 4 Bytes. */
579 ol4_len = l2_hdr - l4.hdr;
580 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
581 HNS3_TXD_L4LEN_S, ol4_len >> 2);
582
583 /* switch IP header ptr from outer to inner header */
584 l3.hdr = skb_inner_network_header(skb);
585
586 /* compute inner l2 header size, defined in 2 Bytes. */
587 l2_len = l3.hdr - l2_hdr;
588 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
589 HNS3_TXD_L2LEN_S, l2_len >> 1);
590 } else {
591 /* skb packet types not supported by hardware,
592 * txbd len fild doesn't be filled.
593 */
594 return;
595 }
596
597 /* switch L4 header pointer from outer to inner */
598 l4.hdr = skb_inner_transport_header(skb);
599
600 l4_proto = il4_proto;
601 }
602
603 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
604 l3_len = l4.hdr - l3.hdr;
605 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
606 HNS3_TXD_L3LEN_S, l3_len >> 2);
607
608 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
609 switch (l4_proto) {
610 case IPPROTO_TCP:
611 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
612 HNS3_TXD_L4LEN_S, l4.tcp->doff);
613 break;
614 case IPPROTO_SCTP:
615 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
616 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
617 break;
618 case IPPROTO_UDP:
619 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
620 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
621 break;
622 default:
623 /* skb packet types not supported by hardware,
624 * txbd len fild doesn't be filled.
625 */
626 return;
627 }
628}
629
630static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
631 u8 il4_proto, u32 *type_cs_vlan_tso,
632 u32 *ol_type_vlan_len_msec)
633{
634 union {
635 struct iphdr *v4;
636 struct ipv6hdr *v6;
637 unsigned char *hdr;
638 } l3;
639 u32 l4_proto = ol4_proto;
640
641 l3.hdr = skb_network_header(skb);
642
643 /* define OL3 type and tunnel type(OL4).*/
644 if (skb->encapsulation) {
645 /* define outer network header type.*/
646 if (skb->protocol == htons(ETH_P_IP)) {
647 if (skb_is_gso(skb))
648 hnae_set_field(*ol_type_vlan_len_msec,
649 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
650 HNS3_OL3T_IPV4_CSUM);
651 else
652 hnae_set_field(*ol_type_vlan_len_msec,
653 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
654 HNS3_OL3T_IPV4_NO_CSUM);
655
656 } else if (skb->protocol == htons(ETH_P_IPV6)) {
657 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
658 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
659 }
660
661 /* define tunnel type(OL4).*/
662 switch (l4_proto) {
663 case IPPROTO_UDP:
664 hnae_set_field(*ol_type_vlan_len_msec,
665 HNS3_TXD_TUNTYPE_M,
666 HNS3_TXD_TUNTYPE_S,
667 HNS3_TUN_MAC_IN_UDP);
668 break;
669 case IPPROTO_GRE:
670 hnae_set_field(*ol_type_vlan_len_msec,
671 HNS3_TXD_TUNTYPE_M,
672 HNS3_TXD_TUNTYPE_S,
673 HNS3_TUN_NVGRE);
674 break;
675 default:
676 /* drop the skb tunnel packet if hardware don't support,
677 * because hardware can't calculate csum when TSO.
678 */
679 if (skb_is_gso(skb))
680 return -EDOM;
681
682 /* the stack computes the IP header already,
683 * driver calculate l4 checksum when not TSO.
684 */
685 skb_checksum_help(skb);
686 return 0;
687 }
688
689 l3.hdr = skb_inner_network_header(skb);
690 l4_proto = il4_proto;
691 }
692
693 if (l3.v4->version == 4) {
694 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
695 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
696
697 /* the stack computes the IP header already, the only time we
698 * need the hardware to recompute it is in the case of TSO.
699 */
700 if (skb_is_gso(skb))
701 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
702
703 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
704 } else if (l3.v6->version == 6) {
705 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
706 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
707 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
708 }
709
710 switch (l4_proto) {
711 case IPPROTO_TCP:
712 hnae_set_field(*type_cs_vlan_tso,
713 HNS3_TXD_L4T_M,
714 HNS3_TXD_L4T_S,
715 HNS3_L4T_TCP);
716 break;
717 case IPPROTO_UDP:
718 hnae_set_field(*type_cs_vlan_tso,
719 HNS3_TXD_L4T_M,
720 HNS3_TXD_L4T_S,
721 HNS3_L4T_UDP);
722 break;
723 case IPPROTO_SCTP:
724 hnae_set_field(*type_cs_vlan_tso,
725 HNS3_TXD_L4T_M,
726 HNS3_TXD_L4T_S,
727 HNS3_L4T_SCTP);
728 break;
729 default:
730 /* drop the skb tunnel packet if hardware don't support,
731 * because hardware can't calculate csum when TSO.
732 */
733 if (skb_is_gso(skb))
734 return -EDOM;
735
736 /* the stack computes the IP header already,
737 * driver calculate l4 checksum when not TSO.
738 */
739 skb_checksum_help(skb);
740 return 0;
741 }
742
743 return 0;
744}
745
746static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
747{
748 /* Config bd buffer end */
749 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
750 HNS3_TXD_BDTYPE_M, 0);
751 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
752 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
7036d26f 753 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 754}
755
9699cffe
PL		 756static int hns3_fill_desc_vtags(struct sk_buff *skb,
757 struct hns3_enet_ring *tx_ring,
758 u32 *inner_vlan_flag,
759 u32 *out_vlan_flag,
760 u16 *inner_vtag,
761 u16 *out_vtag)
762{
763#define HNS3_TX_VLAN_PRIO_SHIFT 13
764
765 if (skb->protocol == htons(ETH_P_8021Q) &&
766 !(tx_ring->tqp->handle->kinfo.netdev->features &
767 NETIF_F_HW_VLAN_CTAG_TX)) {
768 /* When HW VLAN acceleration is turned off, and the stack
769 * sets the protocol to 802.1q, the driver just need to
770 * set the protocol to the encapsulated ethertype.
771 */
772 skb->protocol = vlan_get_protocol(skb);
773 return 0;
774 }
775
776 if (skb_vlan_tag_present(skb)) {
777 u16 vlan_tag;
778
779 vlan_tag = skb_vlan_tag_get(skb);
780 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
781
782 /* Based on hw strategy, use out_vtag in two layer tag case,
783 * and use inner_vtag in one tag case.
784 */
785 if (skb->protocol == htons(ETH_P_8021Q)) {
786 hnae_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
787 *out_vtag = vlan_tag;
788 } else {
789 hnae_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
790 *inner_vtag = vlan_tag;
791 }
792 } else if (skb->protocol == htons(ETH_P_8021Q)) {
793 struct vlan_ethhdr *vhdr;
794 int rc;
795
796 rc = skb_cow_head(skb, 0);
797 if (rc < 0)
798 return rc;
799 vhdr = (struct vlan_ethhdr *)skb->data;
800 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
801 << HNS3_TX_VLAN_PRIO_SHIFT);
802 }
803
804 skb->protocol = vlan_get_protocol(skb);
805 return 0;
806}
807
76ad4f0e
S		 808static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
809 int size, dma_addr_t dma, int frag_end,
810 enum hns_desc_type type)
811{
812 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
813 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
814 u32 ol_type_vlan_len_msec = 0;
815 u16 bdtp_fe_sc_vld_ra_ri = 0;
816 u32 type_cs_vlan_tso = 0;
817 struct sk_buff *skb;
9699cffe
PL		 818 u16 inner_vtag = 0;
819 u16 out_vtag = 0;
76ad4f0e
S		 820 u32 paylen = 0;
821 u16 mss = 0;
822 __be16 protocol;
823 u8 ol4_proto;
824 u8 il4_proto;
825 int ret;
826
827 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
828 desc_cb->priv = priv;
829 desc_cb->length = size;
830 desc_cb->dma = dma;
831 desc_cb->type = type;
832
833 /* now, fill the descriptor */
834 desc->addr = cpu_to_le64(dma);
835 desc->tx.send_size = cpu_to_le16((u16)size);
836 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
837 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
838
839 if (type == DESC_TYPE_SKB) {
840 skb = (struct sk_buff *)priv;
a90bb9a5 841 paylen = skb->len;
76ad4f0e 842
9699cffe
PL		 843 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
844 &ol_type_vlan_len_msec,
845 &inner_vtag, &out_vtag);
846 if (unlikely(ret))
847 return ret;
848
76ad4f0e
S		 849 if (skb->ip_summed == CHECKSUM_PARTIAL) {
850 skb_reset_mac_len(skb);
851 protocol = skb->protocol;
852
1898d4e4
S		 853 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
854 if (ret)
855 return ret;
76ad4f0e
S		 856 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
857 &type_cs_vlan_tso,
858 &ol_type_vlan_len_msec);
859 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
860 &type_cs_vlan_tso,
861 &ol_type_vlan_len_msec);
862 if (ret)
863 return ret;
864
865 ret = hns3_set_tso(skb, &paylen, &mss,
866 &type_cs_vlan_tso);
867 if (ret)
868 return ret;
869 }
870
871 /* Set txbd */
872 desc->tx.ol_type_vlan_len_msec =
873 cpu_to_le32(ol_type_vlan_len_msec);
874 desc->tx.type_cs_vlan_tso_len =
875 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 876 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 877 desc->tx.mss = cpu_to_le16(mss);
9699cffe
PL		 878 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
879 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
76ad4f0e
S		 880 }
881
882 /* move ring pointer to next.*/
883 ring_ptr_move_fw(ring, next_to_use);
884
885 return 0;
886}
887
888static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
889 int size, dma_addr_t dma, int frag_end,
890 enum hns_desc_type type)
891{
892 unsigned int frag_buf_num;
893 unsigned int k;
894 int sizeoflast;
895 int ret;
896
897 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
898 sizeoflast = size % HNS3_MAX_BD_SIZE;
899 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
900
901 /* When the frag size is bigger than hardware, split this frag */
902 for (k = 0; k < frag_buf_num; k++) {
903 ret = hns3_fill_desc(ring, priv,
904 (k == frag_buf_num - 1) ?
905 sizeoflast : HNS3_MAX_BD_SIZE,
906 dma + HNS3_MAX_BD_SIZE * k,
907 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
908 (type == DESC_TYPE_SKB && !k) ?
909 DESC_TYPE_SKB : DESC_TYPE_PAGE);
910 if (ret)
911 return ret;
912 }
913
914 return 0;
915}
916
917static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
918 struct hns3_enet_ring *ring)
919{
920 struct sk_buff *skb = *out_skb;
921 struct skb_frag_struct *frag;
922 int bdnum_for_frag;
923 int frag_num;
924 int buf_num;
925 int size;
926 int i;
927
928 size = skb_headlen(skb);
929 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
930
931 frag_num = skb_shinfo(skb)->nr_frags;
932 for (i = 0; i < frag_num; i++) {
933 frag = &skb_shinfo(skb)->frags[i];
934 size = skb_frag_size(frag);
935 bdnum_for_frag =
936 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
937 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
938 return -ENOMEM;
939
940 buf_num += bdnum_for_frag;
941 }
942
943 if (buf_num > ring_space(ring))
944 return -EBUSY;
945
946 *bnum = buf_num;
947 return 0;
948}
949
950static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
951 struct hns3_enet_ring *ring)
952{
953 struct sk_buff *skb = *out_skb;
954 int buf_num;
955
956 /* No. of segments (plus a header) */
957 buf_num = skb_shinfo(skb)->nr_frags + 1;
958
959 if (buf_num > ring_space(ring))
960 return -EBUSY;
961
962 *bnum = buf_num;
963
964 return 0;
965}
966
967static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
968{
969 struct device *dev = ring_to_dev(ring);
970 unsigned int i;
971
972 for (i = 0; i < ring->desc_num; i++) {
973 /* check if this is where we started */
974 if (ring->next_to_use == next_to_use_orig)
975 break;
976
977 /* unmap the descriptor dma address */
978 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
979 dma_unmap_single(dev,
980 ring->desc_cb[ring->next_to_use].dma,
981 ring->desc_cb[ring->next_to_use].length,
982 DMA_TO_DEVICE);
983 else
984 dma_unmap_page(dev,
985 ring->desc_cb[ring->next_to_use].dma,
986 ring->desc_cb[ring->next_to_use].length,
987 DMA_TO_DEVICE);
988
989 /* rollback one */
990 ring_ptr_move_bw(ring, next_to_use);
991 }
992}
993
d43e5aca 994netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 995{
996 struct hns3_nic_priv *priv = netdev_priv(netdev);
997 struct hns3_nic_ring_data *ring_data =
998 &tx_ring_data(priv, skb->queue_mapping);
999 struct hns3_enet_ring *ring = ring_data->ring;
1000 struct device *dev = priv->dev;
1001 struct netdev_queue *dev_queue;
1002 struct skb_frag_struct *frag;
1003 int next_to_use_head;
1004 int next_to_use_frag;
1005 dma_addr_t dma;
1006 int buf_num;
1007 int seg_num;
1008 int size;
1009 int ret;
1010 int i;
1011
1012 /* Prefetch the data used later */
1013 prefetch(skb->data);
1014
1015 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1016 case -EBUSY:
1017 u64_stats_update_begin(&ring->syncp);
1018 ring->stats.tx_busy++;
1019 u64_stats_update_end(&ring->syncp);
1020
1021 goto out_net_tx_busy;
1022 case -ENOMEM:
1023 u64_stats_update_begin(&ring->syncp);
1024 ring->stats.sw_err_cnt++;
1025 u64_stats_update_end(&ring->syncp);
1026 netdev_err(netdev, "no memory to xmit!\n");
1027
1028 goto out_err_tx_ok;
1029 default:
1030 break;
1031 }
1032
1033 /* No. of segments (plus a header) */
1034 seg_num = skb_shinfo(skb)->nr_frags + 1;
1035 /* Fill the first part */
1036 size = skb_headlen(skb);
1037
1038 next_to_use_head = ring->next_to_use;
1039
1040 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1041 if (dma_mapping_error(dev, dma)) {
1042 netdev_err(netdev, "TX head DMA map failed\n");
1043 ring->stats.sw_err_cnt++;
1044 goto out_err_tx_ok;
1045 }
1046
1047 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
1048 DESC_TYPE_SKB);
1049 if (ret)
1050 goto head_dma_map_err;
1051
1052 next_to_use_frag = ring->next_to_use;
1053 /* Fill the fragments */
1054 for (i = 1; i < seg_num; i++) {
1055 frag = &skb_shinfo(skb)->frags[i - 1];
1056 size = skb_frag_size(frag);
1057 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1058 if (dma_mapping_error(dev, dma)) {
1059 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
1060 ring->stats.sw_err_cnt++;
1061 goto frag_dma_map_err;
1062 }
1063 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
1064 seg_num - 1 == i ? 1 : 0,
1065 DESC_TYPE_PAGE);
1066
1067 if (ret)
1068 goto frag_dma_map_err;
1069 }
1070
1071 /* Complete translate all packets */
1072 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1073 netdev_tx_sent_queue(dev_queue, skb->len);
1074
1075 wmb(); /* Commit all data before submit */
1076
1077 hnae_queue_xmit(ring->tqp, buf_num);
1078
1079 return NETDEV_TX_OK;
1080
1081frag_dma_map_err:
1082 hns_nic_dma_unmap(ring, next_to_use_frag);
1083
1084head_dma_map_err:
1085 hns_nic_dma_unmap(ring, next_to_use_head);
1086
1087out_err_tx_ok:
1088 dev_kfree_skb_any(skb);
1089 return NETDEV_TX_OK;
1090
1091out_net_tx_busy:
1092 netif_stop_subqueue(netdev, ring_data->queue_index);
1093 smp_mb(); /* Commit all data before submit */
1094
1095 return NETDEV_TX_BUSY;
1096}
1097
1098static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1099{
9780cb97 1100 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1101 struct sockaddr *mac_addr = p;
1102 int ret;
1103
1104 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1105 return -EADDRNOTAVAIL;
1106
1107 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
1108 if (ret) {
1109 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1110 return ret;
1111 }
1112
1113 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1114
1115 return 0;
1116}
1117
1118static int hns3_nic_set_features(struct net_device *netdev,
1119 netdev_features_t features)
1120{
1121 struct hns3_nic_priv *priv = netdev_priv(netdev);
052ece6d
PL		 1122 struct hnae3_handle *h = priv->ae_handle;
1123 netdev_features_t changed;
1124 int ret;
76ad4f0e
S		 1125
1126 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1127 priv->ops.fill_desc = hns3_fill_desc_tso;
1128 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1129 } else {
1130 priv->ops.fill_desc = hns3_fill_desc;
1131 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1132 }
1133
391b5e93
JS		 1134 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1135 h->ae_algo->ops->enable_vlan_filter(h, true);
1136 else
1137 h->ae_algo->ops->enable_vlan_filter(h, false);
1138
052ece6d
PL		 1139 changed = netdev->features ^ features;
1140 if (changed & NETIF_F_HW_VLAN_CTAG_RX) {
1141 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1142 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1143 else
1144 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1145
1146 if (ret)
1147 return ret;
1148 }
1149
76ad4f0e
S		 1150 netdev->features = features;
1151 return 0;
1152}
1153
6c88d9d7
PL		 1154static void hns3_nic_get_stats64(struct net_device *netdev,
1155 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1156{
1157 struct hns3_nic_priv *priv = netdev_priv(netdev);
1158 int queue_num = priv->ae_handle->kinfo.num_tqps;
c5f65480 1159 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1160 struct hns3_enet_ring *ring;
1161 unsigned int start;
1162 unsigned int idx;
1163 u64 tx_bytes = 0;
1164 u64 rx_bytes = 0;
1165 u64 tx_pkts = 0;
1166 u64 rx_pkts = 0;
d2a5dca8
JS		 1167 u64 tx_drop = 0;
1168 u64 rx_drop = 0;
76ad4f0e 1169
b875cc37
JS		 1170 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1171 return;
1172
c5f65480
JS		 1173 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1174
76ad4f0e
S		 1175 for (idx = 0; idx < queue_num; idx++) {
1176 /* fetch the tx stats */
1177 ring = priv->ring_data[idx].ring;
1178 do {
d36d36ce 1179 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1180 tx_bytes += ring->stats.tx_bytes;
1181 tx_pkts += ring->stats.tx_pkts;
d2a5dca8
JS		 1182 tx_drop += ring->stats.tx_busy;
1183 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1184 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1185
1186 /* fetch the rx stats */
1187 ring = priv->ring_data[idx + queue_num].ring;
1188 do {
d36d36ce 1189 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1190 rx_bytes += ring->stats.rx_bytes;
1191 rx_pkts += ring->stats.rx_pkts;
d2a5dca8
JS		 1192 rx_drop += ring->stats.non_vld_descs;
1193 rx_drop += ring->stats.err_pkt_len;
1194 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1195 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1196 }
1197
1198 stats->tx_bytes = tx_bytes;
1199 stats->tx_packets = tx_pkts;
1200 stats->rx_bytes = rx_bytes;
1201 stats->rx_packets = rx_pkts;
1202
1203 stats->rx_errors = netdev->stats.rx_errors;
1204 stats->multicast = netdev->stats.multicast;
1205 stats->rx_length_errors = netdev->stats.rx_length_errors;
1206 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1207 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1208
1209 stats->tx_errors = netdev->stats.tx_errors;
d2a5dca8
JS		 1210 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1211 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1212 stats->collisions = netdev->stats.collisions;
1213 stats->rx_over_errors = netdev->stats.rx_over_errors;
1214 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1215 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1216 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1217 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1218 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1219 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1220 stats->tx_window_errors = netdev->stats.tx_window_errors;
1221 stats->rx_compressed = netdev->stats.rx_compressed;
1222 stats->tx_compressed = netdev->stats.tx_compressed;
1223}
1224
1225static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1226 enum hns3_udp_tnl_type type)
1227{
1228 struct hns3_nic_priv *priv = netdev_priv(netdev);
1229 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1230 struct hnae3_handle *h = priv->ae_handle;
1231
1232 if (udp_tnl->used && udp_tnl->dst_port == port) {
1233 udp_tnl->used++;
1234 return;
1235 }
1236
1237 if (udp_tnl->used) {
1238 netdev_warn(netdev,
1239 "UDP tunnel [%d], port [%d] offload\n", type, port);
1240 return;
1241 }
1242
1243 udp_tnl->dst_port = port;
1244 udp_tnl->used = 1;
1245 /* TBD send command to hardware to add port */
1246 if (h->ae_algo->ops->add_tunnel_udp)
1247 h->ae_algo->ops->add_tunnel_udp(h, port);
1248}
1249
1250static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1251 enum hns3_udp_tnl_type type)
1252{
1253 struct hns3_nic_priv *priv = netdev_priv(netdev);
1254 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1255 struct hnae3_handle *h = priv->ae_handle;
1256
1257 if (!udp_tnl->used || udp_tnl->dst_port != port) {
1258 netdev_warn(netdev,
1259 "Invalid UDP tunnel port %d\n", port);
1260 return;
1261 }
1262
1263 udp_tnl->used--;
1264 if (udp_tnl->used)
1265 return;
1266
1267 udp_tnl->dst_port = 0;
1268 /* TBD send command to hardware to del port */
1269 if (h->ae_algo->ops->del_tunnel_udp)
9537e7cb 1270 h->ae_algo->ops->del_tunnel_udp(h, port);
76ad4f0e
S		 1271}
1272
1273/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1274 * @netdev: This physical ports's netdev
1275 * @ti: Tunnel information
1276 */
1277static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1278 struct udp_tunnel_info *ti)
1279{
1280 u16 port_n = ntohs(ti->port);
1281
1282 switch (ti->type) {
1283 case UDP_TUNNEL_TYPE_VXLAN:
1284 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1285 break;
1286 case UDP_TUNNEL_TYPE_GENEVE:
1287 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1288 break;
1289 default:
1290 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1291 break;
1292 }
1293}
1294
1295static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1296 struct udp_tunnel_info *ti)
1297{
1298 u16 port_n = ntohs(ti->port);
1299
1300 switch (ti->type) {
1301 case UDP_TUNNEL_TYPE_VXLAN:
1302 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1303 break;
1304 case UDP_TUNNEL_TYPE_GENEVE:
1305 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1306 break;
1307 default:
1308 break;
1309 }
1310}
1311
30d240df 1312static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1313{
30d240df 1314 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1315 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1316 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1317 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1318 u8 tc = mqprio_qopt->qopt.num_tc;
1319 u16 mode = mqprio_qopt->mode;
1320 u8 hw = mqprio_qopt->qopt.hw;
1321 bool if_running;
76ad4f0e
S		 1322 unsigned int i;
1323 int ret;
1324
30d240df
YL		 1325 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1326 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1327 return -EOPNOTSUPP;
1328
76ad4f0e
S		 1329 if (tc > HNAE3_MAX_TC)
1330 return -EINVAL;
1331
76ad4f0e
S		 1332 if (!netdev)
1333 return -EINVAL;
1334
30d240df
YL		 1335 if_running = netif_running(netdev);
1336 if (if_running) {
1337 hns3_nic_net_stop(netdev);
1338 msleep(100);
76ad4f0e
S		 1339 }
1340
30d240df
YL		 1341 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1342 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1343 if (ret)
30d240df
YL		 1344 goto out;
1345
1346 if (tc <= 1) {
1347 netdev_reset_tc(netdev);
1348 } else {
1349 ret = netdev_set_num_tc(netdev, tc);
1350 if (ret)
1351 goto out;
1352
1353 for (i = 0; i < HNAE3_MAX_TC; i++) {
1354 if (!kinfo->tc_info[i].enable)
1355 continue;
76ad4f0e 1356
76ad4f0e
S		 1357 netdev_set_tc_queue(netdev,
1358 kinfo->tc_info[i].tc,
1359 kinfo->tc_info[i].tqp_count,
1360 kinfo->tc_info[i].tqp_offset);
30d240df 1361 }
76ad4f0e
S		 1362 }
1363
30d240df
YL		 1364 ret = hns3_nic_set_real_num_queue(netdev);
1365
1366out:
1367 if (if_running)
1368 hns3_nic_net_open(netdev);
1369
1370 return ret;
76ad4f0e
S		 1371}
1372
2572ac53 1373static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1374 void *type_data)
76ad4f0e 1375{
575ed7d3 1376 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1377 return -EOPNOTSUPP;
76ad4f0e 1378
30d240df 1379 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1380}
1381
1382static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1383 __be16 proto, u16 vid)
1384{
9780cb97 1385 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1386 int ret = -EIO;
1387
1388 if (h->ae_algo->ops->set_vlan_filter)
1389 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1390
1391 return ret;
1392}
1393
1394static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1395 __be16 proto, u16 vid)
1396{
9780cb97 1397 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1398 int ret = -EIO;
1399
1400 if (h->ae_algo->ops->set_vlan_filter)
1401 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1402
1403 return ret;
1404}
1405
1406static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1407 u8 qos, __be16 vlan_proto)
1408{
9780cb97 1409 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1410 int ret = -EIO;
1411
1412 if (h->ae_algo->ops->set_vf_vlan_filter)
1413 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1414 qos, vlan_proto);
1415
1416 return ret;
1417}
1418
a8e8b7ff
S		 1419static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1420{
9780cb97 1421 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1422 bool if_running = netif_running(netdev);
1423 int ret;
1424
1425 if (!h->ae_algo->ops->set_mtu)
1426 return -EOPNOTSUPP;
1427
1428 /* if this was called with netdev up then bring netdevice down */
1429 if (if_running) {
1430 (void)hns3_nic_net_stop(netdev);
1431 msleep(100);
1432 }
1433
1434 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1435 if (ret) {
1436 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1437 ret);
1438 return ret;
1439 }
1440
5bad95a1
FL		 1441 netdev->mtu = new_mtu;
1442
a8e8b7ff
S		 1443 /* if the netdev was running earlier, bring it up again */
1444 if (if_running && hns3_nic_net_open(netdev))
1445 ret = -EINVAL;
1446
1447 return ret;
1448}
1449
f8fa222c
L		 1450static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1451{
1452 struct hns3_nic_priv *priv = netdev_priv(ndev);
1453 struct hns3_enet_ring *tx_ring = NULL;
1454 int timeout_queue = 0;
1455 int hw_head, hw_tail;
1456 int i;
1457
1458 /* Find the stopped queue the same way the stack does */
1459 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1460 struct netdev_queue *q;
1461 unsigned long trans_start;
1462
1463 q = netdev_get_tx_queue(ndev, i);
1464 trans_start = q->trans_start;
1465 if (netif_xmit_stopped(q) &&
1466 time_after(jiffies,
1467 (trans_start + ndev->watchdog_timeo))) {
1468 timeout_queue = i;
1469 break;
1470 }
1471 }
1472
1473 if (i == ndev->num_tx_queues) {
1474 netdev_info(ndev,
1475 "no netdev TX timeout queue found, timeout count: %llu\n",
1476 priv->tx_timeout_count);
1477 return false;
1478 }
1479
1480 tx_ring = priv->ring_data[timeout_queue].ring;
1481
1482 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1483 HNS3_RING_TX_RING_HEAD_REG);
1484 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1485 HNS3_RING_TX_RING_TAIL_REG);
1486 netdev_info(ndev,
1487 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1488 priv->tx_timeout_count,
1489 timeout_queue,
1490 tx_ring->next_to_use,
1491 tx_ring->next_to_clean,
1492 hw_head,
1493 hw_tail,
1494 readl(tx_ring->tqp_vector->mask_addr));
1495
1496 return true;
1497}
1498
1499static void hns3_nic_net_timeout(struct net_device *ndev)
1500{
1501 struct hns3_nic_priv *priv = netdev_priv(ndev);
1502 unsigned long last_reset_time = priv->last_reset_time;
1503 struct hnae3_handle *h = priv->ae_handle;
1504
1505 if (!hns3_get_tx_timeo_queue_info(ndev))
1506 return;
1507
1508 priv->tx_timeout_count++;
1509
1510 /* This timeout is far away enough from last timeout,
1511 * if timeout again,set the reset type to PF reset
1512 */
1513 if (time_after(jiffies, (last_reset_time + 20 * HZ)))
1514 priv->reset_level = HNAE3_FUNC_RESET;
1515
1516 /* Don't do any new action before the next timeout */
1517 else if (time_before(jiffies, (last_reset_time + ndev->watchdog_timeo)))
1518 return;
1519
1520 priv->last_reset_time = jiffies;
1521
1522 if (h->ae_algo->ops->reset_event)
1523 h->ae_algo->ops->reset_event(h, priv->reset_level);
1524
1525 priv->reset_level++;
1526 if (priv->reset_level > HNAE3_GLOBAL_RESET)
1527 priv->reset_level = HNAE3_GLOBAL_RESET;
1528}
1529
76ad4f0e
S		 1530static const struct net_device_ops hns3_nic_netdev_ops = {
1531 .ndo_open = hns3_nic_net_open,
1532 .ndo_stop = hns3_nic_net_stop,
1533 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1534 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1535 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a8e8b7ff 1536 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1537 .ndo_set_features = hns3_nic_set_features,
1538 .ndo_get_stats64 = hns3_nic_get_stats64,
1539 .ndo_setup_tc = hns3_nic_setup_tc,
1540 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
1541 .ndo_udp_tunnel_add = hns3_nic_udp_tunnel_add,
1542 .ndo_udp_tunnel_del = hns3_nic_udp_tunnel_del,
1543 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1544 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1545 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1546};
1547
1548/* hns3_probe - Device initialization routine
1549 * @pdev: PCI device information struct
1550 * @ent: entry in hns3_pci_tbl
1551 *
1552 * hns3_probe initializes a PF identified by a pci_dev structure.
1553 * The OS initialization, configuring of the PF private structure,
1554 * and a hardware reset occur.
1555 *
1556 * Returns 0 on success, negative on failure
1557 */
1558static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1559{
1560 struct hnae3_ae_dev *ae_dev;
1561 int ret;
1562
1563 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1564 GFP_KERNEL);
1565 if (!ae_dev) {
1566 ret = -ENOMEM;
1567 return ret;
1568 }
1569
1570 ae_dev->pdev = pdev;
e92a0843 1571 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 1572 ae_dev->dev_type = HNAE3_DEV_KNIC;
1573 pci_set_drvdata(pdev, ae_dev);
1574
1575 return hnae3_register_ae_dev(ae_dev);
1576}
1577
1578/* hns3_remove - Device removal routine
1579 * @pdev: PCI device information struct
1580 */
1581static void hns3_remove(struct pci_dev *pdev)
1582{
1583 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1584
1585 hnae3_unregister_ae_dev(ae_dev);
1586
1587 devm_kfree(&pdev->dev, ae_dev);
1588
1589 pci_set_drvdata(pdev, NULL);
1590}
1591
1592static struct pci_driver hns3_driver = {
1593 .name = hns3_driver_name,
1594 .id_table = hns3_pci_tbl,
1595 .probe = hns3_probe,
1596 .remove = hns3_remove,
1597};
1598
1599/* set default feature to hns3 */
1600static void hns3_set_default_feature(struct net_device *netdev)
1601{
391b5e93
JS		 1602 struct hnae3_handle *h = hns3_get_handle(netdev);
1603
76ad4f0e
S		 1604 netdev->priv_flags |= IFF_UNICAST_FLT;
1605
1606 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1607 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1608 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1609 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1610 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1611
1612 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1613
1614 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1615
1616 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1617 NETIF_F_HW_VLAN_CTAG_FILTER |
052ece6d 1618 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1619 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1620 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1621 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1622 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1623
1624 netdev->vlan_features |=
1625 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1626 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1627 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1628 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1629 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1630
1631 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
30ba2ab9 1632 NETIF_F_HW_VLAN_CTAG_TX |
76ad4f0e
S		 1633 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1634 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1635 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1636 NETIF_F_GSO_UDP_TUNNEL_CSUM;
391b5e93
JS		 1637
1638 if (!(h->flags & HNAE3_SUPPORT_VF))
30ba2ab9
JS		 1639 netdev->hw_features |=
1640 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
76ad4f0e
S		 1641}
1642
1643static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1644 struct hns3_desc_cb *cb)
1645{
1646 unsigned int order = hnae_page_order(ring);
1647 struct page *p;
1648
1649 p = dev_alloc_pages(order);
1650 if (!p)
1651 return -ENOMEM;
1652
1653 cb->priv = p;
1654 cb->page_offset = 0;
1655 cb->reuse_flag = 0;
1656 cb->buf = page_address(p);
1657 cb->length = hnae_page_size(ring);
1658 cb->type = DESC_TYPE_PAGE;
1659
76ad4f0e
S		 1660 return 0;
1661}
1662
1663static void hns3_free_buffer(struct hns3_enet_ring *ring,
1664 struct hns3_desc_cb *cb)
1665{
1666 if (cb->type == DESC_TYPE_SKB)
1667 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1668 else if (!HNAE3_IS_TX_RING(ring))
1669 put_page((struct page *)cb->priv);
1670 memset(cb, 0, sizeof(*cb));
1671}
1672
1673static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1674{
1675 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1676 cb->length, ring_to_dma_dir(ring));
1677
1678 if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1679 return -EIO;
1680
1681 return 0;
1682}
1683
1684static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1685 struct hns3_desc_cb *cb)
1686{
1687 if (cb->type == DESC_TYPE_SKB)
1688 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1689 ring_to_dma_dir(ring));
1690 else
1691 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1692 ring_to_dma_dir(ring));
1693}
1694
1695static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1696{
1697 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1698 ring->desc[i].addr = 0;
1699}
1700
1701static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1702{
1703 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1704
1705 if (!ring->desc_cb[i].dma)
1706 return;
1707
1708 hns3_buffer_detach(ring, i);
1709 hns3_free_buffer(ring, cb);
1710}
1711
1712static void hns3_free_buffers(struct hns3_enet_ring *ring)
1713{
1714 int i;
1715
1716 for (i = 0; i < ring->desc_num; i++)
1717 hns3_free_buffer_detach(ring, i);
1718}
1719
1720/* free desc along with its attached buffer */
1721static void hns3_free_desc(struct hns3_enet_ring *ring)
1722{
1723 hns3_free_buffers(ring);
1724
1725 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1726 ring->desc_num * sizeof(ring->desc[0]),
1727 DMA_BIDIRECTIONAL);
1728 ring->desc_dma_addr = 0;
1729 kfree(ring->desc);
1730 ring->desc = NULL;
1731}
1732
1733static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1734{
1735 int size = ring->desc_num * sizeof(ring->desc[0]);
1736
1737 ring->desc = kzalloc(size, GFP_KERNEL);
1738 if (!ring->desc)
1739 return -ENOMEM;
1740
1741 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1742 size, DMA_BIDIRECTIONAL);
1743 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1744 ring->desc_dma_addr = 0;
1745 kfree(ring->desc);
1746 ring->desc = NULL;
1747 return -ENOMEM;
1748 }
1749
1750 return 0;
1751}
1752
1753static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1754 struct hns3_desc_cb *cb)
1755{
1756 int ret;
1757
1758 ret = hns3_alloc_buffer(ring, cb);
1759 if (ret)
1760 goto out;
1761
1762 ret = hns3_map_buffer(ring, cb);
1763 if (ret)
1764 goto out_with_buf;
1765
1766 return 0;
1767
1768out_with_buf:
564883bb 1769 hns3_free_buffer(ring, cb);
76ad4f0e
S		 1770out:
1771 return ret;
1772}
1773
1774static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1775{
1776 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1777
1778 if (ret)
1779 return ret;
1780
1781 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1782
1783 return 0;
1784}
1785
1786/* Allocate memory for raw pkg, and map with dma */
1787static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1788{
1789 int i, j, ret;
1790
1791 for (i = 0; i < ring->desc_num; i++) {
1792 ret = hns3_alloc_buffer_attach(ring, i);
1793 if (ret)
1794 goto out_buffer_fail;
1795 }
1796
1797 return 0;
1798
1799out_buffer_fail:
1800 for (j = i - 1; j >= 0; j--)
1801 hns3_free_buffer_detach(ring, j);
1802 return ret;
1803}
1804
1805/* detach a in-used buffer and replace with a reserved one */
1806static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1807 struct hns3_desc_cb *res_cb)
1808{
b9077428 1809 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 1810 ring->desc_cb[i] = *res_cb;
1811 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1812}
1813
1814static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1815{
1816 ring->desc_cb[i].reuse_flag = 0;
1817 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1818 + ring->desc_cb[i].page_offset);
1819}
1820
1821static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1822 int *pkts)
1823{
1824 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1825
1826 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1827 (*bytes) += desc_cb->length;
1828 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1829 hns3_free_buffer_detach(ring, ring->next_to_clean);
1830
1831 ring_ptr_move_fw(ring, next_to_clean);
1832}
1833
1834static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1835{
1836 int u = ring->next_to_use;
1837 int c = ring->next_to_clean;
1838
1839 if (unlikely(h > ring->desc_num))
1840 return 0;
1841
1842 return u > c ? (h > c && h <= u) : (h > c || h <= u);
1843}
1844
24e750c4 1845bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e
S		 1846{
1847 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1848 struct netdev_queue *dev_queue;
1849 int bytes, pkts;
1850 int head;
1851
1852 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1853 rmb(); /* Make sure head is ready before touch any data */
1854
1855 if (is_ring_empty(ring) || head == ring->next_to_clean)
24e750c4 1856 return true; /* no data to poll */
76ad4f0e
S		 1857
1858 if (!is_valid_clean_head(ring, head)) {
1859 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1860 ring->next_to_use, ring->next_to_clean);
1861
1862 u64_stats_update_begin(&ring->syncp);
1863 ring->stats.io_err_cnt++;
1864 u64_stats_update_end(&ring->syncp);
24e750c4 1865 return true;
76ad4f0e
S		 1866 }
1867
1868 bytes = 0;
1869 pkts = 0;
1870 while (head != ring->next_to_clean && budget) {
1871 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1872 /* Issue prefetch for next Tx descriptor */
1873 prefetch(&ring->desc_cb[ring->next_to_clean]);
1874 budget--;
1875 }
1876
1877 ring->tqp_vector->tx_group.total_bytes += bytes;
1878 ring->tqp_vector->tx_group.total_packets += pkts;
1879
1880 u64_stats_update_begin(&ring->syncp);
1881 ring->stats.tx_bytes += bytes;
1882 ring->stats.tx_pkts += pkts;
1883 u64_stats_update_end(&ring->syncp);
1884
1885 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1886 netdev_tx_completed_queue(dev_queue, pkts, bytes);
1887
1888 if (unlikely(pkts && netif_carrier_ok(netdev) &&
1889 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1890 /* Make sure that anybody stopping the queue after this
1891 * sees the new next_to_clean.
1892 */
1893 smp_mb();
1894 if (netif_tx_queue_stopped(dev_queue)) {
1895 netif_tx_wake_queue(dev_queue);
1896 ring->stats.restart_queue++;
1897 }
1898 }
1899
1900 return !!budget;
1901}
1902
1903static int hns3_desc_unused(struct hns3_enet_ring *ring)
1904{
1905 int ntc = ring->next_to_clean;
1906 int ntu = ring->next_to_use;
1907
1908 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1909}
1910
1911static void
1912hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1913{
1914 struct hns3_desc_cb *desc_cb;
1915 struct hns3_desc_cb res_cbs;
1916 int i, ret;
1917
1918 for (i = 0; i < cleand_count; i++) {
1919 desc_cb = &ring->desc_cb[ring->next_to_use];
1920 if (desc_cb->reuse_flag) {
1921 u64_stats_update_begin(&ring->syncp);
1922 ring->stats.reuse_pg_cnt++;
1923 u64_stats_update_end(&ring->syncp);
1924
1925 hns3_reuse_buffer(ring, ring->next_to_use);
1926 } else {
1927 ret = hns3_reserve_buffer_map(ring, &res_cbs);
1928 if (ret) {
1929 u64_stats_update_begin(&ring->syncp);
1930 ring->stats.sw_err_cnt++;
1931 u64_stats_update_end(&ring->syncp);
1932
1933 netdev_err(ring->tqp->handle->kinfo.netdev,
1934 "hnae reserve buffer map failed.\n");
1935 break;
1936 }
1937 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1938 }
1939
1940 ring_ptr_move_fw(ring, next_to_use);
1941 }
1942
1943 wmb(); /* Make all data has been write before submit */
1944 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1945}
1946
1947/* hns3_nic_get_headlen - determine size of header for LRO/GRO
1948 * @data: pointer to the start of the headers
1949 * @max: total length of section to find headers in
1950 *
1951 * This function is meant to determine the length of headers that will
1952 * be recognized by hardware for LRO, GRO, and RSC offloads. The main
1953 * motivation of doing this is to only perform one pull for IPv4 TCP
1954 * packets so that we can do basic things like calculating the gso_size
1955 * based on the average data per packet.
1956 */
1957static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1958 unsigned int max_size)
1959{
1960 unsigned char *network;
1961 u8 hlen;
1962
1963 /* This should never happen, but better safe than sorry */
1964 if (max_size < ETH_HLEN)
1965 return max_size;
1966
1967 /* Initialize network frame pointer */
1968 network = data;
1969
1970 /* Set first protocol and move network header forward */
1971 network += ETH_HLEN;
1972
1973 /* Handle any vlan tag if present */
1974 if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1975 == HNS3_RX_FLAG_VLAN_PRESENT) {
1976 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1977 return max_size;
1978
1979 network += VLAN_HLEN;
1980 }
1981
1982 /* Handle L3 protocols */
1983 if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1984 == HNS3_RX_FLAG_L3ID_IPV4) {
1985 if ((typeof(max_size))(network - data) >
1986 (max_size - sizeof(struct iphdr)))
1987 return max_size;
1988
1989 /* Access ihl as a u8 to avoid unaligned access on ia64 */
1990 hlen = (network[0] & 0x0F) << 2;
1991
1992 /* Verify hlen meets minimum size requirements */
1993 if (hlen < sizeof(struct iphdr))
1994 return network - data;
1995
1996 /* Record next protocol if header is present */
1997 } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1998 == HNS3_RX_FLAG_L3ID_IPV6) {
1999 if ((typeof(max_size))(network - data) >
2000 (max_size - sizeof(struct ipv6hdr)))
2001 return max_size;
2002
2003 /* Record next protocol */
2004 hlen = sizeof(struct ipv6hdr);
2005 } else {
2006 return network - data;
2007 }
2008
2009 /* Relocate pointer to start of L4 header */
2010 network += hlen;
2011
2012 /* Finally sort out TCP/UDP */
2013 if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
2014 == HNS3_RX_FLAG_L4ID_TCP) {
2015 if ((typeof(max_size))(network - data) >
2016 (max_size - sizeof(struct tcphdr)))
2017 return max_size;
2018
2019 /* Access doff as a u8 to avoid unaligned access on ia64 */
2020 hlen = (network[12] & 0xF0) >> 2;
2021
2022 /* Verify hlen meets minimum size requirements */
2023 if (hlen < sizeof(struct tcphdr))
2024 return network - data;
2025
2026 network += hlen;
2027 } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
2028 == HNS3_RX_FLAG_L4ID_UDP) {
2029 if ((typeof(max_size))(network - data) >
2030 (max_size - sizeof(struct udphdr)))
2031 return max_size;
2032
2033 network += sizeof(struct udphdr);
2034 }
2035
2036 /* If everything has gone correctly network should be the
2037 * data section of the packet and will be the end of the header.
2038 * If not then it probably represents the end of the last recognized
2039 * header.
2040 */
2041 if ((typeof(max_size))(network - data) < max_size)
2042 return network - data;
2043 else
2044 return max_size;
2045}
2046
2047static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2048 struct hns3_enet_ring *ring, int pull_len,
2049 struct hns3_desc_cb *desc_cb)
2050{
2051 struct hns3_desc *desc;
2052 int truesize, size;
2053 int last_offset;
2054 bool twobufs;
2055
2056 twobufs = ((PAGE_SIZE < 8192) &&
2057 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2058
2059 desc = &ring->desc[ring->next_to_clean];
2060 size = le16_to_cpu(desc->rx.size);
2061
2062 if (twobufs) {
2063 truesize = hnae_buf_size(ring);
2064 } else {
2065 truesize = ALIGN(size, L1_CACHE_BYTES);
2066 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
2067 }
2068
2069 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2070 size - pull_len, truesize - pull_len);
2071
2072 /* Avoid re-using remote pages,flag default unreuse */
2073 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2074 return;
2075
2076 if (twobufs) {
2077 /* If we are only owner of page we can reuse it */
2078 if (likely(page_count(desc_cb->priv) == 1)) {
2079 /* Flip page offset to other buffer */
2080 desc_cb->page_offset ^= truesize;
2081
2082 desc_cb->reuse_flag = 1;
2083 /* bump ref count on page before it is given*/
2084 get_page(desc_cb->priv);
2085 }
2086 return;
2087 }
2088
2089 /* Move offset up to the next cache line */
2090 desc_cb->page_offset += truesize;
2091
2092 if (desc_cb->page_offset <= last_offset) {
2093 desc_cb->reuse_flag = 1;
2094 /* Bump ref count on page before it is given*/
2095 get_page(desc_cb->priv);
2096 }
2097}
2098
2099static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2100 struct hns3_desc *desc)
2101{
2102 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2103 int l3_type, l4_type;
2104 u32 bd_base_info;
2105 int ol4_type;
2106 u32 l234info;
2107
2108 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2109 l234info = le32_to_cpu(desc->rx.l234_info);
2110
2111 skb->ip_summed = CHECKSUM_NONE;
2112
2113 skb_checksum_none_assert(skb);
2114
2115 if (!(netdev->features & NETIF_F_RXCSUM))
2116 return;
2117
2118 /* check if hardware has done checksum */
2119 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2120 return;
2121
2122 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
2123 hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
2124 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2125 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2126 netdev_err(netdev, "L3/L4 error pkt\n");
2127 u64_stats_update_begin(&ring->syncp);
2128 ring->stats.l3l4_csum_err++;
2129 u64_stats_update_end(&ring->syncp);
2130
2131 return;
2132 }
2133
2134 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
2135 HNS3_RXD_L3ID_S);
2136 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
2137 HNS3_RXD_L4ID_S);
2138
2139 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
2140 switch (ol4_type) {
2141 case HNS3_OL4_TYPE_MAC_IN_UDP:
2142 case HNS3_OL4_TYPE_NVGRE:
2143 skb->csum_level = 1;
2144 case HNS3_OL4_TYPE_NO_TUN:
2145 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2146 if (l3_type == HNS3_L3_TYPE_IPV4 ||
2147 (l3_type == HNS3_L3_TYPE_IPV6 &&
2148 (l4_type == HNS3_L4_TYPE_UDP ||
2149 l4_type == HNS3_L4_TYPE_TCP ||
2150 l4_type == HNS3_L4_TYPE_SCTP)))
2151 skb->ip_summed = CHECKSUM_UNNECESSARY;
2152 break;
2153 }
2154}
2155
d43e5aca
YL		 2156static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2157{
2158 napi_gro_receive(&ring->tqp_vector->napi, skb);
2159}
2160
76ad4f0e
S		 2161static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2162 struct sk_buff **out_skb, int *out_bnum)
2163{
2164 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2165 struct hns3_desc_cb *desc_cb;
2166 struct hns3_desc *desc;
2167 struct sk_buff *skb;
2168 unsigned char *va;
2169 u32 bd_base_info;
2170 int pull_len;
2171 u32 l234info;
2172 int length;
2173 int bnum;
2174
2175 desc = &ring->desc[ring->next_to_clean];
2176 desc_cb = &ring->desc_cb[ring->next_to_clean];
2177
2178 prefetch(desc);
2179
2180 length = le16_to_cpu(desc->rx.pkt_len);
2181 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2182 l234info = le32_to_cpu(desc->rx.l234_info);
2183
2184 /* Check valid BD */
2185 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2186 return -EFAULT;
2187
2188 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2189
2190 /* Prefetch first cache line of first page
2191 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2192 * line size is 64B so need to prefetch twice to make it 128B. But in
2193 * actual we can have greater size of caches with 128B Level 1 cache
2194 * lines. In such a case, single fetch would suffice to cache in the
2195 * relevant part of the header.
2196 */
2197 prefetch(va);
2198#if L1_CACHE_BYTES < 128
2199 prefetch(va + L1_CACHE_BYTES);
2200#endif
2201
2202 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2203 HNS3_RX_HEAD_SIZE);
2204 if (unlikely(!skb)) {
2205 netdev_err(netdev, "alloc rx skb fail\n");
2206
2207 u64_stats_update_begin(&ring->syncp);
2208 ring->stats.sw_err_cnt++;
2209 u64_stats_update_end(&ring->syncp);
2210
2211 return -ENOMEM;
2212 }
2213
2214 prefetchw(skb->data);
2215
9699cffe
PL		 2216 /* Based on hw strategy, the tag offloaded will be stored at
2217 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2218 * in one layer tag case.
2219 */
2220 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2221 u16 vlan_tag;
2222
2223 vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2224 if (!(vlan_tag & VLAN_VID_MASK))
2225 vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2226 if (vlan_tag & VLAN_VID_MASK)
2227 __vlan_hwaccel_put_tag(skb,
2228 htons(ETH_P_8021Q),
2229 vlan_tag);
2230 }
2231
76ad4f0e
S		 2232 bnum = 1;
2233 if (length <= HNS3_RX_HEAD_SIZE) {
2234 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2235
2236 /* We can reuse buffer as-is, just make sure it is local */
2237 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2238 desc_cb->reuse_flag = 1;
2239 else /* This page cannot be reused so discard it */
2240 put_page(desc_cb->priv);
2241
2242 ring_ptr_move_fw(ring, next_to_clean);
2243 } else {
2244 u64_stats_update_begin(&ring->syncp);
2245 ring->stats.seg_pkt_cnt++;
2246 u64_stats_update_end(&ring->syncp);
2247
2248 pull_len = hns3_nic_get_headlen(va, l234info,
2249 HNS3_RX_HEAD_SIZE);
2250 memcpy(__skb_put(skb, pull_len), va,
2251 ALIGN(pull_len, sizeof(long)));
2252
2253 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2254 ring_ptr_move_fw(ring, next_to_clean);
2255
2256 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2257 desc = &ring->desc[ring->next_to_clean];
2258 desc_cb = &ring->desc_cb[ring->next_to_clean];
2259 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2260 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2261 ring_ptr_move_fw(ring, next_to_clean);
2262 bnum++;
2263 }
2264 }
2265
2266 *out_bnum = bnum;
2267
2268 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2269 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2270 ((u64 *)desc)[0], ((u64 *)desc)[1]);
2271 u64_stats_update_begin(&ring->syncp);
2272 ring->stats.non_vld_descs++;
2273 u64_stats_update_end(&ring->syncp);
2274
2275 dev_kfree_skb_any(skb);
2276 return -EINVAL;
2277 }
2278
2279 if (unlikely((!desc->rx.pkt_len) ||
2280 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2281 netdev_err(netdev, "truncated pkt\n");
2282 u64_stats_update_begin(&ring->syncp);
2283 ring->stats.err_pkt_len++;
2284 u64_stats_update_end(&ring->syncp);
2285
2286 dev_kfree_skb_any(skb);
2287 return -EFAULT;
2288 }
2289
2290 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2291 netdev_err(netdev, "L2 error pkt\n");
2292 u64_stats_update_begin(&ring->syncp);
2293 ring->stats.l2_err++;
2294 u64_stats_update_end(&ring->syncp);
2295
2296 dev_kfree_skb_any(skb);
2297 return -EFAULT;
2298 }
2299
2300 u64_stats_update_begin(&ring->syncp);
2301 ring->stats.rx_pkts++;
2302 ring->stats.rx_bytes += skb->len;
2303 u64_stats_update_end(&ring->syncp);
2304
2305 ring->tqp_vector->rx_group.total_bytes += skb->len;
2306
2307 hns3_rx_checksum(ring, skb, desc);
2308 return 0;
2309}
2310
d43e5aca
YL		 2311int hns3_clean_rx_ring(
2312 struct hns3_enet_ring *ring, int budget,
2313 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2314{
2315#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2316 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2317 int recv_pkts, recv_bds, clean_count, err;
2318 int unused_count = hns3_desc_unused(ring);
2319 struct sk_buff *skb = NULL;
2320 int num, bnum = 0;
2321
2322 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2323 rmb(); /* Make sure num taken effect before the other data is touched */
2324
2325 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2326 num -= unused_count;
2327
2328 while (recv_pkts < budget && recv_bds < num) {
2329 /* Reuse or realloc buffers */
2330 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2331 hns3_nic_alloc_rx_buffers(ring,
2332 clean_count + unused_count);
2333 clean_count = 0;
2334 unused_count = hns3_desc_unused(ring);
2335 }
2336
2337 /* Poll one pkt */
2338 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2339 if (unlikely(!skb)) /* This fault cannot be repaired */
2340 goto out;
2341
2342 recv_bds += bnum;
2343 clean_count += bnum;
2344 if (unlikely(err)) { /* Do jump the err */
2345 recv_pkts++;
2346 continue;
2347 }
2348
2349 /* Do update ip stack process */
2350 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2351 rx_fn(ring, skb);
76ad4f0e
S		 2352
2353 recv_pkts++;
2354 }
2355
2356out:
2357 /* Make all data has been write before submit */
2358 if (clean_count + unused_count > 0)
2359 hns3_nic_alloc_rx_buffers(ring,
2360 clean_count + unused_count);
2361
2362 return recv_pkts;
2363}
2364
2365static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2366{
2367#define HNS3_RX_ULTRA_PACKET_RATE 40000
2368 enum hns3_flow_level_range new_flow_level;
2369 struct hns3_enet_tqp_vector *tqp_vector;
2370 int packets_per_secs;
2371 int bytes_per_usecs;
2372 u16 new_int_gl;
2373 int usecs;
2374
2375 if (!ring_group->int_gl)
2376 return false;
2377
2378 if (ring_group->total_packets == 0) {
2379 ring_group->int_gl = HNS3_INT_GL_50K;
2380 ring_group->flow_level = HNS3_FLOW_LOW;
2381 return true;
2382 }
2383
2384 /* Simple throttlerate management
2385 * 0-10MB/s lower (50000 ints/s)
2386 * 10-20MB/s middle (20000 ints/s)
2387 * 20-1249MB/s high (18000 ints/s)
2388 * > 40000pps ultra (8000 ints/s)
2389 */
2390 new_flow_level = ring_group->flow_level;
2391 new_int_gl = ring_group->int_gl;
2392 tqp_vector = ring_group->ring->tqp_vector;
2393 usecs = (ring_group->int_gl << 1);
2394 bytes_per_usecs = ring_group->total_bytes / usecs;
2395 /* 1000000 microseconds */
2396 packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2397
2398 switch (new_flow_level) {
2399 case HNS3_FLOW_LOW:
2400 if (bytes_per_usecs > 10)
2401 new_flow_level = HNS3_FLOW_MID;
2402 break;
2403 case HNS3_FLOW_MID:
2404 if (bytes_per_usecs > 20)
2405 new_flow_level = HNS3_FLOW_HIGH;
2406 else if (bytes_per_usecs <= 10)
2407 new_flow_level = HNS3_FLOW_LOW;
2408 break;
2409 case HNS3_FLOW_HIGH:
2410 case HNS3_FLOW_ULTRA:
2411 default:
2412 if (bytes_per_usecs <= 20)
2413 new_flow_level = HNS3_FLOW_MID;
2414 break;
2415 }
2416
2417 if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2418 (&tqp_vector->rx_group == ring_group))
2419 new_flow_level = HNS3_FLOW_ULTRA;
2420
2421 switch (new_flow_level) {
2422 case HNS3_FLOW_LOW:
2423 new_int_gl = HNS3_INT_GL_50K;
2424 break;
2425 case HNS3_FLOW_MID:
2426 new_int_gl = HNS3_INT_GL_20K;
2427 break;
2428 case HNS3_FLOW_HIGH:
2429 new_int_gl = HNS3_INT_GL_18K;
2430 break;
2431 case HNS3_FLOW_ULTRA:
2432 new_int_gl = HNS3_INT_GL_8K;
2433 break;
2434 default:
2435 break;
2436 }
2437
2438 ring_group->total_bytes = 0;
2439 ring_group->total_packets = 0;
2440 ring_group->flow_level = new_flow_level;
2441 if (new_int_gl != ring_group->int_gl) {
2442 ring_group->int_gl = new_int_gl;
2443 return true;
2444 }
2445 return false;
2446}
2447
2448static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2449{
8b1ff1ea
FL		 2450 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2451 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2452 bool rx_update, tx_update;
2453
2454 if (rx_group->gl_adapt_enable) {
2455 rx_update = hns3_get_new_int_gl(rx_group);
2456 if (rx_update)
2457 hns3_set_vector_coalesce_rx_gl(tqp_vector,
2458 rx_group->int_gl);
2459 }
2460
2461 if (tx_group->gl_adapt_enable) {
2462 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2463 if (tx_update)
2464 hns3_set_vector_coalesce_tx_gl(tqp_vector,
2465 tx_group->int_gl);
76ad4f0e
S		 2466 }
2467}
2468
2469static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2470{
2471 struct hns3_enet_ring *ring;
2472 int rx_pkt_total = 0;
2473
2474 struct hns3_enet_tqp_vector *tqp_vector =
2475 container_of(napi, struct hns3_enet_tqp_vector, napi);
2476 bool clean_complete = true;
2477 int rx_budget;
2478
2479 /* Since the actual Tx work is minimal, we can give the Tx a larger
2480 * budget and be more aggressive about cleaning up the Tx descriptors.
2481 */
2482 hns3_for_each_ring(ring, tqp_vector->tx_group) {
2483 if (!hns3_clean_tx_ring(ring, budget))
2484 clean_complete = false;
2485 }
2486
2487 /* make sure rx ring budget not smaller than 1 */
2488 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2489
2490 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2491 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2492 hns3_rx_skb);
76ad4f0e
S		 2493
2494 if (rx_cleaned >= rx_budget)
2495 clean_complete = false;
2496
2497 rx_pkt_total += rx_cleaned;
2498 }
2499
2500 tqp_vector->rx_group.total_packets += rx_pkt_total;
2501
2502 if (!clean_complete)
2503 return budget;
2504
2505 napi_complete(napi);
2506 hns3_update_new_int_gl(tqp_vector);
2507 hns3_mask_vector_irq(tqp_vector, 1);
2508
2509 return rx_pkt_total;
2510}
2511
2512static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2513 struct hnae3_ring_chain_node *head)
2514{
2515 struct pci_dev *pdev = tqp_vector->handle->pdev;
2516 struct hnae3_ring_chain_node *cur_chain = head;
2517 struct hnae3_ring_chain_node *chain;
2518 struct hns3_enet_ring *tx_ring;
2519 struct hns3_enet_ring *rx_ring;
2520
2521 tx_ring = tqp_vector->tx_group.ring;
2522 if (tx_ring) {
2523 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2524 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2525 HNAE3_RING_TYPE_TX);
2526
2527 cur_chain->next = NULL;
2528
2529 while (tx_ring->next) {
2530 tx_ring = tx_ring->next;
2531
2532 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2533 GFP_KERNEL);
2534 if (!chain)
2535 return -ENOMEM;
2536
2537 cur_chain->next = chain;
2538 chain->tqp_index = tx_ring->tqp->tqp_index;
2539 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2540 HNAE3_RING_TYPE_TX);
2541
2542 cur_chain = chain;
2543 }
2544 }
2545
2546 rx_ring = tqp_vector->rx_group.ring;
2547 if (!tx_ring && rx_ring) {
2548 cur_chain->next = NULL;
2549 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2550 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2551 HNAE3_RING_TYPE_RX);
2552
2553 rx_ring = rx_ring->next;
2554 }
2555
2556 while (rx_ring) {
2557 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2558 if (!chain)
2559 return -ENOMEM;
2560
2561 cur_chain->next = chain;
2562 chain->tqp_index = rx_ring->tqp->tqp_index;
2563 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2564 HNAE3_RING_TYPE_RX);
2565 cur_chain = chain;
2566
2567 rx_ring = rx_ring->next;
2568 }
2569
2570 return 0;
2571}
2572
2573static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2574 struct hnae3_ring_chain_node *head)
2575{
2576 struct pci_dev *pdev = tqp_vector->handle->pdev;
2577 struct hnae3_ring_chain_node *chain_tmp, *chain;
2578
2579 chain = head->next;
2580
2581 while (chain) {
2582 chain_tmp = chain->next;
2583 devm_kfree(&pdev->dev, chain);
2584 chain = chain_tmp;
2585 }
2586}
2587
2588static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2589 struct hns3_enet_ring *ring)
2590{
2591 ring->next = group->ring;
2592 group->ring = ring;
2593
2594 group->count++;
2595}
2596
2597static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2598{
2599 struct hnae3_ring_chain_node vector_ring_chain;
2600 struct hnae3_handle *h = priv->ae_handle;
2601 struct hns3_enet_tqp_vector *tqp_vector;
2602 struct hnae3_vector_info *vector;
2603 struct pci_dev *pdev = h->pdev;
2604 u16 tqp_num = h->kinfo.num_tqps;
2605 u16 vector_num;
2606 int ret = 0;
2607 u16 i;
2608
2609 /* RSS size, cpu online and vector_num should be the same */
2610 /* Should consider 2p/4p later */
2611 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2612 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2613 GFP_KERNEL);
2614 if (!vector)
2615 return -ENOMEM;
2616
2617 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2618
2619 priv->vector_num = vector_num;
2620 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2621 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2622 GFP_KERNEL);
2623 if (!priv->tqp_vector)
2624 return -ENOMEM;
2625
2626 for (i = 0; i < tqp_num; i++) {
2627 u16 vector_i = i % vector_num;
2628
2629 tqp_vector = &priv->tqp_vector[vector_i];
2630
2631 hns3_add_ring_to_group(&tqp_vector->tx_group,
2632 priv->ring_data[i].ring);
2633
2634 hns3_add_ring_to_group(&tqp_vector->rx_group,
2635 priv->ring_data[i + tqp_num].ring);
2636
2637 tqp_vector->idx = vector_i;
2638 tqp_vector->mask_addr = vector[vector_i].io_addr;
2639 tqp_vector->vector_irq = vector[vector_i].vector;
2640 tqp_vector->num_tqps++;
2641
2642 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2643 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2644 }
2645
2646 for (i = 0; i < vector_num; i++) {
2647 tqp_vector = &priv->tqp_vector[i];
2648
2649 tqp_vector->rx_group.total_bytes = 0;
2650 tqp_vector->rx_group.total_packets = 0;
2651 tqp_vector->tx_group.total_bytes = 0;
2652 tqp_vector->tx_group.total_packets = 0;
5fd4789a 2653 hns3_vector_gl_rl_init(tqp_vector, priv);
76ad4f0e
S		 2654 tqp_vector->handle = h;
2655
2656 ret = hns3_get_vector_ring_chain(tqp_vector,
2657 &vector_ring_chain);
2658 if (ret)
2659 goto out;
2660
2661 ret = h->ae_algo->ops->map_ring_to_vector(h,
2662 tqp_vector->vector_irq, &vector_ring_chain);
2663 if (ret)
2664 goto out;
2665
2666 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2667
2668 netif_napi_add(priv->netdev, &tqp_vector->napi,
2669 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2670 }
2671
2672out:
2673 devm_kfree(&pdev->dev, vector);
2674 return ret;
2675}
2676
2677static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2678{
2679 struct hnae3_ring_chain_node vector_ring_chain;
2680 struct hnae3_handle *h = priv->ae_handle;
2681 struct hns3_enet_tqp_vector *tqp_vector;
2682 struct pci_dev *pdev = h->pdev;
2683 int i, ret;
2684
2685 for (i = 0; i < priv->vector_num; i++) {
2686 tqp_vector = &priv->tqp_vector[i];
2687
2688 ret = hns3_get_vector_ring_chain(tqp_vector,
2689 &vector_ring_chain);
2690 if (ret)
2691 return ret;
2692
2693 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2694 tqp_vector->vector_irq, &vector_ring_chain);
2695 if (ret)
2696 return ret;
2697
2698 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2699
2700 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2701 (void)irq_set_affinity_hint(
2702 priv->tqp_vector[i].vector_irq,
2703 NULL);
ae064e61 2704 free_irq(priv->tqp_vector[i].vector_irq,
2705 &priv->tqp_vector[i]);
76ad4f0e
S		 2706 }
2707
2708 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2709
2710 netif_napi_del(&priv->tqp_vector[i].napi);
2711 }
2712
2713 devm_kfree(&pdev->dev, priv->tqp_vector);
2714
2715 return 0;
2716}
2717
2718static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2719 int ring_type)
2720{
2721 struct hns3_nic_ring_data *ring_data = priv->ring_data;
2722 int queue_num = priv->ae_handle->kinfo.num_tqps;
2723 struct pci_dev *pdev = priv->ae_handle->pdev;
2724 struct hns3_enet_ring *ring;
2725
2726 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2727 if (!ring)
2728 return -ENOMEM;
2729
2730 if (ring_type == HNAE3_RING_TYPE_TX) {
2731 ring_data[q->tqp_index].ring = ring;
66b44730 2732 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 2733 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2734 } else {
2735 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 2736 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 2737 ring->io_base = q->io_base;
2738 }
2739
2740 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2741
76ad4f0e
S		 2742 ring->tqp = q;
2743 ring->desc = NULL;
2744 ring->desc_cb = NULL;
2745 ring->dev = priv->dev;
2746 ring->desc_dma_addr = 0;
2747 ring->buf_size = q->buf_size;
2748 ring->desc_num = q->desc_num;
2749 ring->next_to_use = 0;
2750 ring->next_to_clean = 0;
2751
2752 return 0;
2753}
2754
2755static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2756 struct hns3_nic_priv *priv)
2757{
2758 int ret;
2759
2760 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2761 if (ret)
2762 return ret;
2763
2764 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2765 if (ret)
2766 return ret;
2767
2768 return 0;
2769}
2770
2771static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2772{
2773 struct hnae3_handle *h = priv->ae_handle;
2774 struct pci_dev *pdev = h->pdev;
2775 int i, ret;
2776
2777 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2778 sizeof(*priv->ring_data) * 2,
2779 GFP_KERNEL);
2780 if (!priv->ring_data)
2781 return -ENOMEM;
2782
2783 for (i = 0; i < h->kinfo.num_tqps; i++) {
2784 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2785 if (ret)
2786 goto err;
2787 }
2788
2789 return 0;
2790err:
2791 devm_kfree(&pdev->dev, priv->ring_data);
2792 return ret;
2793}
2794
09f2af64
PL		 2795static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2796{
2797 struct hnae3_handle *h = priv->ae_handle;
2798 int i;
2799
2800 for (i = 0; i < h->kinfo.num_tqps; i++) {
2801 devm_kfree(priv->dev, priv->ring_data[i].ring);
2802 devm_kfree(priv->dev,
2803 priv->ring_data[i + h->kinfo.num_tqps].ring);
2804 }
2805 devm_kfree(priv->dev, priv->ring_data);
2806}
2807
76ad4f0e
S		 2808static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2809{
2810 int ret;
2811
2812 if (ring->desc_num <= 0 || ring->buf_size <= 0)
2813 return -EINVAL;
2814
2815 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2816 GFP_KERNEL);
2817 if (!ring->desc_cb) {
2818 ret = -ENOMEM;
2819 goto out;
2820 }
2821
2822 ret = hns3_alloc_desc(ring);
2823 if (ret)
2824 goto out_with_desc_cb;
2825
2826 if (!HNAE3_IS_TX_RING(ring)) {
2827 ret = hns3_alloc_ring_buffers(ring);
2828 if (ret)
2829 goto out_with_desc;
2830 }
2831
2832 return 0;
2833
2834out_with_desc:
2835 hns3_free_desc(ring);
2836out_with_desc_cb:
2837 kfree(ring->desc_cb);
2838 ring->desc_cb = NULL;
2839out:
2840 return ret;
2841}
2842
2843static void hns3_fini_ring(struct hns3_enet_ring *ring)
2844{
2845 hns3_free_desc(ring);
2846 kfree(ring->desc_cb);
2847 ring->desc_cb = NULL;
2848 ring->next_to_clean = 0;
2849 ring->next_to_use = 0;
2850}
2851
1db9b1bf 2852static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 2853{
2854 int bd_size_type;
2855
2856 switch (buf_size) {
2857 case 512:
2858 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2859 break;
2860 case 1024:
2861 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2862 break;
2863 case 2048:
2864 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2865 break;
2866 case 4096:
2867 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2868 break;
2869 default:
2870 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2871 }
2872
2873 return bd_size_type;
2874}
2875
2876static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2877{
2878 dma_addr_t dma = ring->desc_dma_addr;
2879 struct hnae3_queue *q = ring->tqp;
2880
2881 if (!HNAE3_IS_TX_RING(ring)) {
2882 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2883 (u32)dma);
2884 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2885 (u32)((dma >> 31) >> 1));
2886
2887 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2888 hns3_buf_size2type(ring->buf_size));
2889 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2890 ring->desc_num / 8 - 1);
2891
2892 } else {
2893 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2894 (u32)dma);
2895 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2896 (u32)((dma >> 31) >> 1));
2897
2898 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2899 hns3_buf_size2type(ring->buf_size));
2900 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2901 ring->desc_num / 8 - 1);
2902 }
2903}
2904
5668abda 2905int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2906{
2907 struct hnae3_handle *h = priv->ae_handle;
2908 int ring_num = h->kinfo.num_tqps * 2;
2909 int i, j;
2910 int ret;
2911
2912 for (i = 0; i < ring_num; i++) {
2913 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2914 if (ret) {
2915 dev_err(priv->dev,
2916 "Alloc ring memory fail! ret=%d\n", ret);
2917 goto out_when_alloc_ring_memory;
2918 }
2919
2920 hns3_init_ring_hw(priv->ring_data[i].ring);
2921
2922 u64_stats_init(&priv->ring_data[i].ring->syncp);
2923 }
2924
2925 return 0;
2926
2927out_when_alloc_ring_memory:
2928 for (j = i - 1; j >= 0; j--)
ee83f776 2929 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 2930
2931 return -ENOMEM;
2932}
2933
5668abda 2934int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2935{
2936 struct hnae3_handle *h = priv->ae_handle;
2937 int i;
2938
2939 for (i = 0; i < h->kinfo.num_tqps; i++) {
2940 if (h->ae_algo->ops->reset_queue)
2941 h->ae_algo->ops->reset_queue(h, i);
2942
2943 hns3_fini_ring(priv->ring_data[i].ring);
99fdf6b1 2944 devm_kfree(priv->dev, priv->ring_data[i].ring);
76ad4f0e 2945 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
99fdf6b1
PL		 2946 devm_kfree(priv->dev,
2947 priv->ring_data[i + h->kinfo.num_tqps].ring);
76ad4f0e 2948 }
99fdf6b1 2949 devm_kfree(priv->dev, priv->ring_data);
76ad4f0e
S		 2950
2951 return 0;
2952}
2953
2954/* Set mac addr if it is configured. or leave it to the AE driver */
2955static void hns3_init_mac_addr(struct net_device *netdev)
2956{
2957 struct hns3_nic_priv *priv = netdev_priv(netdev);
2958 struct hnae3_handle *h = priv->ae_handle;
2959 u8 mac_addr_temp[ETH_ALEN];
2960
2961 if (h->ae_algo->ops->get_mac_addr) {
2962 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
2963 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
2964 }
2965
2966 /* Check if the MAC address is valid, if not get a random one */
2967 if (!is_valid_ether_addr(netdev->dev_addr)) {
2968 eth_hw_addr_random(netdev);
2969 dev_warn(priv->dev, "using random MAC address %pM\n",
2970 netdev->dev_addr);
76ad4f0e 2971 }
139e8792
L		 2972
2973 if (h->ae_algo->ops->set_mac_addr)
2974 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
2975
76ad4f0e
S		 2976}
2977
2978static void hns3_nic_set_priv_ops(struct net_device *netdev)
2979{
2980 struct hns3_nic_priv *priv = netdev_priv(netdev);
2981
2982 if ((netdev->features & NETIF_F_TSO) ||
2983 (netdev->features & NETIF_F_TSO6)) {
2984 priv->ops.fill_desc = hns3_fill_desc_tso;
2985 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
2986 } else {
2987 priv->ops.fill_desc = hns3_fill_desc;
2988 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
2989 }
2990}
2991
2992static int hns3_client_init(struct hnae3_handle *handle)
2993{
2994 struct pci_dev *pdev = handle->pdev;
2995 struct hns3_nic_priv *priv;
2996 struct net_device *netdev;
2997 int ret;
2998
2999 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
3000 handle->kinfo.num_tqps);
3001 if (!netdev)
3002 return -ENOMEM;
3003
3004 priv = netdev_priv(netdev);
3005 priv->dev = &pdev->dev;
3006 priv->netdev = netdev;
3007 priv->ae_handle = handle;
f8fa222c
L		 3008 priv->last_reset_time = jiffies;
3009 priv->reset_level = HNAE3_FUNC_RESET;
3010 priv->tx_timeout_count = 0;
76ad4f0e
S		 3011
3012 handle->kinfo.netdev = netdev;
3013 handle->priv = (void *)priv;
3014
3015 hns3_init_mac_addr(netdev);
3016
3017 hns3_set_default_feature(netdev);
3018
3019 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3020 netdev->priv_flags |= IFF_UNICAST_FLT;
3021 netdev->netdev_ops = &hns3_nic_netdev_ops;
3022 SET_NETDEV_DEV(netdev, &pdev->dev);
3023 hns3_ethtool_set_ops(netdev);
3024 hns3_nic_set_priv_ops(netdev);
3025
3026 /* Carrier off reporting is important to ethtool even BEFORE open */
3027 netif_carrier_off(netdev);
3028
3029 ret = hns3_get_ring_config(priv);
3030 if (ret) {
3031 ret = -ENOMEM;
3032 goto out_get_ring_cfg;
3033 }
3034
3035 ret = hns3_nic_init_vector_data(priv);
3036 if (ret) {
3037 ret = -ENOMEM;
3038 goto out_init_vector_data;
3039 }
3040
3041 ret = hns3_init_all_ring(priv);
3042 if (ret) {
3043 ret = -ENOMEM;
3044 goto out_init_ring_data;
3045 }
3046
3047 ret = register_netdev(netdev);
3048 if (ret) {
3049 dev_err(priv->dev, "probe register netdev fail!\n");
3050 goto out_reg_netdev_fail;
3051 }
3052
986743db
YL		 3053 hns3_dcbnl_setup(handle);
3054
a8e8b7ff
S		 3055 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3056 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3057
76ad4f0e
S		 3058 return ret;
3059
3060out_reg_netdev_fail:
3061out_init_ring_data:
3062 (void)hns3_nic_uninit_vector_data(priv);
3063 priv->ring_data = NULL;
3064out_init_vector_data:
3065out_get_ring_cfg:
3066 priv->ae_handle = NULL;
3067 free_netdev(netdev);
3068 return ret;
3069}
3070
3071static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3072{
3073 struct net_device *netdev = handle->kinfo.netdev;
3074 struct hns3_nic_priv *priv = netdev_priv(netdev);
3075 int ret;
3076
3077 if (netdev->reg_state != NETREG_UNINITIALIZED)
3078 unregister_netdev(netdev);
3079
3080 ret = hns3_nic_uninit_vector_data(priv);
3081 if (ret)
3082 netdev_err(netdev, "uninit vector error\n");
3083
3084 ret = hns3_uninit_all_ring(priv);
3085 if (ret)
3086 netdev_err(netdev, "uninit ring error\n");
3087
3088 priv->ring_data = NULL;
3089
3090 free_netdev(netdev);
3091}
3092
3093static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3094{
3095 struct net_device *netdev = handle->kinfo.netdev;
3096
3097 if (!netdev)
3098 return;
3099
3100 if (linkup) {
3101 netif_carrier_on(netdev);
3102 netif_tx_wake_all_queues(netdev);
3103 netdev_info(netdev, "link up\n");
3104 } else {
3105 netif_carrier_off(netdev);
3106 netif_tx_stop_all_queues(netdev);
3107 netdev_info(netdev, "link down\n");
3108 }
3109}
3110
9df8f79a
YL		 3111static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3112{
3113 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3114 struct net_device *ndev = kinfo->netdev;
075cfdd6 3115 bool if_running;
9df8f79a
YL		 3116 int ret;
3117 u8 i;
3118
3119 if (tc > HNAE3_MAX_TC)
3120 return -EINVAL;
3121
3122 if (!ndev)
3123 return -ENODEV;
3124
075cfdd6
CIK		 3125 if_running = netif_running(ndev);
3126
9df8f79a
YL		 3127 ret = netdev_set_num_tc(ndev, tc);
3128 if (ret)
3129 return ret;
3130
3131 if (if_running) {
3132 (void)hns3_nic_net_stop(ndev);
3133 msleep(100);
3134 }
3135
3136 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3137 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3138 if (ret)
3139 goto err_out;
3140
3141 if (tc <= 1) {
3142 netdev_reset_tc(ndev);
3143 goto out;
3144 }
3145
3146 for (i = 0; i < HNAE3_MAX_TC; i++) {
3147 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3148
3149 if (tc_info->enable)
3150 netdev_set_tc_queue(ndev,
3151 tc_info->tc,
3152 tc_info->tqp_count,
3153 tc_info->tqp_offset);
3154 }
3155
3156 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
3157 netdev_set_prio_tc_map(ndev, i,
3158 kinfo->prio_tc[i]);
3159 }
3160
3161out:
3162 ret = hns3_nic_set_real_num_queue(ndev);
3163
3164err_out:
3165 if (if_running)
3166 (void)hns3_nic_net_open(ndev);
3167
3168 return ret;
3169}
3170
bb6b94a8
L		 3171static void hns3_recover_hw_addr(struct net_device *ndev)
3172{
3173 struct netdev_hw_addr_list *list;
3174 struct netdev_hw_addr *ha, *tmp;
3175
3176 /* go through and sync uc_addr entries to the device */
3177 list = &ndev->uc;
3178 list_for_each_entry_safe(ha, tmp, &list->list, list)
3179 hns3_nic_uc_sync(ndev, ha->addr);
3180
3181 /* go through and sync mc_addr entries to the device */
3182 list = &ndev->mc;
3183 list_for_each_entry_safe(ha, tmp, &list->list, list)
3184 hns3_nic_mc_sync(ndev, ha->addr);
3185}
3186
3187static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb)
3188{
3189 dev_kfree_skb_any(skb);
3190}
3191
3192static void hns3_clear_all_ring(struct hnae3_handle *h)
3193{
3194 struct net_device *ndev = h->kinfo.netdev;
3195 struct hns3_nic_priv *priv = netdev_priv(ndev);
3196 u32 i;
3197
3198 for (i = 0; i < h->kinfo.num_tqps; i++) {
3199 struct netdev_queue *dev_queue;
3200 struct hns3_enet_ring *ring;
3201
3202 ring = priv->ring_data[i].ring;
3203 hns3_clean_tx_ring(ring, ring->desc_num);
3204 dev_queue = netdev_get_tx_queue(ndev,
3205 priv->ring_data[i].queue_index);
3206 netdev_tx_reset_queue(dev_queue);
3207
3208 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3209 hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data);
3210 }
3211}
3212
3213static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3214{
3215 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3216 struct net_device *ndev = kinfo->netdev;
3217
3218 if (!netif_running(ndev))
3219 return -EIO;
3220
3221 return hns3_nic_net_stop(ndev);
3222}
3223
3224static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3225{
3226 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3227 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
3228 int ret = 0;
3229
3230 if (netif_running(kinfo->netdev)) {
3231 ret = hns3_nic_net_up(kinfo->netdev);
3232 if (ret) {
3233 netdev_err(kinfo->netdev,
3234 "hns net up fail, ret=%d!\n", ret);
3235 return ret;
3236 }
3237
3238 priv->last_reset_time = jiffies;
3239 }
3240
3241 return ret;
3242}
3243
3244static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3245{
3246 struct net_device *netdev = handle->kinfo.netdev;
3247 struct hns3_nic_priv *priv = netdev_priv(netdev);
3248 int ret;
3249
3250 priv->reset_level = 1;
3251 hns3_init_mac_addr(netdev);
3252 hns3_nic_set_rx_mode(netdev);
3253 hns3_recover_hw_addr(netdev);
3254
3255 /* Carrier off reporting is important to ethtool even BEFORE open */
3256 netif_carrier_off(netdev);
3257
3258 ret = hns3_get_ring_config(priv);
3259 if (ret)
3260 return ret;
3261
3262 ret = hns3_nic_init_vector_data(priv);
3263 if (ret)
3264 return ret;
3265
3266 ret = hns3_init_all_ring(priv);
3267 if (ret) {
3268 hns3_nic_uninit_vector_data(priv);
3269 priv->ring_data = NULL;
3270 }
3271
3272 return ret;
3273}
3274
3275static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3276{
3277 struct net_device *netdev = handle->kinfo.netdev;
3278 struct hns3_nic_priv *priv = netdev_priv(netdev);
3279 int ret;
3280
3281 hns3_clear_all_ring(handle);
3282
3283 ret = hns3_nic_uninit_vector_data(priv);
3284 if (ret) {
3285 netdev_err(netdev, "uninit vector error\n");
3286 return ret;
3287 }
3288
3289 ret = hns3_uninit_all_ring(priv);
3290 if (ret)
3291 netdev_err(netdev, "uninit ring error\n");
3292
3293 priv->ring_data = NULL;
3294
3295 return ret;
3296}
3297
3298static int hns3_reset_notify(struct hnae3_handle *handle,
3299 enum hnae3_reset_notify_type type)
3300{
3301 int ret = 0;
3302
3303 switch (type) {
3304 case HNAE3_UP_CLIENT:
3305 ret = hns3_reset_notify_up_enet(handle);
3306 break;
3307 case HNAE3_DOWN_CLIENT:
3308 ret = hns3_reset_notify_down_enet(handle);
3309 break;
3310 case HNAE3_INIT_CLIENT:
3311 ret = hns3_reset_notify_init_enet(handle);
3312 break;
3313 case HNAE3_UNINIT_CLIENT:
3314 ret = hns3_reset_notify_uninit_enet(handle);
3315 break;
3316 default:
3317 break;
3318 }
3319
3320 return ret;
3321}
3322
09f2af64
PL		 3323static u16 hns3_get_max_available_channels(struct net_device *netdev)
3324{
3325 struct hnae3_handle *h = hns3_get_handle(netdev);
3326 u16 free_tqps, max_rss_size, max_tqps;
3327
3328 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
3329 max_tqps = h->kinfo.num_tc * max_rss_size;
3330
3331 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
3332}
3333
3334static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
3335{
3336 struct hns3_nic_priv *priv = netdev_priv(netdev);
3337 struct hnae3_handle *h = hns3_get_handle(netdev);
3338 int ret;
3339
3340 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3341 if (ret)
3342 return ret;
3343
3344 ret = hns3_get_ring_config(priv);
3345 if (ret)
3346 return ret;
3347
3348 ret = hns3_nic_init_vector_data(priv);
3349 if (ret)
3350 goto err_uninit_vector;
3351
3352 ret = hns3_init_all_ring(priv);
3353 if (ret)
3354 goto err_put_ring;
3355
3356 return 0;
3357
3358err_put_ring:
3359 hns3_put_ring_config(priv);
3360err_uninit_vector:
3361 hns3_nic_uninit_vector_data(priv);
3362 return ret;
3363}
3364
3365static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3366{
3367 return (new_tqp_num / num_tc) * num_tc;
3368}
3369
3370int hns3_set_channels(struct net_device *netdev,
3371 struct ethtool_channels *ch)
3372{
3373 struct hns3_nic_priv *priv = netdev_priv(netdev);
3374 struct hnae3_handle *h = hns3_get_handle(netdev);
3375 struct hnae3_knic_private_info *kinfo = &h->kinfo;
3376 bool if_running = netif_running(netdev);
3377 u32 new_tqp_num = ch->combined_count;
3378 u16 org_tqp_num;
3379 int ret;
3380
3381 if (ch->rx_count || ch->tx_count)
3382 return -EINVAL;
3383
3384 if (new_tqp_num > hns3_get_max_available_channels(netdev) ||
3385 new_tqp_num < kinfo->num_tc) {
3386 dev_err(&netdev->dev,
3387 "Change tqps fail, the tqp range is from %d to %d",
3388 kinfo->num_tc,
3389 hns3_get_max_available_channels(netdev));
3390 return -EINVAL;
3391 }
3392
3393 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3394 if (kinfo->num_tqps == new_tqp_num)
3395 return 0;
3396
3397 if (if_running)
3398 dev_close(netdev);
3399
3400 hns3_clear_all_ring(h);
3401
3402 ret = hns3_nic_uninit_vector_data(priv);
3403 if (ret) {
3404 dev_err(&netdev->dev,
3405 "Unbind vector with tqp fail, nothing is changed");
3406 goto open_netdev;
3407 }
3408
3409 hns3_uninit_all_ring(priv);
3410
3411 org_tqp_num = h->kinfo.num_tqps;
3412 ret = hns3_modify_tqp_num(netdev, new_tqp_num);
3413 if (ret) {
3414 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
3415 if (ret) {
3416 /* If revert to old tqp failed, fatal error occurred */
3417 dev_err(&netdev->dev,
3418 "Revert to old tqp num fail, ret=%d", ret);
3419 return ret;
3420 }
3421 dev_info(&netdev->dev,
3422 "Change tqp num fail, Revert to old tqp num");
3423 }
3424
3425open_netdev:
3426 if (if_running)
3427 dev_open(netdev);
3428
3429 return ret;
3430}
3431
1db9b1bf 3432static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 3433 .init_instance = hns3_client_init,
3434 .uninit_instance = hns3_client_uninit,
3435 .link_status_change = hns3_link_status_change,
9df8f79a 3436 .setup_tc = hns3_client_setup_tc,
bb6b94a8 3437 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 3438};
3439
3440/* hns3_init_module - Driver registration routine
3441 * hns3_init_module is the first routine called when the driver is
3442 * loaded. All it does is register with the PCI subsystem.
3443 */
3444static int __init hns3_init_module(void)
3445{
3446 int ret;
3447
3448 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3449 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3450
3451 client.type = HNAE3_CLIENT_KNIC;
3452 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3453 hns3_driver_name);
3454
3455 client.ops = &client_ops;
3456
3457 ret = hnae3_register_client(&client);
3458 if (ret)
3459 return ret;
3460
3461 ret = pci_register_driver(&hns3_driver);
3462 if (ret)
3463 hnae3_unregister_client(&client);
3464
3465 return ret;
3466}
3467module_init(hns3_init_module);
3468
3469/* hns3_exit_module - Driver exit cleanup routine
3470 * hns3_exit_module is called just before the driver is removed
3471 * from memory.
3472 */
3473static void __exit hns3_exit_module(void)
3474{
3475 pci_unregister_driver(&hns3_driver);
3476 hnae3_unregister_client(&client);
3477}
3478module_exit(hns3_exit_module);
3479
3480MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3481MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3482MODULE_LICENSE("GPL");
3483MODULE_ALIAS("pci:hns-nic");
Ubuntu Focal Linux kernel mirror