]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: add ethtool related offload command
[mirror_ubuntu-bionic-kernel.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
76ad4f0e
S		 1/*
2 * Copyright (c) 2016~2017 Hisilicon Limited.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/etherdevice.h>
12#include <linux/interrupt.h>
13#include <linux/if_vlan.h>
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/skbuff.h>
19#include <linux/sctp.h>
20#include <linux/vermagic.h>
21#include <net/gre.h>
30d240df 22#include <net/pkt_cls.h>
76ad4f0e
S		 23#include <net/vxlan.h>
24
25#include "hnae3.h"
26#include "hns3_enet.h"
27
1db9b1bf 28static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 29const char hns3_driver_version[] = VERMAGIC_STRING;
30static const char hns3_driver_string[] =
31 "Hisilicon Ethernet Network Driver for Hip08 Family";
32static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33static struct hnae3_client client;
34
35/* hns3_pci_tbl - PCI Device ID Table
36 *
37 * Last entry must be all 0s
38 *
39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40 * Class, Class Mask, private data (not used) }
41 */
42static const struct pci_device_id hns3_pci_tbl[] = {
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
a9c89a3f
SM		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62static irqreturn_t hns3_irq_handle(int irq, void *dev)
63{
64 struct hns3_enet_tqp_vector *tqp_vector = dev;
65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
71static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
72{
73 struct hns3_enet_tqp_vector *tqp_vectors;
74 unsigned int i;
75
76 for (i = 0; i < priv->vector_num; i++) {
77 tqp_vectors = &priv->tqp_vector[i];
78
79 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
80 continue;
81
82 /* release the irq resource */
83 free_irq(tqp_vectors->vector_irq, tqp_vectors);
84 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
85 }
86}
87
88static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 int txrx_int_idx = 0;
92 int rx_int_idx = 0;
93 int tx_int_idx = 0;
94 unsigned int i;
95 int ret;
96
97 for (i = 0; i < priv->vector_num; i++) {
98 tqp_vectors = &priv->tqp_vector[i];
99
100 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
101 continue;
102
103 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
104 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
105 "%s-%s-%d", priv->netdev->name, "TxRx",
106 txrx_int_idx++);
107 txrx_int_idx++;
108 } else if (tqp_vectors->rx_group.ring) {
109 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
110 "%s-%s-%d", priv->netdev->name, "Rx",
111 rx_int_idx++);
112 } else if (tqp_vectors->tx_group.ring) {
113 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
114 "%s-%s-%d", priv->netdev->name, "Tx",
115 tx_int_idx++);
116 } else {
117 /* Skip this unused q_vector */
118 continue;
119 }
120
121 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
122
123 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
124 tqp_vectors->name,
125 tqp_vectors);
126 if (ret) {
127 netdev_err(priv->netdev, "request irq(%d) fail\n",
128 tqp_vectors->vector_irq);
129 return ret;
130 }
131
132 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
133 }
134
135 return 0;
136}
137
138static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
139 u32 mask_en)
140{
141 writel(mask_en, tqp_vector->mask_addr);
142}
143
144static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
145{
146 napi_enable(&tqp_vector->napi);
147
148 /* enable vector */
149 hns3_mask_vector_irq(tqp_vector, 1);
150}
151
152static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
153{
154 /* disable vector */
155 hns3_mask_vector_irq(tqp_vector, 0);
156
157 disable_irq(tqp_vector->vector_irq);
158 napi_disable(&tqp_vector->napi);
159}
160
161static void hns3_set_vector_coalesc_gl(struct hns3_enet_tqp_vector *tqp_vector,
162 u32 gl_value)
163{
164 /* this defines the configuration for GL (Interrupt Gap Limiter)
165 * GL defines inter interrupt gap.
166 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
167 */
168 writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
169 writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
170 writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL2_OFFSET);
171}
172
173static void hns3_set_vector_coalesc_rl(struct hns3_enet_tqp_vector *tqp_vector,
174 u32 rl_value)
175{
176 /* this defines the configuration for RL (Interrupt Rate Limiter).
177 * Rl defines rate of interrupts i.e. number of interrupts-per-second
178 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
179 */
180 writel(rl_value, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
181}
182
183static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector)
184{
185 /* initialize the configuration for interrupt coalescing.
186 * 1. GL (Interrupt Gap Limiter)
187 * 2. RL (Interrupt Rate Limiter)
188 */
189
190 /* Default :enable interrupt coalesce */
191 tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
192 tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
193 hns3_set_vector_coalesc_gl(tqp_vector, HNS3_INT_GL_50K);
194 /* for now we are disabling Interrupt RL - we
195 * will re-enable later
196 */
197 hns3_set_vector_coalesc_rl(tqp_vector, 0);
198 tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
199 tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
200}
201
9df8f79a
YL		 202static int hns3_nic_set_real_num_queue(struct net_device *netdev)
203{
9780cb97 204 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 205 struct hnae3_knic_private_info *kinfo = &h->kinfo;
206 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
207 int ret;
208
209 ret = netif_set_real_num_tx_queues(netdev, queue_size);
210 if (ret) {
211 netdev_err(netdev,
212 "netif_set_real_num_tx_queues fail, ret=%d!\n",
213 ret);
214 return ret;
215 }
216
217 ret = netif_set_real_num_rx_queues(netdev, queue_size);
218 if (ret) {
219 netdev_err(netdev,
220 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
221 return ret;
222 }
223
224 return 0;
225}
226
76ad4f0e
S		 227static int hns3_nic_net_up(struct net_device *netdev)
228{
229 struct hns3_nic_priv *priv = netdev_priv(netdev);
230 struct hnae3_handle *h = priv->ae_handle;
231 int i, j;
232 int ret;
233
234 /* get irq resource for all vectors */
235 ret = hns3_nic_init_irq(priv);
236 if (ret) {
237 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
238 return ret;
239 }
240
241 /* enable the vectors */
242 for (i = 0; i < priv->vector_num; i++)
243 hns3_vector_enable(&priv->tqp_vector[i]);
244
245 /* start the ae_dev */
246 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
247 if (ret)
248 goto out_start_err;
249
250 return 0;
251
252out_start_err:
253 for (j = i - 1; j >= 0; j--)
254 hns3_vector_disable(&priv->tqp_vector[j]);
255
256 hns3_nic_uninit_irq(priv);
257
258 return ret;
259}
260
261static int hns3_nic_net_open(struct net_device *netdev)
262{
f8fa222c 263 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 264 int ret;
265
266 netif_carrier_off(netdev);
267
9df8f79a
YL		 268 ret = hns3_nic_set_real_num_queue(netdev);
269 if (ret)
76ad4f0e 270 return ret;
76ad4f0e
S		 271
272 ret = hns3_nic_net_up(netdev);
273 if (ret) {
274 netdev_err(netdev,
275 "hns net up fail, ret=%d!\n", ret);
276 return ret;
277 }
278
f8fa222c 279 priv->last_reset_time = jiffies;
76ad4f0e
S		 280 return 0;
281}
282
283static void hns3_nic_net_down(struct net_device *netdev)
284{
285 struct hns3_nic_priv *priv = netdev_priv(netdev);
286 const struct hnae3_ae_ops *ops;
287 int i;
288
289 /* stop ae_dev */
290 ops = priv->ae_handle->ae_algo->ops;
291 if (ops->stop)
292 ops->stop(priv->ae_handle);
293
294 /* disable vectors */
295 for (i = 0; i < priv->vector_num; i++)
296 hns3_vector_disable(&priv->tqp_vector[i]);
297
298 /* free irq resources */
299 hns3_nic_uninit_irq(priv);
300}
301
302static int hns3_nic_net_stop(struct net_device *netdev)
303{
304 netif_tx_stop_all_queues(netdev);
305 netif_carrier_off(netdev);
306
307 hns3_nic_net_down(netdev);
308
309 return 0;
310}
311
76ad4f0e
S		 312static int hns3_nic_uc_sync(struct net_device *netdev,
313 const unsigned char *addr)
314{
9780cb97 315 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 316
317 if (h->ae_algo->ops->add_uc_addr)
318 return h->ae_algo->ops->add_uc_addr(h, addr);
319
320 return 0;
321}
322
323static int hns3_nic_uc_unsync(struct net_device *netdev,
324 const unsigned char *addr)
325{
9780cb97 326 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 327
328 if (h->ae_algo->ops->rm_uc_addr)
329 return h->ae_algo->ops->rm_uc_addr(h, addr);
330
331 return 0;
332}
333
334static int hns3_nic_mc_sync(struct net_device *netdev,
335 const unsigned char *addr)
336{
9780cb97 337 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 338
720a8478 339 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 340 return h->ae_algo->ops->add_mc_addr(h, addr);
341
342 return 0;
343}
344
345static int hns3_nic_mc_unsync(struct net_device *netdev,
346 const unsigned char *addr)
347{
9780cb97 348 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 349
720a8478 350 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 351 return h->ae_algo->ops->rm_mc_addr(h, addr);
352
353 return 0;
354}
355
1db9b1bf 356static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 357{
9780cb97 358 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 359
360 if (h->ae_algo->ops->set_promisc_mode) {
361 if (netdev->flags & IFF_PROMISC)
362 h->ae_algo->ops->set_promisc_mode(h, 1);
363 else
364 h->ae_algo->ops->set_promisc_mode(h, 0);
365 }
366 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
367 netdev_err(netdev, "sync uc address fail\n");
368 if (netdev->flags & IFF_MULTICAST)
369 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
370 netdev_err(netdev, "sync mc address fail\n");
371}
372
373static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
374 u16 *mss, u32 *type_cs_vlan_tso)
375{
376 u32 l4_offset, hdr_len;
377 union l3_hdr_info l3;
378 union l4_hdr_info l4;
379 u32 l4_paylen;
380 int ret;
381
382 if (!skb_is_gso(skb))
383 return 0;
384
385 ret = skb_cow_head(skb, 0);
386 if (ret)
387 return ret;
388
389 l3.hdr = skb_network_header(skb);
390 l4.hdr = skb_transport_header(skb);
391
392 /* Software should clear the IPv4's checksum field when tso is
393 * needed.
394 */
395 if (l3.v4->version == 4)
396 l3.v4->check = 0;
397
398 /* tunnel packet.*/
399 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
400 SKB_GSO_GRE_CSUM |
401 SKB_GSO_UDP_TUNNEL |
402 SKB_GSO_UDP_TUNNEL_CSUM)) {
403 if ((!(skb_shinfo(skb)->gso_type &
404 SKB_GSO_PARTIAL)) &&
405 (skb_shinfo(skb)->gso_type &
406 SKB_GSO_UDP_TUNNEL_CSUM)) {
407 /* Software should clear the udp's checksum
408 * field when tso is needed.
409 */
410 l4.udp->check = 0;
411 }
412 /* reset l3&l4 pointers from outer to inner headers */
413 l3.hdr = skb_inner_network_header(skb);
414 l4.hdr = skb_inner_transport_header(skb);
415
416 /* Software should clear the IPv4's checksum field when
417 * tso is needed.
418 */
419 if (l3.v4->version == 4)
420 l3.v4->check = 0;
421 }
422
423 /* normal or tunnel packet*/
424 l4_offset = l4.hdr - skb->data;
425 hdr_len = (l4.tcp->doff * 4) + l4_offset;
426
427 /* remove payload length from inner pseudo checksum when tso*/
428 l4_paylen = skb->len - l4_offset;
429 csum_replace_by_diff(&l4.tcp->check,
430 (__force __wsum)htonl(l4_paylen));
431
432 /* find the txbd field values */
433 *paylen = skb->len - hdr_len;
434 hnae_set_bit(*type_cs_vlan_tso,
435 HNS3_TXD_TSO_B, 1);
436
437 /* get MSS for TSO */
438 *mss = skb_shinfo(skb)->gso_size;
439
440 return 0;
441}
442
1898d4e4
S		 443static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
444 u8 *il4_proto)
76ad4f0e
S		 445{
446 union {
447 struct iphdr *v4;
448 struct ipv6hdr *v6;
449 unsigned char *hdr;
450 } l3;
451 unsigned char *l4_hdr;
452 unsigned char *exthdr;
453 u8 l4_proto_tmp;
454 __be16 frag_off;
455
456 /* find outer header point */
457 l3.hdr = skb_network_header(skb);
458 l4_hdr = skb_inner_transport_header(skb);
459
460 if (skb->protocol == htons(ETH_P_IPV6)) {
461 exthdr = l3.hdr + sizeof(*l3.v6);
462 l4_proto_tmp = l3.v6->nexthdr;
463 if (l4_hdr != exthdr)
464 ipv6_skip_exthdr(skb, exthdr - skb->data,
465 &l4_proto_tmp, &frag_off);
466 } else if (skb->protocol == htons(ETH_P_IP)) {
467 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 468 } else {
469 return -EINVAL;
76ad4f0e
S		 470 }
471
472 *ol4_proto = l4_proto_tmp;
473
474 /* tunnel packet */
475 if (!skb->encapsulation) {
476 *il4_proto = 0;
1898d4e4 477 return 0;
76ad4f0e
S		 478 }
479
480 /* find inner header point */
481 l3.hdr = skb_inner_network_header(skb);
482 l4_hdr = skb_inner_transport_header(skb);
483
484 if (l3.v6->version == 6) {
485 exthdr = l3.hdr + sizeof(*l3.v6);
486 l4_proto_tmp = l3.v6->nexthdr;
487 if (l4_hdr != exthdr)
488 ipv6_skip_exthdr(skb, exthdr - skb->data,
489 &l4_proto_tmp, &frag_off);
490 } else if (l3.v4->version == 4) {
491 l4_proto_tmp = l3.v4->protocol;
492 }
493
494 *il4_proto = l4_proto_tmp;
1898d4e4
S		 495
496 return 0;
76ad4f0e
S		 497}
498
499static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
500 u8 il4_proto, u32 *type_cs_vlan_tso,
501 u32 *ol_type_vlan_len_msec)
502{
503 union {
504 struct iphdr *v4;
505 struct ipv6hdr *v6;
506 unsigned char *hdr;
507 } l3;
508 union {
509 struct tcphdr *tcp;
510 struct udphdr *udp;
511 struct gre_base_hdr *gre;
512 unsigned char *hdr;
513 } l4;
514 unsigned char *l2_hdr;
515 u8 l4_proto = ol4_proto;
516 u32 ol2_len;
517 u32 ol3_len;
518 u32 ol4_len;
519 u32 l2_len;
520 u32 l3_len;
521
522 l3.hdr = skb_network_header(skb);
523 l4.hdr = skb_transport_header(skb);
524
525 /* compute L2 header size for normal packet, defined in 2 Bytes */
526 l2_len = l3.hdr - skb->data;
527 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
528 HNS3_TXD_L2LEN_S, l2_len >> 1);
529
530 /* tunnel packet*/
531 if (skb->encapsulation) {
532 /* compute OL2 header size, defined in 2 Bytes */
533 ol2_len = l2_len;
534 hnae_set_field(*ol_type_vlan_len_msec,
535 HNS3_TXD_L2LEN_M,
536 HNS3_TXD_L2LEN_S, ol2_len >> 1);
537
538 /* compute OL3 header size, defined in 4 Bytes */
539 ol3_len = l4.hdr - l3.hdr;
540 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
541 HNS3_TXD_L3LEN_S, ol3_len >> 2);
542
543 /* MAC in UDP, MAC in GRE (0x6558)*/
544 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
545 /* switch MAC header ptr from outer to inner header.*/
546 l2_hdr = skb_inner_mac_header(skb);
547
548 /* compute OL4 header size, defined in 4 Bytes. */
549 ol4_len = l2_hdr - l4.hdr;
550 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
551 HNS3_TXD_L4LEN_S, ol4_len >> 2);
552
553 /* switch IP header ptr from outer to inner header */
554 l3.hdr = skb_inner_network_header(skb);
555
556 /* compute inner l2 header size, defined in 2 Bytes. */
557 l2_len = l3.hdr - l2_hdr;
558 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
559 HNS3_TXD_L2LEN_S, l2_len >> 1);
560 } else {
561 /* skb packet types not supported by hardware,
562 * txbd len fild doesn't be filled.
563 */
564 return;
565 }
566
567 /* switch L4 header pointer from outer to inner */
568 l4.hdr = skb_inner_transport_header(skb);
569
570 l4_proto = il4_proto;
571 }
572
573 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
574 l3_len = l4.hdr - l3.hdr;
575 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
576 HNS3_TXD_L3LEN_S, l3_len >> 2);
577
578 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
579 switch (l4_proto) {
580 case IPPROTO_TCP:
581 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
582 HNS3_TXD_L4LEN_S, l4.tcp->doff);
583 break;
584 case IPPROTO_SCTP:
585 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
586 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
587 break;
588 case IPPROTO_UDP:
589 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
590 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
591 break;
592 default:
593 /* skb packet types not supported by hardware,
594 * txbd len fild doesn't be filled.
595 */
596 return;
597 }
598}
599
600static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
601 u8 il4_proto, u32 *type_cs_vlan_tso,
602 u32 *ol_type_vlan_len_msec)
603{
604 union {
605 struct iphdr *v4;
606 struct ipv6hdr *v6;
607 unsigned char *hdr;
608 } l3;
609 u32 l4_proto = ol4_proto;
610
611 l3.hdr = skb_network_header(skb);
612
613 /* define OL3 type and tunnel type(OL4).*/
614 if (skb->encapsulation) {
615 /* define outer network header type.*/
616 if (skb->protocol == htons(ETH_P_IP)) {
617 if (skb_is_gso(skb))
618 hnae_set_field(*ol_type_vlan_len_msec,
619 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
620 HNS3_OL3T_IPV4_CSUM);
621 else
622 hnae_set_field(*ol_type_vlan_len_msec,
623 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
624 HNS3_OL3T_IPV4_NO_CSUM);
625
626 } else if (skb->protocol == htons(ETH_P_IPV6)) {
627 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
628 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
629 }
630
631 /* define tunnel type(OL4).*/
632 switch (l4_proto) {
633 case IPPROTO_UDP:
634 hnae_set_field(*ol_type_vlan_len_msec,
635 HNS3_TXD_TUNTYPE_M,
636 HNS3_TXD_TUNTYPE_S,
637 HNS3_TUN_MAC_IN_UDP);
638 break;
639 case IPPROTO_GRE:
640 hnae_set_field(*ol_type_vlan_len_msec,
641 HNS3_TXD_TUNTYPE_M,
642 HNS3_TXD_TUNTYPE_S,
643 HNS3_TUN_NVGRE);
644 break;
645 default:
646 /* drop the skb tunnel packet if hardware don't support,
647 * because hardware can't calculate csum when TSO.
648 */
649 if (skb_is_gso(skb))
650 return -EDOM;
651
652 /* the stack computes the IP header already,
653 * driver calculate l4 checksum when not TSO.
654 */
655 skb_checksum_help(skb);
656 return 0;
657 }
658
659 l3.hdr = skb_inner_network_header(skb);
660 l4_proto = il4_proto;
661 }
662
663 if (l3.v4->version == 4) {
664 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
665 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
666
667 /* the stack computes the IP header already, the only time we
668 * need the hardware to recompute it is in the case of TSO.
669 */
670 if (skb_is_gso(skb))
671 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
672
673 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
674 } else if (l3.v6->version == 6) {
675 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
676 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
677 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
678 }
679
680 switch (l4_proto) {
681 case IPPROTO_TCP:
682 hnae_set_field(*type_cs_vlan_tso,
683 HNS3_TXD_L4T_M,
684 HNS3_TXD_L4T_S,
685 HNS3_L4T_TCP);
686 break;
687 case IPPROTO_UDP:
688 hnae_set_field(*type_cs_vlan_tso,
689 HNS3_TXD_L4T_M,
690 HNS3_TXD_L4T_S,
691 HNS3_L4T_UDP);
692 break;
693 case IPPROTO_SCTP:
694 hnae_set_field(*type_cs_vlan_tso,
695 HNS3_TXD_L4T_M,
696 HNS3_TXD_L4T_S,
697 HNS3_L4T_SCTP);
698 break;
699 default:
700 /* drop the skb tunnel packet if hardware don't support,
701 * because hardware can't calculate csum when TSO.
702 */
703 if (skb_is_gso(skb))
704 return -EDOM;
705
706 /* the stack computes the IP header already,
707 * driver calculate l4 checksum when not TSO.
708 */
709 skb_checksum_help(skb);
710 return 0;
711 }
712
713 return 0;
714}
715
716static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
717{
718 /* Config bd buffer end */
719 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
720 HNS3_TXD_BDTYPE_M, 0);
721 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
722 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
7036d26f 723 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 724}
725
726static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
727 int size, dma_addr_t dma, int frag_end,
728 enum hns_desc_type type)
729{
730 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
731 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
732 u32 ol_type_vlan_len_msec = 0;
733 u16 bdtp_fe_sc_vld_ra_ri = 0;
734 u32 type_cs_vlan_tso = 0;
735 struct sk_buff *skb;
736 u32 paylen = 0;
737 u16 mss = 0;
738 __be16 protocol;
739 u8 ol4_proto;
740 u8 il4_proto;
741 int ret;
742
743 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
744 desc_cb->priv = priv;
745 desc_cb->length = size;
746 desc_cb->dma = dma;
747 desc_cb->type = type;
748
749 /* now, fill the descriptor */
750 desc->addr = cpu_to_le64(dma);
751 desc->tx.send_size = cpu_to_le16((u16)size);
752 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
753 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
754
755 if (type == DESC_TYPE_SKB) {
756 skb = (struct sk_buff *)priv;
a90bb9a5 757 paylen = skb->len;
76ad4f0e
S		 758
759 if (skb->ip_summed == CHECKSUM_PARTIAL) {
760 skb_reset_mac_len(skb);
761 protocol = skb->protocol;
762
763 /* vlan packet*/
764 if (protocol == htons(ETH_P_8021Q)) {
765 protocol = vlan_get_protocol(skb);
766 skb->protocol = protocol;
767 }
1898d4e4
S		 768 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
769 if (ret)
770 return ret;
76ad4f0e
S		 771 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
772 &type_cs_vlan_tso,
773 &ol_type_vlan_len_msec);
774 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
775 &type_cs_vlan_tso,
776 &ol_type_vlan_len_msec);
777 if (ret)
778 return ret;
779
780 ret = hns3_set_tso(skb, &paylen, &mss,
781 &type_cs_vlan_tso);
782 if (ret)
783 return ret;
784 }
785
786 /* Set txbd */
787 desc->tx.ol_type_vlan_len_msec =
788 cpu_to_le32(ol_type_vlan_len_msec);
789 desc->tx.type_cs_vlan_tso_len =
790 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 791 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e
S		 792 desc->tx.mss = cpu_to_le16(mss);
793 }
794
795 /* move ring pointer to next.*/
796 ring_ptr_move_fw(ring, next_to_use);
797
798 return 0;
799}
800
801static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
802 int size, dma_addr_t dma, int frag_end,
803 enum hns_desc_type type)
804{
805 unsigned int frag_buf_num;
806 unsigned int k;
807 int sizeoflast;
808 int ret;
809
810 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
811 sizeoflast = size % HNS3_MAX_BD_SIZE;
812 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
813
814 /* When the frag size is bigger than hardware, split this frag */
815 for (k = 0; k < frag_buf_num; k++) {
816 ret = hns3_fill_desc(ring, priv,
817 (k == frag_buf_num - 1) ?
818 sizeoflast : HNS3_MAX_BD_SIZE,
819 dma + HNS3_MAX_BD_SIZE * k,
820 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
821 (type == DESC_TYPE_SKB && !k) ?
822 DESC_TYPE_SKB : DESC_TYPE_PAGE);
823 if (ret)
824 return ret;
825 }
826
827 return 0;
828}
829
830static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
831 struct hns3_enet_ring *ring)
832{
833 struct sk_buff *skb = *out_skb;
834 struct skb_frag_struct *frag;
835 int bdnum_for_frag;
836 int frag_num;
837 int buf_num;
838 int size;
839 int i;
840
841 size = skb_headlen(skb);
842 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
843
844 frag_num = skb_shinfo(skb)->nr_frags;
845 for (i = 0; i < frag_num; i++) {
846 frag = &skb_shinfo(skb)->frags[i];
847 size = skb_frag_size(frag);
848 bdnum_for_frag =
849 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
850 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
851 return -ENOMEM;
852
853 buf_num += bdnum_for_frag;
854 }
855
856 if (buf_num > ring_space(ring))
857 return -EBUSY;
858
859 *bnum = buf_num;
860 return 0;
861}
862
863static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
864 struct hns3_enet_ring *ring)
865{
866 struct sk_buff *skb = *out_skb;
867 int buf_num;
868
869 /* No. of segments (plus a header) */
870 buf_num = skb_shinfo(skb)->nr_frags + 1;
871
872 if (buf_num > ring_space(ring))
873 return -EBUSY;
874
875 *bnum = buf_num;
876
877 return 0;
878}
879
880static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
881{
882 struct device *dev = ring_to_dev(ring);
883 unsigned int i;
884
885 for (i = 0; i < ring->desc_num; i++) {
886 /* check if this is where we started */
887 if (ring->next_to_use == next_to_use_orig)
888 break;
889
890 /* unmap the descriptor dma address */
891 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
892 dma_unmap_single(dev,
893 ring->desc_cb[ring->next_to_use].dma,
894 ring->desc_cb[ring->next_to_use].length,
895 DMA_TO_DEVICE);
896 else
897 dma_unmap_page(dev,
898 ring->desc_cb[ring->next_to_use].dma,
899 ring->desc_cb[ring->next_to_use].length,
900 DMA_TO_DEVICE);
901
902 /* rollback one */
903 ring_ptr_move_bw(ring, next_to_use);
904 }
905}
906
d43e5aca 907netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 908{
909 struct hns3_nic_priv *priv = netdev_priv(netdev);
910 struct hns3_nic_ring_data *ring_data =
911 &tx_ring_data(priv, skb->queue_mapping);
912 struct hns3_enet_ring *ring = ring_data->ring;
913 struct device *dev = priv->dev;
914 struct netdev_queue *dev_queue;
915 struct skb_frag_struct *frag;
916 int next_to_use_head;
917 int next_to_use_frag;
918 dma_addr_t dma;
919 int buf_num;
920 int seg_num;
921 int size;
922 int ret;
923 int i;
924
925 /* Prefetch the data used later */
926 prefetch(skb->data);
927
928 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
929 case -EBUSY:
930 u64_stats_update_begin(&ring->syncp);
931 ring->stats.tx_busy++;
932 u64_stats_update_end(&ring->syncp);
933
934 goto out_net_tx_busy;
935 case -ENOMEM:
936 u64_stats_update_begin(&ring->syncp);
937 ring->stats.sw_err_cnt++;
938 u64_stats_update_end(&ring->syncp);
939 netdev_err(netdev, "no memory to xmit!\n");
940
941 goto out_err_tx_ok;
942 default:
943 break;
944 }
945
946 /* No. of segments (plus a header) */
947 seg_num = skb_shinfo(skb)->nr_frags + 1;
948 /* Fill the first part */
949 size = skb_headlen(skb);
950
951 next_to_use_head = ring->next_to_use;
952
953 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
954 if (dma_mapping_error(dev, dma)) {
955 netdev_err(netdev, "TX head DMA map failed\n");
956 ring->stats.sw_err_cnt++;
957 goto out_err_tx_ok;
958 }
959
960 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
961 DESC_TYPE_SKB);
962 if (ret)
963 goto head_dma_map_err;
964
965 next_to_use_frag = ring->next_to_use;
966 /* Fill the fragments */
967 for (i = 1; i < seg_num; i++) {
968 frag = &skb_shinfo(skb)->frags[i - 1];
969 size = skb_frag_size(frag);
970 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
971 if (dma_mapping_error(dev, dma)) {
972 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
973 ring->stats.sw_err_cnt++;
974 goto frag_dma_map_err;
975 }
976 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
977 seg_num - 1 == i ? 1 : 0,
978 DESC_TYPE_PAGE);
979
980 if (ret)
981 goto frag_dma_map_err;
982 }
983
984 /* Complete translate all packets */
985 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
986 netdev_tx_sent_queue(dev_queue, skb->len);
987
988 wmb(); /* Commit all data before submit */
989
990 hnae_queue_xmit(ring->tqp, buf_num);
991
992 return NETDEV_TX_OK;
993
994frag_dma_map_err:
995 hns_nic_dma_unmap(ring, next_to_use_frag);
996
997head_dma_map_err:
998 hns_nic_dma_unmap(ring, next_to_use_head);
999
1000out_err_tx_ok:
1001 dev_kfree_skb_any(skb);
1002 return NETDEV_TX_OK;
1003
1004out_net_tx_busy:
1005 netif_stop_subqueue(netdev, ring_data->queue_index);
1006 smp_mb(); /* Commit all data before submit */
1007
1008 return NETDEV_TX_BUSY;
1009}
1010
1011static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1012{
9780cb97 1013 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1014 struct sockaddr *mac_addr = p;
1015 int ret;
1016
1017 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1018 return -EADDRNOTAVAIL;
1019
1020 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
1021 if (ret) {
1022 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1023 return ret;
1024 }
1025
1026 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1027
1028 return 0;
1029}
1030
1031static int hns3_nic_set_features(struct net_device *netdev,
1032 netdev_features_t features)
1033{
1034 struct hns3_nic_priv *priv = netdev_priv(netdev);
5f9a7732
PL		 1035 struct hnae3_handle *h = priv->ae_handle;
1036 netdev_features_t changed;
1037 int ret;
76ad4f0e
S		 1038
1039 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1040 priv->ops.fill_desc = hns3_fill_desc_tso;
1041 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1042 } else {
1043 priv->ops.fill_desc = hns3_fill_desc;
1044 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1045 }
1046
5f9a7732
PL		 1047 changed = netdev->features ^ features;
1048 if (changed & NETIF_F_HW_VLAN_CTAG_RX) {
1049 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1050 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1051 else
1052 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1053
1054 if (ret)
1055 return ret;
1056 }
1057
76ad4f0e
S		 1058 netdev->features = features;
1059 return 0;
1060}
1061
1062static void
1063hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
1064{
1065 struct hns3_nic_priv *priv = netdev_priv(netdev);
1066 int queue_num = priv->ae_handle->kinfo.num_tqps;
1067 struct hns3_enet_ring *ring;
1068 unsigned int start;
1069 unsigned int idx;
1070 u64 tx_bytes = 0;
1071 u64 rx_bytes = 0;
1072 u64 tx_pkts = 0;
1073 u64 rx_pkts = 0;
1074
1075 for (idx = 0; idx < queue_num; idx++) {
1076 /* fetch the tx stats */
1077 ring = priv->ring_data[idx].ring;
1078 do {
d36d36ce 1079 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1080 tx_bytes += ring->stats.tx_bytes;
1081 tx_pkts += ring->stats.tx_pkts;
1082 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1083
1084 /* fetch the rx stats */
1085 ring = priv->ring_data[idx + queue_num].ring;
1086 do {
d36d36ce 1087 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1088 rx_bytes += ring->stats.rx_bytes;
1089 rx_pkts += ring->stats.rx_pkts;
1090 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1091 }
1092
1093 stats->tx_bytes = tx_bytes;
1094 stats->tx_packets = tx_pkts;
1095 stats->rx_bytes = rx_bytes;
1096 stats->rx_packets = rx_pkts;
1097
1098 stats->rx_errors = netdev->stats.rx_errors;
1099 stats->multicast = netdev->stats.multicast;
1100 stats->rx_length_errors = netdev->stats.rx_length_errors;
1101 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1102 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1103
1104 stats->tx_errors = netdev->stats.tx_errors;
1105 stats->rx_dropped = netdev->stats.rx_dropped;
1106 stats->tx_dropped = netdev->stats.tx_dropped;
1107 stats->collisions = netdev->stats.collisions;
1108 stats->rx_over_errors = netdev->stats.rx_over_errors;
1109 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1110 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1111 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1112 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1113 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1114 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1115 stats->tx_window_errors = netdev->stats.tx_window_errors;
1116 stats->rx_compressed = netdev->stats.rx_compressed;
1117 stats->tx_compressed = netdev->stats.tx_compressed;
1118}
1119
1120static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1121 enum hns3_udp_tnl_type type)
1122{
1123 struct hns3_nic_priv *priv = netdev_priv(netdev);
1124 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1125 struct hnae3_handle *h = priv->ae_handle;
1126
1127 if (udp_tnl->used && udp_tnl->dst_port == port) {
1128 udp_tnl->used++;
1129 return;
1130 }
1131
1132 if (udp_tnl->used) {
1133 netdev_warn(netdev,
1134 "UDP tunnel [%d], port [%d] offload\n", type, port);
1135 return;
1136 }
1137
1138 udp_tnl->dst_port = port;
1139 udp_tnl->used = 1;
1140 /* TBD send command to hardware to add port */
1141 if (h->ae_algo->ops->add_tunnel_udp)
1142 h->ae_algo->ops->add_tunnel_udp(h, port);
1143}
1144
1145static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1146 enum hns3_udp_tnl_type type)
1147{
1148 struct hns3_nic_priv *priv = netdev_priv(netdev);
1149 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1150 struct hnae3_handle *h = priv->ae_handle;
1151
1152 if (!udp_tnl->used || udp_tnl->dst_port != port) {
1153 netdev_warn(netdev,
1154 "Invalid UDP tunnel port %d\n", port);
1155 return;
1156 }
1157
1158 udp_tnl->used--;
1159 if (udp_tnl->used)
1160 return;
1161
1162 udp_tnl->dst_port = 0;
1163 /* TBD send command to hardware to del port */
1164 if (h->ae_algo->ops->del_tunnel_udp)
9537e7cb 1165 h->ae_algo->ops->del_tunnel_udp(h, port);
76ad4f0e
S		 1166}
1167
1168/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1169 * @netdev: This physical ports's netdev
1170 * @ti: Tunnel information
1171 */
1172static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1173 struct udp_tunnel_info *ti)
1174{
1175 u16 port_n = ntohs(ti->port);
1176
1177 switch (ti->type) {
1178 case UDP_TUNNEL_TYPE_VXLAN:
1179 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1180 break;
1181 case UDP_TUNNEL_TYPE_GENEVE:
1182 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1183 break;
1184 default:
1185 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1186 break;
1187 }
1188}
1189
1190static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1191 struct udp_tunnel_info *ti)
1192{
1193 u16 port_n = ntohs(ti->port);
1194
1195 switch (ti->type) {
1196 case UDP_TUNNEL_TYPE_VXLAN:
1197 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1198 break;
1199 case UDP_TUNNEL_TYPE_GENEVE:
1200 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1201 break;
1202 default:
1203 break;
1204 }
1205}
1206
30d240df 1207static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1208{
30d240df 1209 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1210 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1211 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1212 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1213 u8 tc = mqprio_qopt->qopt.num_tc;
1214 u16 mode = mqprio_qopt->mode;
1215 u8 hw = mqprio_qopt->qopt.hw;
1216 bool if_running;
76ad4f0e
S		 1217 unsigned int i;
1218 int ret;
1219
30d240df
YL		 1220 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1221 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1222 return -EOPNOTSUPP;
1223
76ad4f0e
S		 1224 if (tc > HNAE3_MAX_TC)
1225 return -EINVAL;
1226
76ad4f0e
S		 1227 if (!netdev)
1228 return -EINVAL;
1229
30d240df
YL		 1230 if_running = netif_running(netdev);
1231 if (if_running) {
1232 hns3_nic_net_stop(netdev);
1233 msleep(100);
76ad4f0e
S		 1234 }
1235
30d240df
YL		 1236 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1237 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1238 if (ret)
30d240df
YL		 1239 goto out;
1240
1241 if (tc <= 1) {
1242 netdev_reset_tc(netdev);
1243 } else {
1244 ret = netdev_set_num_tc(netdev, tc);
1245 if (ret)
1246 goto out;
1247
1248 for (i = 0; i < HNAE3_MAX_TC; i++) {
1249 if (!kinfo->tc_info[i].enable)
1250 continue;
76ad4f0e 1251
76ad4f0e
S		 1252 netdev_set_tc_queue(netdev,
1253 kinfo->tc_info[i].tc,
1254 kinfo->tc_info[i].tqp_count,
1255 kinfo->tc_info[i].tqp_offset);
30d240df 1256 }
76ad4f0e
S		 1257 }
1258
30d240df
YL		 1259 ret = hns3_nic_set_real_num_queue(netdev);
1260
1261out:
1262 if (if_running)
1263 hns3_nic_net_open(netdev);
1264
1265 return ret;
76ad4f0e
S		 1266}
1267
2572ac53 1268static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1269 void *type_data)
76ad4f0e 1270{
575ed7d3 1271 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1272 return -EOPNOTSUPP;
76ad4f0e 1273
30d240df 1274 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1275}
1276
1277static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1278 __be16 proto, u16 vid)
1279{
9780cb97 1280 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1281 int ret = -EIO;
1282
1283 if (h->ae_algo->ops->set_vlan_filter)
1284 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1285
1286 return ret;
1287}
1288
1289static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1290 __be16 proto, u16 vid)
1291{
9780cb97 1292 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1293 int ret = -EIO;
1294
1295 if (h->ae_algo->ops->set_vlan_filter)
1296 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1297
1298 return ret;
1299}
1300
1301static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1302 u8 qos, __be16 vlan_proto)
1303{
9780cb97 1304 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1305 int ret = -EIO;
1306
1307 if (h->ae_algo->ops->set_vf_vlan_filter)
1308 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1309 qos, vlan_proto);
1310
1311 return ret;
1312}
1313
a8e8b7ff
S		 1314static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1315{
9780cb97 1316 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1317 bool if_running = netif_running(netdev);
1318 int ret;
1319
1320 if (!h->ae_algo->ops->set_mtu)
1321 return -EOPNOTSUPP;
1322
1323 /* if this was called with netdev up then bring netdevice down */
1324 if (if_running) {
1325 (void)hns3_nic_net_stop(netdev);
1326 msleep(100);
1327 }
1328
1329 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1330 if (ret) {
1331 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1332 ret);
1333 return ret;
1334 }
1335
1336 /* if the netdev was running earlier, bring it up again */
1337 if (if_running && hns3_nic_net_open(netdev))
1338 ret = -EINVAL;
1339
1340 return ret;
1341}
1342
f8fa222c
L		 1343static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1344{
1345 struct hns3_nic_priv *priv = netdev_priv(ndev);
1346 struct hns3_enet_ring *tx_ring = NULL;
1347 int timeout_queue = 0;
1348 int hw_head, hw_tail;
1349 int i;
1350
1351 /* Find the stopped queue the same way the stack does */
1352 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1353 struct netdev_queue *q;
1354 unsigned long trans_start;
1355
1356 q = netdev_get_tx_queue(ndev, i);
1357 trans_start = q->trans_start;
1358 if (netif_xmit_stopped(q) &&
1359 time_after(jiffies,
1360 (trans_start + ndev->watchdog_timeo))) {
1361 timeout_queue = i;
1362 break;
1363 }
1364 }
1365
1366 if (i == ndev->num_tx_queues) {
1367 netdev_info(ndev,
1368 "no netdev TX timeout queue found, timeout count: %llu\n",
1369 priv->tx_timeout_count);
1370 return false;
1371 }
1372
1373 tx_ring = priv->ring_data[timeout_queue].ring;
1374
1375 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1376 HNS3_RING_TX_RING_HEAD_REG);
1377 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1378 HNS3_RING_TX_RING_TAIL_REG);
1379 netdev_info(ndev,
1380 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1381 priv->tx_timeout_count,
1382 timeout_queue,
1383 tx_ring->next_to_use,
1384 tx_ring->next_to_clean,
1385 hw_head,
1386 hw_tail,
1387 readl(tx_ring->tqp_vector->mask_addr));
1388
1389 return true;
1390}
1391
1392static void hns3_nic_net_timeout(struct net_device *ndev)
1393{
1394 struct hns3_nic_priv *priv = netdev_priv(ndev);
1395 unsigned long last_reset_time = priv->last_reset_time;
1396 struct hnae3_handle *h = priv->ae_handle;
1397
1398 if (!hns3_get_tx_timeo_queue_info(ndev))
1399 return;
1400
1401 priv->tx_timeout_count++;
1402
1403 /* This timeout is far away enough from last timeout,
1404 * if timeout again,set the reset type to PF reset
1405 */
1406 if (time_after(jiffies, (last_reset_time + 20 * HZ)))
1407 priv->reset_level = HNAE3_FUNC_RESET;
1408
1409 /* Don't do any new action before the next timeout */
1410 else if (time_before(jiffies, (last_reset_time + ndev->watchdog_timeo)))
1411 return;
1412
1413 priv->last_reset_time = jiffies;
1414
1415 if (h->ae_algo->ops->reset_event)
1416 h->ae_algo->ops->reset_event(h, priv->reset_level);
1417
1418 priv->reset_level++;
1419 if (priv->reset_level > HNAE3_GLOBAL_RESET)
1420 priv->reset_level = HNAE3_GLOBAL_RESET;
1421}
1422
76ad4f0e
S		 1423static const struct net_device_ops hns3_nic_netdev_ops = {
1424 .ndo_open = hns3_nic_net_open,
1425 .ndo_stop = hns3_nic_net_stop,
1426 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1427 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1428 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a8e8b7ff 1429 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1430 .ndo_set_features = hns3_nic_set_features,
1431 .ndo_get_stats64 = hns3_nic_get_stats64,
1432 .ndo_setup_tc = hns3_nic_setup_tc,
1433 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
1434 .ndo_udp_tunnel_add = hns3_nic_udp_tunnel_add,
1435 .ndo_udp_tunnel_del = hns3_nic_udp_tunnel_del,
1436 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1437 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1438 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1439};
1440
1441/* hns3_probe - Device initialization routine
1442 * @pdev: PCI device information struct
1443 * @ent: entry in hns3_pci_tbl
1444 *
1445 * hns3_probe initializes a PF identified by a pci_dev structure.
1446 * The OS initialization, configuring of the PF private structure,
1447 * and a hardware reset occur.
1448 *
1449 * Returns 0 on success, negative on failure
1450 */
1451static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1452{
1453 struct hnae3_ae_dev *ae_dev;
1454 int ret;
1455
1456 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1457 GFP_KERNEL);
1458 if (!ae_dev) {
1459 ret = -ENOMEM;
1460 return ret;
1461 }
1462
1463 ae_dev->pdev = pdev;
e92a0843 1464 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 1465 ae_dev->dev_type = HNAE3_DEV_KNIC;
1466 pci_set_drvdata(pdev, ae_dev);
1467
1468 return hnae3_register_ae_dev(ae_dev);
1469}
1470
1471/* hns3_remove - Device removal routine
1472 * @pdev: PCI device information struct
1473 */
1474static void hns3_remove(struct pci_dev *pdev)
1475{
1476 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1477
1478 hnae3_unregister_ae_dev(ae_dev);
1479
1480 devm_kfree(&pdev->dev, ae_dev);
1481
1482 pci_set_drvdata(pdev, NULL);
1483}
1484
1485static struct pci_driver hns3_driver = {
1486 .name = hns3_driver_name,
1487 .id_table = hns3_pci_tbl,
1488 .probe = hns3_probe,
1489 .remove = hns3_remove,
1490};
1491
1492/* set default feature to hns3 */
1493static void hns3_set_default_feature(struct net_device *netdev)
1494{
1495 netdev->priv_flags |= IFF_UNICAST_FLT;
1496
1497 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1498 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1499 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1500 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1501 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1502
1503 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1504
1505 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1506
1507 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1508 NETIF_F_HW_VLAN_CTAG_FILTER |
5f9a7732 1509 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1510 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1511 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1512 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1513 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1514
1515 netdev->vlan_features |=
1516 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1517 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1518 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1519 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1520 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1521
1522 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1523 NETIF_F_HW_VLAN_CTAG_FILTER |
5f9a7732 1524 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1525 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1526 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1527 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1528 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1529}
1530
1531static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1532 struct hns3_desc_cb *cb)
1533{
1534 unsigned int order = hnae_page_order(ring);
1535 struct page *p;
1536
1537 p = dev_alloc_pages(order);
1538 if (!p)
1539 return -ENOMEM;
1540
1541 cb->priv = p;
1542 cb->page_offset = 0;
1543 cb->reuse_flag = 0;
1544 cb->buf = page_address(p);
1545 cb->length = hnae_page_size(ring);
1546 cb->type = DESC_TYPE_PAGE;
1547
76ad4f0e
S		 1548 return 0;
1549}
1550
1551static void hns3_free_buffer(struct hns3_enet_ring *ring,
1552 struct hns3_desc_cb *cb)
1553{
1554 if (cb->type == DESC_TYPE_SKB)
1555 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1556 else if (!HNAE3_IS_TX_RING(ring))
1557 put_page((struct page *)cb->priv);
1558 memset(cb, 0, sizeof(*cb));
1559}
1560
1561static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1562{
1563 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1564 cb->length, ring_to_dma_dir(ring));
1565
1566 if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1567 return -EIO;
1568
1569 return 0;
1570}
1571
1572static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1573 struct hns3_desc_cb *cb)
1574{
1575 if (cb->type == DESC_TYPE_SKB)
1576 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1577 ring_to_dma_dir(ring));
1578 else
1579 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1580 ring_to_dma_dir(ring));
1581}
1582
1583static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1584{
1585 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1586 ring->desc[i].addr = 0;
1587}
1588
1589static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1590{
1591 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1592
1593 if (!ring->desc_cb[i].dma)
1594 return;
1595
1596 hns3_buffer_detach(ring, i);
1597 hns3_free_buffer(ring, cb);
1598}
1599
1600static void hns3_free_buffers(struct hns3_enet_ring *ring)
1601{
1602 int i;
1603
1604 for (i = 0; i < ring->desc_num; i++)
1605 hns3_free_buffer_detach(ring, i);
1606}
1607
1608/* free desc along with its attached buffer */
1609static void hns3_free_desc(struct hns3_enet_ring *ring)
1610{
1611 hns3_free_buffers(ring);
1612
1613 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1614 ring->desc_num * sizeof(ring->desc[0]),
1615 DMA_BIDIRECTIONAL);
1616 ring->desc_dma_addr = 0;
1617 kfree(ring->desc);
1618 ring->desc = NULL;
1619}
1620
1621static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1622{
1623 int size = ring->desc_num * sizeof(ring->desc[0]);
1624
1625 ring->desc = kzalloc(size, GFP_KERNEL);
1626 if (!ring->desc)
1627 return -ENOMEM;
1628
1629 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1630 size, DMA_BIDIRECTIONAL);
1631 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1632 ring->desc_dma_addr = 0;
1633 kfree(ring->desc);
1634 ring->desc = NULL;
1635 return -ENOMEM;
1636 }
1637
1638 return 0;
1639}
1640
1641static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1642 struct hns3_desc_cb *cb)
1643{
1644 int ret;
1645
1646 ret = hns3_alloc_buffer(ring, cb);
1647 if (ret)
1648 goto out;
1649
1650 ret = hns3_map_buffer(ring, cb);
1651 if (ret)
1652 goto out_with_buf;
1653
1654 return 0;
1655
1656out_with_buf:
564883bb 1657 hns3_free_buffer(ring, cb);
76ad4f0e
S		 1658out:
1659 return ret;
1660}
1661
1662static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1663{
1664 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1665
1666 if (ret)
1667 return ret;
1668
1669 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1670
1671 return 0;
1672}
1673
1674/* Allocate memory for raw pkg, and map with dma */
1675static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1676{
1677 int i, j, ret;
1678
1679 for (i = 0; i < ring->desc_num; i++) {
1680 ret = hns3_alloc_buffer_attach(ring, i);
1681 if (ret)
1682 goto out_buffer_fail;
1683 }
1684
1685 return 0;
1686
1687out_buffer_fail:
1688 for (j = i - 1; j >= 0; j--)
1689 hns3_free_buffer_detach(ring, j);
1690 return ret;
1691}
1692
1693/* detach a in-used buffer and replace with a reserved one */
1694static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1695 struct hns3_desc_cb *res_cb)
1696{
b9077428 1697 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 1698 ring->desc_cb[i] = *res_cb;
1699 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1700}
1701
1702static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1703{
1704 ring->desc_cb[i].reuse_flag = 0;
1705 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1706 + ring->desc_cb[i].page_offset);
1707}
1708
1709static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1710 int *pkts)
1711{
1712 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1713
1714 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1715 (*bytes) += desc_cb->length;
1716 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1717 hns3_free_buffer_detach(ring, ring->next_to_clean);
1718
1719 ring_ptr_move_fw(ring, next_to_clean);
1720}
1721
1722static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1723{
1724 int u = ring->next_to_use;
1725 int c = ring->next_to_clean;
1726
1727 if (unlikely(h > ring->desc_num))
1728 return 0;
1729
1730 return u > c ? (h > c && h <= u) : (h > c || h <= u);
1731}
1732
24e750c4 1733bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e
S		 1734{
1735 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1736 struct netdev_queue *dev_queue;
1737 int bytes, pkts;
1738 int head;
1739
1740 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1741 rmb(); /* Make sure head is ready before touch any data */
1742
1743 if (is_ring_empty(ring) || head == ring->next_to_clean)
24e750c4 1744 return true; /* no data to poll */
76ad4f0e
S		 1745
1746 if (!is_valid_clean_head(ring, head)) {
1747 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1748 ring->next_to_use, ring->next_to_clean);
1749
1750 u64_stats_update_begin(&ring->syncp);
1751 ring->stats.io_err_cnt++;
1752 u64_stats_update_end(&ring->syncp);
24e750c4 1753 return true;
76ad4f0e
S		 1754 }
1755
1756 bytes = 0;
1757 pkts = 0;
1758 while (head != ring->next_to_clean && budget) {
1759 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1760 /* Issue prefetch for next Tx descriptor */
1761 prefetch(&ring->desc_cb[ring->next_to_clean]);
1762 budget--;
1763 }
1764
1765 ring->tqp_vector->tx_group.total_bytes += bytes;
1766 ring->tqp_vector->tx_group.total_packets += pkts;
1767
1768 u64_stats_update_begin(&ring->syncp);
1769 ring->stats.tx_bytes += bytes;
1770 ring->stats.tx_pkts += pkts;
1771 u64_stats_update_end(&ring->syncp);
1772
1773 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1774 netdev_tx_completed_queue(dev_queue, pkts, bytes);
1775
1776 if (unlikely(pkts && netif_carrier_ok(netdev) &&
1777 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1778 /* Make sure that anybody stopping the queue after this
1779 * sees the new next_to_clean.
1780 */
1781 smp_mb();
1782 if (netif_tx_queue_stopped(dev_queue)) {
1783 netif_tx_wake_queue(dev_queue);
1784 ring->stats.restart_queue++;
1785 }
1786 }
1787
1788 return !!budget;
1789}
1790
1791static int hns3_desc_unused(struct hns3_enet_ring *ring)
1792{
1793 int ntc = ring->next_to_clean;
1794 int ntu = ring->next_to_use;
1795
1796 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1797}
1798
1799static void
1800hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1801{
1802 struct hns3_desc_cb *desc_cb;
1803 struct hns3_desc_cb res_cbs;
1804 int i, ret;
1805
1806 for (i = 0; i < cleand_count; i++) {
1807 desc_cb = &ring->desc_cb[ring->next_to_use];
1808 if (desc_cb->reuse_flag) {
1809 u64_stats_update_begin(&ring->syncp);
1810 ring->stats.reuse_pg_cnt++;
1811 u64_stats_update_end(&ring->syncp);
1812
1813 hns3_reuse_buffer(ring, ring->next_to_use);
1814 } else {
1815 ret = hns3_reserve_buffer_map(ring, &res_cbs);
1816 if (ret) {
1817 u64_stats_update_begin(&ring->syncp);
1818 ring->stats.sw_err_cnt++;
1819 u64_stats_update_end(&ring->syncp);
1820
1821 netdev_err(ring->tqp->handle->kinfo.netdev,
1822 "hnae reserve buffer map failed.\n");
1823 break;
1824 }
1825 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1826 }
1827
1828 ring_ptr_move_fw(ring, next_to_use);
1829 }
1830
1831 wmb(); /* Make all data has been write before submit */
1832 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1833}
1834
1835/* hns3_nic_get_headlen - determine size of header for LRO/GRO
1836 * @data: pointer to the start of the headers
1837 * @max: total length of section to find headers in
1838 *
1839 * This function is meant to determine the length of headers that will
1840 * be recognized by hardware for LRO, GRO, and RSC offloads. The main
1841 * motivation of doing this is to only perform one pull for IPv4 TCP
1842 * packets so that we can do basic things like calculating the gso_size
1843 * based on the average data per packet.
1844 */
1845static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1846 unsigned int max_size)
1847{
1848 unsigned char *network;
1849 u8 hlen;
1850
1851 /* This should never happen, but better safe than sorry */
1852 if (max_size < ETH_HLEN)
1853 return max_size;
1854
1855 /* Initialize network frame pointer */
1856 network = data;
1857
1858 /* Set first protocol and move network header forward */
1859 network += ETH_HLEN;
1860
1861 /* Handle any vlan tag if present */
1862 if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1863 == HNS3_RX_FLAG_VLAN_PRESENT) {
1864 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1865 return max_size;
1866
1867 network += VLAN_HLEN;
1868 }
1869
1870 /* Handle L3 protocols */
1871 if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1872 == HNS3_RX_FLAG_L3ID_IPV4) {
1873 if ((typeof(max_size))(network - data) >
1874 (max_size - sizeof(struct iphdr)))
1875 return max_size;
1876
1877 /* Access ihl as a u8 to avoid unaligned access on ia64 */
1878 hlen = (network[0] & 0x0F) << 2;
1879
1880 /* Verify hlen meets minimum size requirements */
1881 if (hlen < sizeof(struct iphdr))
1882 return network - data;
1883
1884 /* Record next protocol if header is present */
1885 } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1886 == HNS3_RX_FLAG_L3ID_IPV6) {
1887 if ((typeof(max_size))(network - data) >
1888 (max_size - sizeof(struct ipv6hdr)))
1889 return max_size;
1890
1891 /* Record next protocol */
1892 hlen = sizeof(struct ipv6hdr);
1893 } else {
1894 return network - data;
1895 }
1896
1897 /* Relocate pointer to start of L4 header */
1898 network += hlen;
1899
1900 /* Finally sort out TCP/UDP */
1901 if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1902 == HNS3_RX_FLAG_L4ID_TCP) {
1903 if ((typeof(max_size))(network - data) >
1904 (max_size - sizeof(struct tcphdr)))
1905 return max_size;
1906
1907 /* Access doff as a u8 to avoid unaligned access on ia64 */
1908 hlen = (network[12] & 0xF0) >> 2;
1909
1910 /* Verify hlen meets minimum size requirements */
1911 if (hlen < sizeof(struct tcphdr))
1912 return network - data;
1913
1914 network += hlen;
1915 } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1916 == HNS3_RX_FLAG_L4ID_UDP) {
1917 if ((typeof(max_size))(network - data) >
1918 (max_size - sizeof(struct udphdr)))
1919 return max_size;
1920
1921 network += sizeof(struct udphdr);
1922 }
1923
1924 /* If everything has gone correctly network should be the
1925 * data section of the packet and will be the end of the header.
1926 * If not then it probably represents the end of the last recognized
1927 * header.
1928 */
1929 if ((typeof(max_size))(network - data) < max_size)
1930 return network - data;
1931 else
1932 return max_size;
1933}
1934
1935static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
1936 struct hns3_enet_ring *ring, int pull_len,
1937 struct hns3_desc_cb *desc_cb)
1938{
1939 struct hns3_desc *desc;
1940 int truesize, size;
1941 int last_offset;
1942 bool twobufs;
1943
1944 twobufs = ((PAGE_SIZE < 8192) &&
1945 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
1946
1947 desc = &ring->desc[ring->next_to_clean];
1948 size = le16_to_cpu(desc->rx.size);
1949
1950 if (twobufs) {
1951 truesize = hnae_buf_size(ring);
1952 } else {
1953 truesize = ALIGN(size, L1_CACHE_BYTES);
1954 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
1955 }
1956
1957 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
1958 size - pull_len, truesize - pull_len);
1959
1960 /* Avoid re-using remote pages,flag default unreuse */
1961 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
1962 return;
1963
1964 if (twobufs) {
1965 /* If we are only owner of page we can reuse it */
1966 if (likely(page_count(desc_cb->priv) == 1)) {
1967 /* Flip page offset to other buffer */
1968 desc_cb->page_offset ^= truesize;
1969
1970 desc_cb->reuse_flag = 1;
1971 /* bump ref count on page before it is given*/
1972 get_page(desc_cb->priv);
1973 }
1974 return;
1975 }
1976
1977 /* Move offset up to the next cache line */
1978 desc_cb->page_offset += truesize;
1979
1980 if (desc_cb->page_offset <= last_offset) {
1981 desc_cb->reuse_flag = 1;
1982 /* Bump ref count on page before it is given*/
1983 get_page(desc_cb->priv);
1984 }
1985}
1986
1987static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
1988 struct hns3_desc *desc)
1989{
1990 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1991 int l3_type, l4_type;
1992 u32 bd_base_info;
1993 int ol4_type;
1994 u32 l234info;
1995
1996 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1997 l234info = le32_to_cpu(desc->rx.l234_info);
1998
1999 skb->ip_summed = CHECKSUM_NONE;
2000
2001 skb_checksum_none_assert(skb);
2002
2003 if (!(netdev->features & NETIF_F_RXCSUM))
2004 return;
2005
2006 /* check if hardware has done checksum */
2007 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2008 return;
2009
2010 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
2011 hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
2012 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2013 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2014 netdev_err(netdev, "L3/L4 error pkt\n");
2015 u64_stats_update_begin(&ring->syncp);
2016 ring->stats.l3l4_csum_err++;
2017 u64_stats_update_end(&ring->syncp);
2018
2019 return;
2020 }
2021
2022 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
2023 HNS3_RXD_L3ID_S);
2024 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
2025 HNS3_RXD_L4ID_S);
2026
2027 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
2028 switch (ol4_type) {
2029 case HNS3_OL4_TYPE_MAC_IN_UDP:
2030 case HNS3_OL4_TYPE_NVGRE:
2031 skb->csum_level = 1;
2032 case HNS3_OL4_TYPE_NO_TUN:
2033 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2034 if (l3_type == HNS3_L3_TYPE_IPV4 ||
2035 (l3_type == HNS3_L3_TYPE_IPV6 &&
2036 (l4_type == HNS3_L4_TYPE_UDP ||
2037 l4_type == HNS3_L4_TYPE_TCP ||
2038 l4_type == HNS3_L4_TYPE_SCTP)))
2039 skb->ip_summed = CHECKSUM_UNNECESSARY;
2040 break;
2041 }
2042}
2043
d43e5aca
YL		 2044static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2045{
2046 napi_gro_receive(&ring->tqp_vector->napi, skb);
2047}
2048
76ad4f0e
S		 2049static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2050 struct sk_buff **out_skb, int *out_bnum)
2051{
2052 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2053 struct hns3_desc_cb *desc_cb;
2054 struct hns3_desc *desc;
2055 struct sk_buff *skb;
2056 unsigned char *va;
2057 u32 bd_base_info;
2058 int pull_len;
2059 u32 l234info;
2060 int length;
2061 int bnum;
2062
2063 desc = &ring->desc[ring->next_to_clean];
2064 desc_cb = &ring->desc_cb[ring->next_to_clean];
2065
2066 prefetch(desc);
2067
2068 length = le16_to_cpu(desc->rx.pkt_len);
2069 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2070 l234info = le32_to_cpu(desc->rx.l234_info);
2071
2072 /* Check valid BD */
2073 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2074 return -EFAULT;
2075
2076 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2077
2078 /* Prefetch first cache line of first page
2079 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2080 * line size is 64B so need to prefetch twice to make it 128B. But in
2081 * actual we can have greater size of caches with 128B Level 1 cache
2082 * lines. In such a case, single fetch would suffice to cache in the
2083 * relevant part of the header.
2084 */
2085 prefetch(va);
2086#if L1_CACHE_BYTES < 128
2087 prefetch(va + L1_CACHE_BYTES);
2088#endif
2089
2090 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2091 HNS3_RX_HEAD_SIZE);
2092 if (unlikely(!skb)) {
2093 netdev_err(netdev, "alloc rx skb fail\n");
2094
2095 u64_stats_update_begin(&ring->syncp);
2096 ring->stats.sw_err_cnt++;
2097 u64_stats_update_end(&ring->syncp);
2098
2099 return -ENOMEM;
2100 }
2101
2102 prefetchw(skb->data);
2103
2104 bnum = 1;
2105 if (length <= HNS3_RX_HEAD_SIZE) {
2106 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2107
2108 /* We can reuse buffer as-is, just make sure it is local */
2109 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2110 desc_cb->reuse_flag = 1;
2111 else /* This page cannot be reused so discard it */
2112 put_page(desc_cb->priv);
2113
2114 ring_ptr_move_fw(ring, next_to_clean);
2115 } else {
2116 u64_stats_update_begin(&ring->syncp);
2117 ring->stats.seg_pkt_cnt++;
2118 u64_stats_update_end(&ring->syncp);
2119
2120 pull_len = hns3_nic_get_headlen(va, l234info,
2121 HNS3_RX_HEAD_SIZE);
2122 memcpy(__skb_put(skb, pull_len), va,
2123 ALIGN(pull_len, sizeof(long)));
2124
2125 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2126 ring_ptr_move_fw(ring, next_to_clean);
2127
2128 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2129 desc = &ring->desc[ring->next_to_clean];
2130 desc_cb = &ring->desc_cb[ring->next_to_clean];
2131 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2132 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2133 ring_ptr_move_fw(ring, next_to_clean);
2134 bnum++;
2135 }
2136 }
2137
2138 *out_bnum = bnum;
2139
2140 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2141 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2142 ((u64 *)desc)[0], ((u64 *)desc)[1]);
2143 u64_stats_update_begin(&ring->syncp);
2144 ring->stats.non_vld_descs++;
2145 u64_stats_update_end(&ring->syncp);
2146
2147 dev_kfree_skb_any(skb);
2148 return -EINVAL;
2149 }
2150
2151 if (unlikely((!desc->rx.pkt_len) ||
2152 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2153 netdev_err(netdev, "truncated pkt\n");
2154 u64_stats_update_begin(&ring->syncp);
2155 ring->stats.err_pkt_len++;
2156 u64_stats_update_end(&ring->syncp);
2157
2158 dev_kfree_skb_any(skb);
2159 return -EFAULT;
2160 }
2161
2162 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2163 netdev_err(netdev, "L2 error pkt\n");
2164 u64_stats_update_begin(&ring->syncp);
2165 ring->stats.l2_err++;
2166 u64_stats_update_end(&ring->syncp);
2167
2168 dev_kfree_skb_any(skb);
2169 return -EFAULT;
2170 }
2171
2172 u64_stats_update_begin(&ring->syncp);
2173 ring->stats.rx_pkts++;
2174 ring->stats.rx_bytes += skb->len;
2175 u64_stats_update_end(&ring->syncp);
2176
2177 ring->tqp_vector->rx_group.total_bytes += skb->len;
2178
2179 hns3_rx_checksum(ring, skb, desc);
2180 return 0;
2181}
2182
d43e5aca
YL		 2183int hns3_clean_rx_ring(
2184 struct hns3_enet_ring *ring, int budget,
2185 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2186{
2187#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2188 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2189 int recv_pkts, recv_bds, clean_count, err;
2190 int unused_count = hns3_desc_unused(ring);
2191 struct sk_buff *skb = NULL;
2192 int num, bnum = 0;
2193
2194 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2195 rmb(); /* Make sure num taken effect before the other data is touched */
2196
2197 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2198 num -= unused_count;
2199
2200 while (recv_pkts < budget && recv_bds < num) {
2201 /* Reuse or realloc buffers */
2202 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2203 hns3_nic_alloc_rx_buffers(ring,
2204 clean_count + unused_count);
2205 clean_count = 0;
2206 unused_count = hns3_desc_unused(ring);
2207 }
2208
2209 /* Poll one pkt */
2210 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2211 if (unlikely(!skb)) /* This fault cannot be repaired */
2212 goto out;
2213
2214 recv_bds += bnum;
2215 clean_count += bnum;
2216 if (unlikely(err)) { /* Do jump the err */
2217 recv_pkts++;
2218 continue;
2219 }
2220
2221 /* Do update ip stack process */
2222 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2223 rx_fn(ring, skb);
76ad4f0e
S		 2224
2225 recv_pkts++;
2226 }
2227
2228out:
2229 /* Make all data has been write before submit */
2230 if (clean_count + unused_count > 0)
2231 hns3_nic_alloc_rx_buffers(ring,
2232 clean_count + unused_count);
2233
2234 return recv_pkts;
2235}
2236
2237static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2238{
2239#define HNS3_RX_ULTRA_PACKET_RATE 40000
2240 enum hns3_flow_level_range new_flow_level;
2241 struct hns3_enet_tqp_vector *tqp_vector;
2242 int packets_per_secs;
2243 int bytes_per_usecs;
2244 u16 new_int_gl;
2245 int usecs;
2246
2247 if (!ring_group->int_gl)
2248 return false;
2249
2250 if (ring_group->total_packets == 0) {
2251 ring_group->int_gl = HNS3_INT_GL_50K;
2252 ring_group->flow_level = HNS3_FLOW_LOW;
2253 return true;
2254 }
2255
2256 /* Simple throttlerate management
2257 * 0-10MB/s lower (50000 ints/s)
2258 * 10-20MB/s middle (20000 ints/s)
2259 * 20-1249MB/s high (18000 ints/s)
2260 * > 40000pps ultra (8000 ints/s)
2261 */
2262 new_flow_level = ring_group->flow_level;
2263 new_int_gl = ring_group->int_gl;
2264 tqp_vector = ring_group->ring->tqp_vector;
2265 usecs = (ring_group->int_gl << 1);
2266 bytes_per_usecs = ring_group->total_bytes / usecs;
2267 /* 1000000 microseconds */
2268 packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2269
2270 switch (new_flow_level) {
2271 case HNS3_FLOW_LOW:
2272 if (bytes_per_usecs > 10)
2273 new_flow_level = HNS3_FLOW_MID;
2274 break;
2275 case HNS3_FLOW_MID:
2276 if (bytes_per_usecs > 20)
2277 new_flow_level = HNS3_FLOW_HIGH;
2278 else if (bytes_per_usecs <= 10)
2279 new_flow_level = HNS3_FLOW_LOW;
2280 break;
2281 case HNS3_FLOW_HIGH:
2282 case HNS3_FLOW_ULTRA:
2283 default:
2284 if (bytes_per_usecs <= 20)
2285 new_flow_level = HNS3_FLOW_MID;
2286 break;
2287 }
2288
2289 if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2290 (&tqp_vector->rx_group == ring_group))
2291 new_flow_level = HNS3_FLOW_ULTRA;
2292
2293 switch (new_flow_level) {
2294 case HNS3_FLOW_LOW:
2295 new_int_gl = HNS3_INT_GL_50K;
2296 break;
2297 case HNS3_FLOW_MID:
2298 new_int_gl = HNS3_INT_GL_20K;
2299 break;
2300 case HNS3_FLOW_HIGH:
2301 new_int_gl = HNS3_INT_GL_18K;
2302 break;
2303 case HNS3_FLOW_ULTRA:
2304 new_int_gl = HNS3_INT_GL_8K;
2305 break;
2306 default:
2307 break;
2308 }
2309
2310 ring_group->total_bytes = 0;
2311 ring_group->total_packets = 0;
2312 ring_group->flow_level = new_flow_level;
2313 if (new_int_gl != ring_group->int_gl) {
2314 ring_group->int_gl = new_int_gl;
2315 return true;
2316 }
2317 return false;
2318}
2319
2320static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2321{
2322 u16 rx_int_gl, tx_int_gl;
2323 bool rx, tx;
2324
2325 rx = hns3_get_new_int_gl(&tqp_vector->rx_group);
2326 tx = hns3_get_new_int_gl(&tqp_vector->tx_group);
2327 rx_int_gl = tqp_vector->rx_group.int_gl;
2328 tx_int_gl = tqp_vector->tx_group.int_gl;
2329 if (rx && tx) {
2330 if (rx_int_gl > tx_int_gl) {
2331 tqp_vector->tx_group.int_gl = rx_int_gl;
2332 tqp_vector->tx_group.flow_level =
2333 tqp_vector->rx_group.flow_level;
2334 hns3_set_vector_coalesc_gl(tqp_vector, rx_int_gl);
2335 } else {
2336 tqp_vector->rx_group.int_gl = tx_int_gl;
2337 tqp_vector->rx_group.flow_level =
2338 tqp_vector->tx_group.flow_level;
2339 hns3_set_vector_coalesc_gl(tqp_vector, tx_int_gl);
2340 }
2341 }
2342}
2343
2344static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2345{
2346 struct hns3_enet_ring *ring;
2347 int rx_pkt_total = 0;
2348
2349 struct hns3_enet_tqp_vector *tqp_vector =
2350 container_of(napi, struct hns3_enet_tqp_vector, napi);
2351 bool clean_complete = true;
2352 int rx_budget;
2353
2354 /* Since the actual Tx work is minimal, we can give the Tx a larger
2355 * budget and be more aggressive about cleaning up the Tx descriptors.
2356 */
2357 hns3_for_each_ring(ring, tqp_vector->tx_group) {
2358 if (!hns3_clean_tx_ring(ring, budget))
2359 clean_complete = false;
2360 }
2361
2362 /* make sure rx ring budget not smaller than 1 */
2363 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2364
2365 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2366 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2367 hns3_rx_skb);
76ad4f0e
S		 2368
2369 if (rx_cleaned >= rx_budget)
2370 clean_complete = false;
2371
2372 rx_pkt_total += rx_cleaned;
2373 }
2374
2375 tqp_vector->rx_group.total_packets += rx_pkt_total;
2376
2377 if (!clean_complete)
2378 return budget;
2379
2380 napi_complete(napi);
2381 hns3_update_new_int_gl(tqp_vector);
2382 hns3_mask_vector_irq(tqp_vector, 1);
2383
2384 return rx_pkt_total;
2385}
2386
2387static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2388 struct hnae3_ring_chain_node *head)
2389{
2390 struct pci_dev *pdev = tqp_vector->handle->pdev;
2391 struct hnae3_ring_chain_node *cur_chain = head;
2392 struct hnae3_ring_chain_node *chain;
2393 struct hns3_enet_ring *tx_ring;
2394 struct hns3_enet_ring *rx_ring;
2395
2396 tx_ring = tqp_vector->tx_group.ring;
2397 if (tx_ring) {
2398 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2399 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2400 HNAE3_RING_TYPE_TX);
2401
2402 cur_chain->next = NULL;
2403
2404 while (tx_ring->next) {
2405 tx_ring = tx_ring->next;
2406
2407 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2408 GFP_KERNEL);
2409 if (!chain)
2410 return -ENOMEM;
2411
2412 cur_chain->next = chain;
2413 chain->tqp_index = tx_ring->tqp->tqp_index;
2414 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2415 HNAE3_RING_TYPE_TX);
2416
2417 cur_chain = chain;
2418 }
2419 }
2420
2421 rx_ring = tqp_vector->rx_group.ring;
2422 if (!tx_ring && rx_ring) {
2423 cur_chain->next = NULL;
2424 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2425 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2426 HNAE3_RING_TYPE_RX);
2427
2428 rx_ring = rx_ring->next;
2429 }
2430
2431 while (rx_ring) {
2432 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2433 if (!chain)
2434 return -ENOMEM;
2435
2436 cur_chain->next = chain;
2437 chain->tqp_index = rx_ring->tqp->tqp_index;
2438 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2439 HNAE3_RING_TYPE_RX);
2440 cur_chain = chain;
2441
2442 rx_ring = rx_ring->next;
2443 }
2444
2445 return 0;
2446}
2447
2448static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2449 struct hnae3_ring_chain_node *head)
2450{
2451 struct pci_dev *pdev = tqp_vector->handle->pdev;
2452 struct hnae3_ring_chain_node *chain_tmp, *chain;
2453
2454 chain = head->next;
2455
2456 while (chain) {
2457 chain_tmp = chain->next;
2458 devm_kfree(&pdev->dev, chain);
2459 chain = chain_tmp;
2460 }
2461}
2462
2463static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2464 struct hns3_enet_ring *ring)
2465{
2466 ring->next = group->ring;
2467 group->ring = ring;
2468
2469 group->count++;
2470}
2471
2472static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2473{
2474 struct hnae3_ring_chain_node vector_ring_chain;
2475 struct hnae3_handle *h = priv->ae_handle;
2476 struct hns3_enet_tqp_vector *tqp_vector;
2477 struct hnae3_vector_info *vector;
2478 struct pci_dev *pdev = h->pdev;
2479 u16 tqp_num = h->kinfo.num_tqps;
2480 u16 vector_num;
2481 int ret = 0;
2482 u16 i;
2483
2484 /* RSS size, cpu online and vector_num should be the same */
2485 /* Should consider 2p/4p later */
2486 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2487 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2488 GFP_KERNEL);
2489 if (!vector)
2490 return -ENOMEM;
2491
2492 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2493
2494 priv->vector_num = vector_num;
2495 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2496 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2497 GFP_KERNEL);
2498 if (!priv->tqp_vector)
2499 return -ENOMEM;
2500
2501 for (i = 0; i < tqp_num; i++) {
2502 u16 vector_i = i % vector_num;
2503
2504 tqp_vector = &priv->tqp_vector[vector_i];
2505
2506 hns3_add_ring_to_group(&tqp_vector->tx_group,
2507 priv->ring_data[i].ring);
2508
2509 hns3_add_ring_to_group(&tqp_vector->rx_group,
2510 priv->ring_data[i + tqp_num].ring);
2511
2512 tqp_vector->idx = vector_i;
2513 tqp_vector->mask_addr = vector[vector_i].io_addr;
2514 tqp_vector->vector_irq = vector[vector_i].vector;
2515 tqp_vector->num_tqps++;
2516
2517 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2518 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2519 }
2520
2521 for (i = 0; i < vector_num; i++) {
2522 tqp_vector = &priv->tqp_vector[i];
2523
2524 tqp_vector->rx_group.total_bytes = 0;
2525 tqp_vector->rx_group.total_packets = 0;
2526 tqp_vector->tx_group.total_bytes = 0;
2527 tqp_vector->tx_group.total_packets = 0;
2528 hns3_vector_gl_rl_init(tqp_vector);
2529 tqp_vector->handle = h;
2530
2531 ret = hns3_get_vector_ring_chain(tqp_vector,
2532 &vector_ring_chain);
2533 if (ret)
2534 goto out;
2535
2536 ret = h->ae_algo->ops->map_ring_to_vector(h,
2537 tqp_vector->vector_irq, &vector_ring_chain);
2538 if (ret)
2539 goto out;
2540
2541 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2542
2543 netif_napi_add(priv->netdev, &tqp_vector->napi,
2544 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2545 }
2546
2547out:
2548 devm_kfree(&pdev->dev, vector);
2549 return ret;
2550}
2551
2552static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2553{
2554 struct hnae3_ring_chain_node vector_ring_chain;
2555 struct hnae3_handle *h = priv->ae_handle;
2556 struct hns3_enet_tqp_vector *tqp_vector;
2557 struct pci_dev *pdev = h->pdev;
2558 int i, ret;
2559
2560 for (i = 0; i < priv->vector_num; i++) {
2561 tqp_vector = &priv->tqp_vector[i];
2562
2563 ret = hns3_get_vector_ring_chain(tqp_vector,
2564 &vector_ring_chain);
2565 if (ret)
2566 return ret;
2567
2568 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2569 tqp_vector->vector_irq, &vector_ring_chain);
2570 if (ret)
2571 return ret;
2572
2573 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2574
2575 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2576 (void)irq_set_affinity_hint(
2577 priv->tqp_vector[i].vector_irq,
2578 NULL);
ae064e61 2579 free_irq(priv->tqp_vector[i].vector_irq,
2580 &priv->tqp_vector[i]);
76ad4f0e
S		 2581 }
2582
2583 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2584
2585 netif_napi_del(&priv->tqp_vector[i].napi);
2586 }
2587
2588 devm_kfree(&pdev->dev, priv->tqp_vector);
2589
2590 return 0;
2591}
2592
2593static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2594 int ring_type)
2595{
2596 struct hns3_nic_ring_data *ring_data = priv->ring_data;
2597 int queue_num = priv->ae_handle->kinfo.num_tqps;
2598 struct pci_dev *pdev = priv->ae_handle->pdev;
2599 struct hns3_enet_ring *ring;
2600
2601 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2602 if (!ring)
2603 return -ENOMEM;
2604
2605 if (ring_type == HNAE3_RING_TYPE_TX) {
2606 ring_data[q->tqp_index].ring = ring;
66b44730 2607 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 2608 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2609 } else {
2610 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 2611 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 2612 ring->io_base = q->io_base;
2613 }
2614
2615 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2616
76ad4f0e
S		 2617 ring->tqp = q;
2618 ring->desc = NULL;
2619 ring->desc_cb = NULL;
2620 ring->dev = priv->dev;
2621 ring->desc_dma_addr = 0;
2622 ring->buf_size = q->buf_size;
2623 ring->desc_num = q->desc_num;
2624 ring->next_to_use = 0;
2625 ring->next_to_clean = 0;
2626
2627 return 0;
2628}
2629
2630static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2631 struct hns3_nic_priv *priv)
2632{
2633 int ret;
2634
2635 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2636 if (ret)
2637 return ret;
2638
2639 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2640 if (ret)
2641 return ret;
2642
2643 return 0;
2644}
2645
2646static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2647{
2648 struct hnae3_handle *h = priv->ae_handle;
2649 struct pci_dev *pdev = h->pdev;
2650 int i, ret;
2651
2652 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2653 sizeof(*priv->ring_data) * 2,
2654 GFP_KERNEL);
2655 if (!priv->ring_data)
2656 return -ENOMEM;
2657
2658 for (i = 0; i < h->kinfo.num_tqps; i++) {
2659 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2660 if (ret)
2661 goto err;
2662 }
2663
2664 return 0;
2665err:
2666 devm_kfree(&pdev->dev, priv->ring_data);
2667 return ret;
2668}
2669
f1f779ce
PL		 2670static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2671{
2672 struct hnae3_handle *h = priv->ae_handle;
2673 int i;
2674
2675 for (i = 0; i < h->kinfo.num_tqps; i++) {
2676 devm_kfree(priv->dev, priv->ring_data[i].ring);
2677 devm_kfree(priv->dev,
2678 priv->ring_data[i + h->kinfo.num_tqps].ring);
2679 }
2680 devm_kfree(priv->dev, priv->ring_data);
2681}
2682
76ad4f0e
S		 2683static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2684{
2685 int ret;
2686
2687 if (ring->desc_num <= 0 || ring->buf_size <= 0)
2688 return -EINVAL;
2689
2690 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2691 GFP_KERNEL);
2692 if (!ring->desc_cb) {
2693 ret = -ENOMEM;
2694 goto out;
2695 }
2696
2697 ret = hns3_alloc_desc(ring);
2698 if (ret)
2699 goto out_with_desc_cb;
2700
2701 if (!HNAE3_IS_TX_RING(ring)) {
2702 ret = hns3_alloc_ring_buffers(ring);
2703 if (ret)
2704 goto out_with_desc;
2705 }
2706
2707 return 0;
2708
2709out_with_desc:
2710 hns3_free_desc(ring);
2711out_with_desc_cb:
2712 kfree(ring->desc_cb);
2713 ring->desc_cb = NULL;
2714out:
2715 return ret;
2716}
2717
2718static void hns3_fini_ring(struct hns3_enet_ring *ring)
2719{
2720 hns3_free_desc(ring);
2721 kfree(ring->desc_cb);
2722 ring->desc_cb = NULL;
2723 ring->next_to_clean = 0;
2724 ring->next_to_use = 0;
2725}
2726
1db9b1bf 2727static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 2728{
2729 int bd_size_type;
2730
2731 switch (buf_size) {
2732 case 512:
2733 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2734 break;
2735 case 1024:
2736 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2737 break;
2738 case 2048:
2739 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2740 break;
2741 case 4096:
2742 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2743 break;
2744 default:
2745 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2746 }
2747
2748 return bd_size_type;
2749}
2750
2751static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2752{
2753 dma_addr_t dma = ring->desc_dma_addr;
2754 struct hnae3_queue *q = ring->tqp;
2755
2756 if (!HNAE3_IS_TX_RING(ring)) {
2757 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2758 (u32)dma);
2759 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2760 (u32)((dma >> 31) >> 1));
2761
2762 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2763 hns3_buf_size2type(ring->buf_size));
2764 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2765 ring->desc_num / 8 - 1);
2766
2767 } else {
2768 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2769 (u32)dma);
2770 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2771 (u32)((dma >> 31) >> 1));
2772
2773 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2774 hns3_buf_size2type(ring->buf_size));
2775 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2776 ring->desc_num / 8 - 1);
2777 }
2778}
2779
5668abda 2780int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2781{
2782 struct hnae3_handle *h = priv->ae_handle;
2783 int ring_num = h->kinfo.num_tqps * 2;
2784 int i, j;
2785 int ret;
2786
2787 for (i = 0; i < ring_num; i++) {
2788 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2789 if (ret) {
2790 dev_err(priv->dev,
2791 "Alloc ring memory fail! ret=%d\n", ret);
2792 goto out_when_alloc_ring_memory;
2793 }
2794
2795 hns3_init_ring_hw(priv->ring_data[i].ring);
2796
2797 u64_stats_init(&priv->ring_data[i].ring->syncp);
2798 }
2799
2800 return 0;
2801
2802out_when_alloc_ring_memory:
2803 for (j = i - 1; j >= 0; j--)
ee83f776 2804 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 2805
2806 return -ENOMEM;
2807}
2808
5668abda 2809int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2810{
2811 struct hnae3_handle *h = priv->ae_handle;
2812 int i;
2813
2814 for (i = 0; i < h->kinfo.num_tqps; i++) {
2815 if (h->ae_algo->ops->reset_queue)
2816 h->ae_algo->ops->reset_queue(h, i);
2817
2818 hns3_fini_ring(priv->ring_data[i].ring);
74303129 2819 devm_kfree(priv->dev, priv->ring_data[i].ring);
76ad4f0e 2820 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
74303129
PL		 2821 devm_kfree(priv->dev,
2822 priv->ring_data[i + h->kinfo.num_tqps].ring);
76ad4f0e 2823 }
74303129 2824 devm_kfree(priv->dev, priv->ring_data);
76ad4f0e
S		 2825
2826 return 0;
2827}
2828
2829/* Set mac addr if it is configured. or leave it to the AE driver */
2830static void hns3_init_mac_addr(struct net_device *netdev)
2831{
2832 struct hns3_nic_priv *priv = netdev_priv(netdev);
2833 struct hnae3_handle *h = priv->ae_handle;
2834 u8 mac_addr_temp[ETH_ALEN];
2835
2836 if (h->ae_algo->ops->get_mac_addr) {
2837 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
2838 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
2839 }
2840
2841 /* Check if the MAC address is valid, if not get a random one */
2842 if (!is_valid_ether_addr(netdev->dev_addr)) {
2843 eth_hw_addr_random(netdev);
2844 dev_warn(priv->dev, "using random MAC address %pM\n",
2845 netdev->dev_addr);
76ad4f0e 2846 }
139e8792
L		 2847
2848 if (h->ae_algo->ops->set_mac_addr)
2849 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
2850
76ad4f0e
S		 2851}
2852
2853static void hns3_nic_set_priv_ops(struct net_device *netdev)
2854{
2855 struct hns3_nic_priv *priv = netdev_priv(netdev);
2856
2857 if ((netdev->features & NETIF_F_TSO) ||
2858 (netdev->features & NETIF_F_TSO6)) {
2859 priv->ops.fill_desc = hns3_fill_desc_tso;
2860 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
2861 } else {
2862 priv->ops.fill_desc = hns3_fill_desc;
2863 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
2864 }
2865}
2866
2867static int hns3_client_init(struct hnae3_handle *handle)
2868{
2869 struct pci_dev *pdev = handle->pdev;
2870 struct hns3_nic_priv *priv;
2871 struct net_device *netdev;
2872 int ret;
2873
2874 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
2875 handle->kinfo.num_tqps);
2876 if (!netdev)
2877 return -ENOMEM;
2878
2879 priv = netdev_priv(netdev);
2880 priv->dev = &pdev->dev;
2881 priv->netdev = netdev;
2882 priv->ae_handle = handle;
f8fa222c
L		 2883 priv->last_reset_time = jiffies;
2884 priv->reset_level = HNAE3_FUNC_RESET;
2885 priv->tx_timeout_count = 0;
76ad4f0e
S		 2886
2887 handle->kinfo.netdev = netdev;
2888 handle->priv = (void *)priv;
2889
2890 hns3_init_mac_addr(netdev);
2891
2892 hns3_set_default_feature(netdev);
2893
2894 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
2895 netdev->priv_flags |= IFF_UNICAST_FLT;
2896 netdev->netdev_ops = &hns3_nic_netdev_ops;
2897 SET_NETDEV_DEV(netdev, &pdev->dev);
2898 hns3_ethtool_set_ops(netdev);
2899 hns3_nic_set_priv_ops(netdev);
2900
2901 /* Carrier off reporting is important to ethtool even BEFORE open */
2902 netif_carrier_off(netdev);
2903
2904 ret = hns3_get_ring_config(priv);
2905 if (ret) {
2906 ret = -ENOMEM;
2907 goto out_get_ring_cfg;
2908 }
2909
2910 ret = hns3_nic_init_vector_data(priv);
2911 if (ret) {
2912 ret = -ENOMEM;
2913 goto out_init_vector_data;
2914 }
2915
2916 ret = hns3_init_all_ring(priv);
2917 if (ret) {
2918 ret = -ENOMEM;
2919 goto out_init_ring_data;
2920 }
2921
2922 ret = register_netdev(netdev);
2923 if (ret) {
2924 dev_err(priv->dev, "probe register netdev fail!\n");
2925 goto out_reg_netdev_fail;
2926 }
2927
986743db
YL		 2928 hns3_dcbnl_setup(handle);
2929
a8e8b7ff
S		 2930 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
2931 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2932
76ad4f0e
S		 2933 return ret;
2934
2935out_reg_netdev_fail:
2936out_init_ring_data:
2937 (void)hns3_nic_uninit_vector_data(priv);
2938 priv->ring_data = NULL;
2939out_init_vector_data:
2940out_get_ring_cfg:
2941 priv->ae_handle = NULL;
2942 free_netdev(netdev);
2943 return ret;
2944}
2945
2946static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
2947{
2948 struct net_device *netdev = handle->kinfo.netdev;
2949 struct hns3_nic_priv *priv = netdev_priv(netdev);
2950 int ret;
2951
2952 if (netdev->reg_state != NETREG_UNINITIALIZED)
2953 unregister_netdev(netdev);
2954
2955 ret = hns3_nic_uninit_vector_data(priv);
2956 if (ret)
2957 netdev_err(netdev, "uninit vector error\n");
2958
2959 ret = hns3_uninit_all_ring(priv);
2960 if (ret)
2961 netdev_err(netdev, "uninit ring error\n");
2962
2963 priv->ring_data = NULL;
2964
2965 free_netdev(netdev);
2966}
2967
2968static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
2969{
2970 struct net_device *netdev = handle->kinfo.netdev;
2971
2972 if (!netdev)
2973 return;
2974
2975 if (linkup) {
2976 netif_carrier_on(netdev);
2977 netif_tx_wake_all_queues(netdev);
2978 netdev_info(netdev, "link up\n");
2979 } else {
2980 netif_carrier_off(netdev);
2981 netif_tx_stop_all_queues(netdev);
2982 netdev_info(netdev, "link down\n");
2983 }
2984}
2985
9df8f79a
YL		 2986static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
2987{
2988 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
2989 struct net_device *ndev = kinfo->netdev;
075cfdd6 2990 bool if_running;
9df8f79a
YL		 2991 int ret;
2992 u8 i;
2993
2994 if (tc > HNAE3_MAX_TC)
2995 return -EINVAL;
2996
2997 if (!ndev)
2998 return -ENODEV;
2999
075cfdd6
CIK		 3000 if_running = netif_running(ndev);
3001
9df8f79a
YL		 3002 ret = netdev_set_num_tc(ndev, tc);
3003 if (ret)
3004 return ret;
3005
3006 if (if_running) {
3007 (void)hns3_nic_net_stop(ndev);
3008 msleep(100);
3009 }
3010
3011 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3012 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3013 if (ret)
3014 goto err_out;
3015
3016 if (tc <= 1) {
3017 netdev_reset_tc(ndev);
3018 goto out;
3019 }
3020
3021 for (i = 0; i < HNAE3_MAX_TC; i++) {
3022 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3023
3024 if (tc_info->enable)
3025 netdev_set_tc_queue(ndev,
3026 tc_info->tc,
3027 tc_info->tqp_count,
3028 tc_info->tqp_offset);
3029 }
3030
3031 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
3032 netdev_set_prio_tc_map(ndev, i,
3033 kinfo->prio_tc[i]);
3034 }
3035
3036out:
3037 ret = hns3_nic_set_real_num_queue(ndev);
3038
3039err_out:
3040 if (if_running)
3041 (void)hns3_nic_net_open(ndev);
3042
3043 return ret;
3044}
3045
bb6b94a8
L		 3046static void hns3_recover_hw_addr(struct net_device *ndev)
3047{
3048 struct netdev_hw_addr_list *list;
3049 struct netdev_hw_addr *ha, *tmp;
3050
3051 /* go through and sync uc_addr entries to the device */
3052 list = &ndev->uc;
3053 list_for_each_entry_safe(ha, tmp, &list->list, list)
3054 hns3_nic_uc_sync(ndev, ha->addr);
3055
3056 /* go through and sync mc_addr entries to the device */
3057 list = &ndev->mc;
3058 list_for_each_entry_safe(ha, tmp, &list->list, list)
3059 hns3_nic_mc_sync(ndev, ha->addr);
3060}
3061
3062static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb)
3063{
3064 dev_kfree_skb_any(skb);
3065}
3066
3067static void hns3_clear_all_ring(struct hnae3_handle *h)
3068{
3069 struct net_device *ndev = h->kinfo.netdev;
3070 struct hns3_nic_priv *priv = netdev_priv(ndev);
3071 u32 i;
3072
3073 for (i = 0; i < h->kinfo.num_tqps; i++) {
3074 struct netdev_queue *dev_queue;
3075 struct hns3_enet_ring *ring;
3076
3077 ring = priv->ring_data[i].ring;
3078 hns3_clean_tx_ring(ring, ring->desc_num);
3079 dev_queue = netdev_get_tx_queue(ndev,
3080 priv->ring_data[i].queue_index);
3081 netdev_tx_reset_queue(dev_queue);
3082
3083 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3084 hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data);
3085 }
3086}
3087
3088static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3089{
3090 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3091 struct net_device *ndev = kinfo->netdev;
3092
3093 if (!netif_running(ndev))
3094 return -EIO;
3095
3096 return hns3_nic_net_stop(ndev);
3097}
3098
3099static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3100{
3101 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3102 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
3103 int ret = 0;
3104
3105 if (netif_running(kinfo->netdev)) {
3106 ret = hns3_nic_net_up(kinfo->netdev);
3107 if (ret) {
3108 netdev_err(kinfo->netdev,
3109 "hns net up fail, ret=%d!\n", ret);
3110 return ret;
3111 }
3112
3113 priv->last_reset_time = jiffies;
3114 }
3115
3116 return ret;
3117}
3118
3119static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3120{
3121 struct net_device *netdev = handle->kinfo.netdev;
3122 struct hns3_nic_priv *priv = netdev_priv(netdev);
3123 int ret;
3124
3125 priv->reset_level = 1;
3126 hns3_init_mac_addr(netdev);
3127 hns3_nic_set_rx_mode(netdev);
3128 hns3_recover_hw_addr(netdev);
3129
3130 /* Carrier off reporting is important to ethtool even BEFORE open */
3131 netif_carrier_off(netdev);
3132
3133 ret = hns3_get_ring_config(priv);
3134 if (ret)
3135 return ret;
3136
3137 ret = hns3_nic_init_vector_data(priv);
3138 if (ret)
3139 return ret;
3140
3141 ret = hns3_init_all_ring(priv);
3142 if (ret) {
3143 hns3_nic_uninit_vector_data(priv);
3144 priv->ring_data = NULL;
3145 }
3146
3147 return ret;
3148}
3149
3150static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3151{
3152 struct net_device *netdev = handle->kinfo.netdev;
3153 struct hns3_nic_priv *priv = netdev_priv(netdev);
3154 int ret;
3155
3156 hns3_clear_all_ring(handle);
3157
3158 ret = hns3_nic_uninit_vector_data(priv);
3159 if (ret) {
3160 netdev_err(netdev, "uninit vector error\n");
3161 return ret;
3162 }
3163
3164 ret = hns3_uninit_all_ring(priv);
3165 if (ret)
3166 netdev_err(netdev, "uninit ring error\n");
3167
3168 priv->ring_data = NULL;
3169
3170 return ret;
3171}
3172
3173static int hns3_reset_notify(struct hnae3_handle *handle,
3174 enum hnae3_reset_notify_type type)
3175{
3176 int ret = 0;
3177
3178 switch (type) {
3179 case HNAE3_UP_CLIENT:
3180 ret = hns3_reset_notify_up_enet(handle);
3181 break;
3182 case HNAE3_DOWN_CLIENT:
3183 ret = hns3_reset_notify_down_enet(handle);
3184 break;
3185 case HNAE3_INIT_CLIENT:
3186 ret = hns3_reset_notify_init_enet(handle);
3187 break;
3188 case HNAE3_UNINIT_CLIENT:
3189 ret = hns3_reset_notify_uninit_enet(handle);
3190 break;
3191 default:
3192 break;
3193 }
3194
3195 return ret;
3196}
3197
f1f779ce
PL		 3198static u16 hns3_get_max_available_channels(struct net_device *netdev)
3199{
3200 struct hnae3_handle *h = hns3_get_handle(netdev);
3201 u16 free_tqps, max_rss_size, max_tqps;
3202
3203 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
3204 max_tqps = h->kinfo.num_tc * max_rss_size;
3205
3206 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
3207}
3208
3209static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
3210{
3211 struct hns3_nic_priv *priv = netdev_priv(netdev);
3212 struct hnae3_handle *h = hns3_get_handle(netdev);
3213 int ret;
3214
3215 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3216 if (ret)
3217 return ret;
3218
3219 ret = hns3_get_ring_config(priv);
3220 if (ret)
3221 return ret;
3222
3223 ret = hns3_nic_init_vector_data(priv);
3224 if (ret)
3225 goto err_uninit_vector;
3226
3227 ret = hns3_init_all_ring(priv);
3228 if (ret)
3229 goto err_put_ring;
3230
3231 return 0;
3232
3233err_put_ring:
3234 hns3_put_ring_config(priv);
3235err_uninit_vector:
3236 hns3_nic_uninit_vector_data(priv);
3237 return ret;
3238}
3239
3240static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3241{
3242 return (new_tqp_num / num_tc) * num_tc;
3243}
3244
3245int hns3_set_channels(struct net_device *netdev,
3246 struct ethtool_channels *ch)
3247{
3248 struct hns3_nic_priv *priv = netdev_priv(netdev);
3249 struct hnae3_handle *h = hns3_get_handle(netdev);
3250 struct hnae3_knic_private_info *kinfo = &h->kinfo;
3251 bool if_running = netif_running(netdev);
3252 u32 new_tqp_num = ch->combined_count;
3253 u16 org_tqp_num;
3254 int ret;
3255
3256 if (ch->rx_count || ch->tx_count)
3257 return -EINVAL;
3258
3259 if (new_tqp_num > hns3_get_max_available_channels(netdev) ||
3260 new_tqp_num < kinfo->num_tc) {
3261 dev_err(&netdev->dev,
3262 "Change tqps fail, the tqp range is from %d to %d",
3263 kinfo->num_tc,
3264 hns3_get_max_available_channels(netdev));
3265 return -EINVAL;
3266 }
3267
3268 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3269 if (kinfo->num_tqps == new_tqp_num)
3270 return 0;
3271
3272 if (if_running)
3273 dev_close(netdev);
3274
3275 hns3_clear_all_ring(h);
3276
3277 ret = hns3_nic_uninit_vector_data(priv);
3278 if (ret) {
3279 dev_err(&netdev->dev,
3280 "Unbind vector with tqp fail, nothing is changed");
3281 goto open_netdev;
3282 }
3283
3284 hns3_uninit_all_ring(priv);
3285
3286 org_tqp_num = h->kinfo.num_tqps;
3287 ret = hns3_modify_tqp_num(netdev, new_tqp_num);
3288 if (ret) {
3289 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
3290 if (ret) {
3291 /* If revert to old tqp failed, fatal error occurred */
3292 dev_err(&netdev->dev,
3293 "Revert to old tqp num fail, ret=%d", ret);
3294 return ret;
3295 }
3296 dev_info(&netdev->dev,
3297 "Change tqp num fail, Revert to old tqp num");
3298 }
3299
3300open_netdev:
3301 if (if_running)
3302 dev_open(netdev);
3303
3304 return ret;
3305}
3306
1db9b1bf 3307static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 3308 .init_instance = hns3_client_init,
3309 .uninit_instance = hns3_client_uninit,
3310 .link_status_change = hns3_link_status_change,
9df8f79a 3311 .setup_tc = hns3_client_setup_tc,
bb6b94a8 3312 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 3313};
3314
3315/* hns3_init_module - Driver registration routine
3316 * hns3_init_module is the first routine called when the driver is
3317 * loaded. All it does is register with the PCI subsystem.
3318 */
3319static int __init hns3_init_module(void)
3320{
3321 int ret;
3322
3323 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3324 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3325
3326 client.type = HNAE3_CLIENT_KNIC;
3327 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3328 hns3_driver_name);
3329
3330 client.ops = &client_ops;
3331
3332 ret = hnae3_register_client(&client);
3333 if (ret)
3334 return ret;
3335
3336 ret = pci_register_driver(&hns3_driver);
3337 if (ret)
3338 hnae3_unregister_client(&client);
3339
3340 return ret;
3341}
3342module_init(hns3_init_module);
3343
3344/* hns3_exit_module - Driver exit cleanup routine
3345 * hns3_exit_module is called just before the driver is removed
3346 * from memory.
3347 */
3348static void __exit hns3_exit_module(void)
3349{
3350 pci_unregister_driver(&hns3_driver);
3351 hnae3_unregister_client(&client);
3352}
3353module_exit(hns3_exit_module);
3354
3355MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3356MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3357MODULE_LICENSE("GPL");
3358MODULE_ALIAS("pci:hns-nic");
ubuntu-bionic-kernel mirror