]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'selftests-forwarding-Tweaks-and-a-new-test'
[mirror_ubuntu-focal-kernel.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
76ad4f0e
S		 1/*
2 * Copyright (c) 2016~2017 Hisilicon Limited.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/etherdevice.h>
12#include <linux/interrupt.h>
13#include <linux/if_vlan.h>
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/skbuff.h>
19#include <linux/sctp.h>
20#include <linux/vermagic.h>
21#include <net/gre.h>
30d240df 22#include <net/pkt_cls.h>
76ad4f0e
S		 23#include <net/vxlan.h>
24
25#include "hnae3.h"
26#include "hns3_enet.h"
27
1db9b1bf 28static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 29const char hns3_driver_version[] = VERMAGIC_STRING;
30static const char hns3_driver_string[] =
31 "Hisilicon Ethernet Network Driver for Hip08 Family";
32static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33static struct hnae3_client client;
34
35/* hns3_pci_tbl - PCI Device ID Table
36 *
37 * Last entry must be all 0s
38 *
39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40 * Class, Class Mask, private data (not used) }
41 */
42static const struct pci_device_id hns3_pci_tbl[] = {
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
424eb834
SM		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62static irqreturn_t hns3_irq_handle(int irq, void *dev)
63{
64 struct hns3_enet_tqp_vector *tqp_vector = dev;
65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
71static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
72{
73 struct hns3_enet_tqp_vector *tqp_vectors;
74 unsigned int i;
75
76 for (i = 0; i < priv->vector_num; i++) {
77 tqp_vectors = &priv->tqp_vector[i];
78
79 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
80 continue;
81
82 /* release the irq resource */
83 free_irq(tqp_vectors->vector_irq, tqp_vectors);
84 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
85 }
86}
87
88static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 int txrx_int_idx = 0;
92 int rx_int_idx = 0;
93 int tx_int_idx = 0;
94 unsigned int i;
95 int ret;
96
97 for (i = 0; i < priv->vector_num; i++) {
98 tqp_vectors = &priv->tqp_vector[i];
99
100 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
101 continue;
102
103 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
104 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
105 "%s-%s-%d", priv->netdev->name, "TxRx",
106 txrx_int_idx++);
107 txrx_int_idx++;
108 } else if (tqp_vectors->rx_group.ring) {
109 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
110 "%s-%s-%d", priv->netdev->name, "Rx",
111 rx_int_idx++);
112 } else if (tqp_vectors->tx_group.ring) {
113 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
114 "%s-%s-%d", priv->netdev->name, "Tx",
115 tx_int_idx++);
116 } else {
117 /* Skip this unused q_vector */
118 continue;
119 }
120
121 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
122
123 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
124 tqp_vectors->name,
125 tqp_vectors);
126 if (ret) {
127 netdev_err(priv->netdev, "request irq(%d) fail\n",
128 tqp_vectors->vector_irq);
129 return ret;
130 }
131
132 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
133 }
134
135 return 0;
136}
137
138static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
139 u32 mask_en)
140{
141 writel(mask_en, tqp_vector->mask_addr);
142}
143
144static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
145{
146 napi_enable(&tqp_vector->napi);
147
148 /* enable vector */
149 hns3_mask_vector_irq(tqp_vector, 1);
150}
151
152static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
153{
154 /* disable vector */
155 hns3_mask_vector_irq(tqp_vector, 0);
156
157 disable_irq(tqp_vector->vector_irq);
158 napi_disable(&tqp_vector->napi);
159}
160
434776a5
FL		 161void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
162 u32 rl_value)
76ad4f0e 163{
434776a5
FL		 164 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
165
76ad4f0e
S		 166 /* this defines the configuration for RL (Interrupt Rate Limiter).
167 * Rl defines rate of interrupts i.e. number of interrupts-per-second
168 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
169 */
434776a5 170
9bc727a9
YL		 171 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
172 !tqp_vector->rx_group.coal.gl_adapt_enable)
434776a5
FL		 173 /* According to the hardware, the range of rl_reg is
174 * 0-59 and the unit is 4.
175 */
176 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
177
178 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
179}
180
181void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
182 u32 gl_value)
183{
184 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
185
186 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
187}
188
189void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
190 u32 gl_value)
191{
192 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
193
194 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 195}
196
5fd4789a
FL		 197static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
198 struct hns3_nic_priv *priv)
76ad4f0e 199{
5fd4789a
FL		 200 struct hnae3_handle *h = priv->ae_handle;
201
76ad4f0e
S		 202 /* initialize the configuration for interrupt coalescing.
203 * 1. GL (Interrupt Gap Limiter)
204 * 2. RL (Interrupt Rate Limiter)
205 */
206
5fd4789a 207 /* Default: enable interrupt coalescing self-adaptive and GL */
9bc727a9
YL		 208 tqp_vector->tx_group.coal.gl_adapt_enable = 1;
209 tqp_vector->rx_group.coal.gl_adapt_enable = 1;
5fd4789a 210
9bc727a9
YL		 211 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
212 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
5fd4789a 213
5fd4789a
FL		 214 /* Default: disable RL */
215 h->kinfo.int_rl_setting = 0;
5fd4789a 216
9bc727a9
YL		 217 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
218 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 219}
220
dd38c726
YL		 221static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
222 struct hns3_nic_priv *priv)
223{
224 struct hnae3_handle *h = priv->ae_handle;
225
226 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 227 tqp_vector->tx_group.coal.int_gl);
dd38c726 228 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 229 tqp_vector->rx_group.coal.int_gl);
dd38c726
YL		 230 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
231}
232
9df8f79a
YL		 233static int hns3_nic_set_real_num_queue(struct net_device *netdev)
234{
9780cb97 235 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 236 struct hnae3_knic_private_info *kinfo = &h->kinfo;
237 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
238 int ret;
239
240 ret = netif_set_real_num_tx_queues(netdev, queue_size);
241 if (ret) {
242 netdev_err(netdev,
243 "netif_set_real_num_tx_queues fail, ret=%d!\n",
244 ret);
245 return ret;
246 }
247
248 ret = netif_set_real_num_rx_queues(netdev, queue_size);
249 if (ret) {
250 netdev_err(netdev,
251 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
252 return ret;
253 }
254
255 return 0;
256}
257
678335a1
PL		 258static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
259{
260 u16 free_tqps, max_rss_size, max_tqps;
261
262 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
263 max_tqps = h->kinfo.num_tc * max_rss_size;
264
265 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
266}
267
76ad4f0e
S		 268static int hns3_nic_net_up(struct net_device *netdev)
269{
270 struct hns3_nic_priv *priv = netdev_priv(netdev);
271 struct hnae3_handle *h = priv->ae_handle;
272 int i, j;
273 int ret;
274
275 /* get irq resource for all vectors */
276 ret = hns3_nic_init_irq(priv);
277 if (ret) {
278 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
279 return ret;
280 }
281
282 /* enable the vectors */
283 for (i = 0; i < priv->vector_num; i++)
284 hns3_vector_enable(&priv->tqp_vector[i]);
285
286 /* start the ae_dev */
287 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
288 if (ret)
289 goto out_start_err;
290
b875cc37
JS		 291 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
292
76ad4f0e
S		 293 return 0;
294
295out_start_err:
296 for (j = i - 1; j >= 0; j--)
297 hns3_vector_disable(&priv->tqp_vector[j]);
298
299 hns3_nic_uninit_irq(priv);
300
301 return ret;
302}
303
304static int hns3_nic_net_open(struct net_device *netdev)
305{
f8fa222c 306 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 307 int ret;
308
309 netif_carrier_off(netdev);
310
9df8f79a
YL		 311 ret = hns3_nic_set_real_num_queue(netdev);
312 if (ret)
76ad4f0e 313 return ret;
76ad4f0e
S		 314
315 ret = hns3_nic_net_up(netdev);
316 if (ret) {
317 netdev_err(netdev,
318 "hns net up fail, ret=%d!\n", ret);
319 return ret;
320 }
321
f8fa222c 322 priv->last_reset_time = jiffies;
76ad4f0e
S		 323 return 0;
324}
325
326static void hns3_nic_net_down(struct net_device *netdev)
327{
328 struct hns3_nic_priv *priv = netdev_priv(netdev);
329 const struct hnae3_ae_ops *ops;
330 int i;
331
b875cc37
JS		 332 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
333 return;
334
76ad4f0e
S		 335 /* stop ae_dev */
336 ops = priv->ae_handle->ae_algo->ops;
337 if (ops->stop)
338 ops->stop(priv->ae_handle);
339
340 /* disable vectors */
341 for (i = 0; i < priv->vector_num; i++)
342 hns3_vector_disable(&priv->tqp_vector[i]);
343
344 /* free irq resources */
345 hns3_nic_uninit_irq(priv);
346}
347
348static int hns3_nic_net_stop(struct net_device *netdev)
349{
350 netif_tx_stop_all_queues(netdev);
351 netif_carrier_off(netdev);
352
353 hns3_nic_net_down(netdev);
354
355 return 0;
356}
357
76ad4f0e
S		 358static int hns3_nic_uc_sync(struct net_device *netdev,
359 const unsigned char *addr)
360{
9780cb97 361 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 362
363 if (h->ae_algo->ops->add_uc_addr)
364 return h->ae_algo->ops->add_uc_addr(h, addr);
365
366 return 0;
367}
368
369static int hns3_nic_uc_unsync(struct net_device *netdev,
370 const unsigned char *addr)
371{
9780cb97 372 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 373
374 if (h->ae_algo->ops->rm_uc_addr)
375 return h->ae_algo->ops->rm_uc_addr(h, addr);
376
377 return 0;
378}
379
380static int hns3_nic_mc_sync(struct net_device *netdev,
381 const unsigned char *addr)
382{
9780cb97 383 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 384
720a8478 385 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 386 return h->ae_algo->ops->add_mc_addr(h, addr);
387
388 return 0;
389}
390
391static int hns3_nic_mc_unsync(struct net_device *netdev,
392 const unsigned char *addr)
393{
9780cb97 394 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 395
720a8478 396 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 397 return h->ae_algo->ops->rm_mc_addr(h, addr);
398
399 return 0;
400}
401
1db9b1bf 402static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 403{
9780cb97 404 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 405
406 if (h->ae_algo->ops->set_promisc_mode) {
407 if (netdev->flags & IFF_PROMISC)
408 h->ae_algo->ops->set_promisc_mode(h, 1);
409 else
410 h->ae_algo->ops->set_promisc_mode(h, 0);
411 }
412 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
413 netdev_err(netdev, "sync uc address fail\n");
414 if (netdev->flags & IFF_MULTICAST)
415 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
416 netdev_err(netdev, "sync mc address fail\n");
417}
418
419static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
420 u16 *mss, u32 *type_cs_vlan_tso)
421{
422 u32 l4_offset, hdr_len;
423 union l3_hdr_info l3;
424 union l4_hdr_info l4;
425 u32 l4_paylen;
426 int ret;
427
428 if (!skb_is_gso(skb))
429 return 0;
430
431 ret = skb_cow_head(skb, 0);
432 if (ret)
433 return ret;
434
435 l3.hdr = skb_network_header(skb);
436 l4.hdr = skb_transport_header(skb);
437
438 /* Software should clear the IPv4's checksum field when tso is
439 * needed.
440 */
441 if (l3.v4->version == 4)
442 l3.v4->check = 0;
443
444 /* tunnel packet.*/
445 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
446 SKB_GSO_GRE_CSUM |
447 SKB_GSO_UDP_TUNNEL |
448 SKB_GSO_UDP_TUNNEL_CSUM)) {
449 if ((!(skb_shinfo(skb)->gso_type &
450 SKB_GSO_PARTIAL)) &&
451 (skb_shinfo(skb)->gso_type &
452 SKB_GSO_UDP_TUNNEL_CSUM)) {
453 /* Software should clear the udp's checksum
454 * field when tso is needed.
455 */
456 l4.udp->check = 0;
457 }
458 /* reset l3&l4 pointers from outer to inner headers */
459 l3.hdr = skb_inner_network_header(skb);
460 l4.hdr = skb_inner_transport_header(skb);
461
462 /* Software should clear the IPv4's checksum field when
463 * tso is needed.
464 */
465 if (l3.v4->version == 4)
466 l3.v4->check = 0;
467 }
468
469 /* normal or tunnel packet*/
470 l4_offset = l4.hdr - skb->data;
471 hdr_len = (l4.tcp->doff * 4) + l4_offset;
472
473 /* remove payload length from inner pseudo checksum when tso*/
474 l4_paylen = skb->len - l4_offset;
475 csum_replace_by_diff(&l4.tcp->check,
476 (__force __wsum)htonl(l4_paylen));
477
478 /* find the txbd field values */
479 *paylen = skb->len - hdr_len;
480 hnae_set_bit(*type_cs_vlan_tso,
481 HNS3_TXD_TSO_B, 1);
482
483 /* get MSS for TSO */
484 *mss = skb_shinfo(skb)->gso_size;
485
486 return 0;
487}
488
1898d4e4
S		 489static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
490 u8 *il4_proto)
76ad4f0e
S		 491{
492 union {
493 struct iphdr *v4;
494 struct ipv6hdr *v6;
495 unsigned char *hdr;
496 } l3;
497 unsigned char *l4_hdr;
498 unsigned char *exthdr;
499 u8 l4_proto_tmp;
500 __be16 frag_off;
501
502 /* find outer header point */
503 l3.hdr = skb_network_header(skb);
504 l4_hdr = skb_inner_transport_header(skb);
505
506 if (skb->protocol == htons(ETH_P_IPV6)) {
507 exthdr = l3.hdr + sizeof(*l3.v6);
508 l4_proto_tmp = l3.v6->nexthdr;
509 if (l4_hdr != exthdr)
510 ipv6_skip_exthdr(skb, exthdr - skb->data,
511 &l4_proto_tmp, &frag_off);
512 } else if (skb->protocol == htons(ETH_P_IP)) {
513 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 514 } else {
515 return -EINVAL;
76ad4f0e
S		 516 }
517
518 *ol4_proto = l4_proto_tmp;
519
520 /* tunnel packet */
521 if (!skb->encapsulation) {
522 *il4_proto = 0;
1898d4e4 523 return 0;
76ad4f0e
S		 524 }
525
526 /* find inner header point */
527 l3.hdr = skb_inner_network_header(skb);
528 l4_hdr = skb_inner_transport_header(skb);
529
530 if (l3.v6->version == 6) {
531 exthdr = l3.hdr + sizeof(*l3.v6);
532 l4_proto_tmp = l3.v6->nexthdr;
533 if (l4_hdr != exthdr)
534 ipv6_skip_exthdr(skb, exthdr - skb->data,
535 &l4_proto_tmp, &frag_off);
536 } else if (l3.v4->version == 4) {
537 l4_proto_tmp = l3.v4->protocol;
538 }
539
540 *il4_proto = l4_proto_tmp;
1898d4e4
S		 541
542 return 0;
76ad4f0e
S		 543}
544
545static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
546 u8 il4_proto, u32 *type_cs_vlan_tso,
547 u32 *ol_type_vlan_len_msec)
548{
549 union {
550 struct iphdr *v4;
551 struct ipv6hdr *v6;
552 unsigned char *hdr;
553 } l3;
554 union {
555 struct tcphdr *tcp;
556 struct udphdr *udp;
557 struct gre_base_hdr *gre;
558 unsigned char *hdr;
559 } l4;
560 unsigned char *l2_hdr;
561 u8 l4_proto = ol4_proto;
562 u32 ol2_len;
563 u32 ol3_len;
564 u32 ol4_len;
565 u32 l2_len;
566 u32 l3_len;
567
568 l3.hdr = skb_network_header(skb);
569 l4.hdr = skb_transport_header(skb);
570
571 /* compute L2 header size for normal packet, defined in 2 Bytes */
572 l2_len = l3.hdr - skb->data;
573 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
574 HNS3_TXD_L2LEN_S, l2_len >> 1);
575
576 /* tunnel packet*/
577 if (skb->encapsulation) {
578 /* compute OL2 header size, defined in 2 Bytes */
579 ol2_len = l2_len;
580 hnae_set_field(*ol_type_vlan_len_msec,
581 HNS3_TXD_L2LEN_M,
582 HNS3_TXD_L2LEN_S, ol2_len >> 1);
583
584 /* compute OL3 header size, defined in 4 Bytes */
585 ol3_len = l4.hdr - l3.hdr;
586 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
587 HNS3_TXD_L3LEN_S, ol3_len >> 2);
588
589 /* MAC in UDP, MAC in GRE (0x6558)*/
590 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
591 /* switch MAC header ptr from outer to inner header.*/
592 l2_hdr = skb_inner_mac_header(skb);
593
594 /* compute OL4 header size, defined in 4 Bytes. */
595 ol4_len = l2_hdr - l4.hdr;
596 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
597 HNS3_TXD_L4LEN_S, ol4_len >> 2);
598
599 /* switch IP header ptr from outer to inner header */
600 l3.hdr = skb_inner_network_header(skb);
601
602 /* compute inner l2 header size, defined in 2 Bytes. */
603 l2_len = l3.hdr - l2_hdr;
604 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
605 HNS3_TXD_L2LEN_S, l2_len >> 1);
606 } else {
607 /* skb packet types not supported by hardware,
608 * txbd len fild doesn't be filled.
609 */
610 return;
611 }
612
613 /* switch L4 header pointer from outer to inner */
614 l4.hdr = skb_inner_transport_header(skb);
615
616 l4_proto = il4_proto;
617 }
618
619 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
620 l3_len = l4.hdr - l3.hdr;
621 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
622 HNS3_TXD_L3LEN_S, l3_len >> 2);
623
624 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
625 switch (l4_proto) {
626 case IPPROTO_TCP:
627 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
628 HNS3_TXD_L4LEN_S, l4.tcp->doff);
629 break;
630 case IPPROTO_SCTP:
631 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
632 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
633 break;
634 case IPPROTO_UDP:
635 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
636 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
637 break;
638 default:
639 /* skb packet types not supported by hardware,
640 * txbd len fild doesn't be filled.
641 */
642 return;
643 }
644}
645
646static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
647 u8 il4_proto, u32 *type_cs_vlan_tso,
648 u32 *ol_type_vlan_len_msec)
649{
650 union {
651 struct iphdr *v4;
652 struct ipv6hdr *v6;
653 unsigned char *hdr;
654 } l3;
655 u32 l4_proto = ol4_proto;
656
657 l3.hdr = skb_network_header(skb);
658
659 /* define OL3 type and tunnel type(OL4).*/
660 if (skb->encapsulation) {
661 /* define outer network header type.*/
662 if (skb->protocol == htons(ETH_P_IP)) {
663 if (skb_is_gso(skb))
664 hnae_set_field(*ol_type_vlan_len_msec,
665 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
666 HNS3_OL3T_IPV4_CSUM);
667 else
668 hnae_set_field(*ol_type_vlan_len_msec,
669 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
670 HNS3_OL3T_IPV4_NO_CSUM);
671
672 } else if (skb->protocol == htons(ETH_P_IPV6)) {
673 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
674 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
675 }
676
677 /* define tunnel type(OL4).*/
678 switch (l4_proto) {
679 case IPPROTO_UDP:
680 hnae_set_field(*ol_type_vlan_len_msec,
681 HNS3_TXD_TUNTYPE_M,
682 HNS3_TXD_TUNTYPE_S,
683 HNS3_TUN_MAC_IN_UDP);
684 break;
685 case IPPROTO_GRE:
686 hnae_set_field(*ol_type_vlan_len_msec,
687 HNS3_TXD_TUNTYPE_M,
688 HNS3_TXD_TUNTYPE_S,
689 HNS3_TUN_NVGRE);
690 break;
691 default:
692 /* drop the skb tunnel packet if hardware don't support,
693 * because hardware can't calculate csum when TSO.
694 */
695 if (skb_is_gso(skb))
696 return -EDOM;
697
698 /* the stack computes the IP header already,
699 * driver calculate l4 checksum when not TSO.
700 */
701 skb_checksum_help(skb);
702 return 0;
703 }
704
705 l3.hdr = skb_inner_network_header(skb);
706 l4_proto = il4_proto;
707 }
708
709 if (l3.v4->version == 4) {
710 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
711 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
712
713 /* the stack computes the IP header already, the only time we
714 * need the hardware to recompute it is in the case of TSO.
715 */
716 if (skb_is_gso(skb))
717 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
718
719 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
720 } else if (l3.v6->version == 6) {
721 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
722 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
723 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
724 }
725
726 switch (l4_proto) {
727 case IPPROTO_TCP:
728 hnae_set_field(*type_cs_vlan_tso,
729 HNS3_TXD_L4T_M,
730 HNS3_TXD_L4T_S,
731 HNS3_L4T_TCP);
732 break;
733 case IPPROTO_UDP:
734 hnae_set_field(*type_cs_vlan_tso,
735 HNS3_TXD_L4T_M,
736 HNS3_TXD_L4T_S,
737 HNS3_L4T_UDP);
738 break;
739 case IPPROTO_SCTP:
740 hnae_set_field(*type_cs_vlan_tso,
741 HNS3_TXD_L4T_M,
742 HNS3_TXD_L4T_S,
743 HNS3_L4T_SCTP);
744 break;
745 default:
746 /* drop the skb tunnel packet if hardware don't support,
747 * because hardware can't calculate csum when TSO.
748 */
749 if (skb_is_gso(skb))
750 return -EDOM;
751
752 /* the stack computes the IP header already,
753 * driver calculate l4 checksum when not TSO.
754 */
755 skb_checksum_help(skb);
756 return 0;
757 }
758
759 return 0;
760}
761
762static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
763{
764 /* Config bd buffer end */
765 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
766 HNS3_TXD_BDTYPE_M, 0);
767 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
768 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
7036d26f 769 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 770}
771
9699cffe
PL		 772static int hns3_fill_desc_vtags(struct sk_buff *skb,
773 struct hns3_enet_ring *tx_ring,
774 u32 *inner_vlan_flag,
775 u32 *out_vlan_flag,
776 u16 *inner_vtag,
777 u16 *out_vtag)
778{
779#define HNS3_TX_VLAN_PRIO_SHIFT 13
780
781 if (skb->protocol == htons(ETH_P_8021Q) &&
782 !(tx_ring->tqp->handle->kinfo.netdev->features &
783 NETIF_F_HW_VLAN_CTAG_TX)) {
784 /* When HW VLAN acceleration is turned off, and the stack
785 * sets the protocol to 802.1q, the driver just need to
786 * set the protocol to the encapsulated ethertype.
787 */
788 skb->protocol = vlan_get_protocol(skb);
789 return 0;
790 }
791
792 if (skb_vlan_tag_present(skb)) {
793 u16 vlan_tag;
794
795 vlan_tag = skb_vlan_tag_get(skb);
796 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
797
798 /* Based on hw strategy, use out_vtag in two layer tag case,
799 * and use inner_vtag in one tag case.
800 */
801 if (skb->protocol == htons(ETH_P_8021Q)) {
802 hnae_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
803 *out_vtag = vlan_tag;
804 } else {
805 hnae_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
806 *inner_vtag = vlan_tag;
807 }
808 } else if (skb->protocol == htons(ETH_P_8021Q)) {
809 struct vlan_ethhdr *vhdr;
810 int rc;
811
812 rc = skb_cow_head(skb, 0);
813 if (rc < 0)
814 return rc;
815 vhdr = (struct vlan_ethhdr *)skb->data;
816 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
817 << HNS3_TX_VLAN_PRIO_SHIFT);
818 }
819
820 skb->protocol = vlan_get_protocol(skb);
821 return 0;
822}
823
76ad4f0e
S		 824static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
825 int size, dma_addr_t dma, int frag_end,
826 enum hns_desc_type type)
827{
828 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
829 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
830 u32 ol_type_vlan_len_msec = 0;
831 u16 bdtp_fe_sc_vld_ra_ri = 0;
832 u32 type_cs_vlan_tso = 0;
833 struct sk_buff *skb;
9699cffe
PL		 834 u16 inner_vtag = 0;
835 u16 out_vtag = 0;
76ad4f0e
S		 836 u32 paylen = 0;
837 u16 mss = 0;
838 __be16 protocol;
839 u8 ol4_proto;
840 u8 il4_proto;
841 int ret;
842
843 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
844 desc_cb->priv = priv;
845 desc_cb->length = size;
846 desc_cb->dma = dma;
847 desc_cb->type = type;
848
849 /* now, fill the descriptor */
850 desc->addr = cpu_to_le64(dma);
851 desc->tx.send_size = cpu_to_le16((u16)size);
852 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
853 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
854
855 if (type == DESC_TYPE_SKB) {
856 skb = (struct sk_buff *)priv;
a90bb9a5 857 paylen = skb->len;
76ad4f0e 858
9699cffe
PL		 859 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
860 &ol_type_vlan_len_msec,
861 &inner_vtag, &out_vtag);
862 if (unlikely(ret))
863 return ret;
864
76ad4f0e
S		 865 if (skb->ip_summed == CHECKSUM_PARTIAL) {
866 skb_reset_mac_len(skb);
867 protocol = skb->protocol;
868
1898d4e4
S		 869 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
870 if (ret)
871 return ret;
76ad4f0e
S		 872 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
873 &type_cs_vlan_tso,
874 &ol_type_vlan_len_msec);
875 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
876 &type_cs_vlan_tso,
877 &ol_type_vlan_len_msec);
878 if (ret)
879 return ret;
880
881 ret = hns3_set_tso(skb, &paylen, &mss,
882 &type_cs_vlan_tso);
883 if (ret)
884 return ret;
885 }
886
887 /* Set txbd */
888 desc->tx.ol_type_vlan_len_msec =
889 cpu_to_le32(ol_type_vlan_len_msec);
890 desc->tx.type_cs_vlan_tso_len =
891 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 892 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 893 desc->tx.mss = cpu_to_le16(mss);
9699cffe
PL		 894 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
895 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
76ad4f0e
S		 896 }
897
898 /* move ring pointer to next.*/
899 ring_ptr_move_fw(ring, next_to_use);
900
901 return 0;
902}
903
904static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
905 int size, dma_addr_t dma, int frag_end,
906 enum hns_desc_type type)
907{
908 unsigned int frag_buf_num;
909 unsigned int k;
910 int sizeoflast;
911 int ret;
912
913 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
914 sizeoflast = size % HNS3_MAX_BD_SIZE;
915 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
916
917 /* When the frag size is bigger than hardware, split this frag */
918 for (k = 0; k < frag_buf_num; k++) {
919 ret = hns3_fill_desc(ring, priv,
920 (k == frag_buf_num - 1) ?
921 sizeoflast : HNS3_MAX_BD_SIZE,
922 dma + HNS3_MAX_BD_SIZE * k,
923 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
924 (type == DESC_TYPE_SKB && !k) ?
925 DESC_TYPE_SKB : DESC_TYPE_PAGE);
926 if (ret)
927 return ret;
928 }
929
930 return 0;
931}
932
933static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
934 struct hns3_enet_ring *ring)
935{
936 struct sk_buff *skb = *out_skb;
937 struct skb_frag_struct *frag;
938 int bdnum_for_frag;
939 int frag_num;
940 int buf_num;
941 int size;
942 int i;
943
944 size = skb_headlen(skb);
945 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
946
947 frag_num = skb_shinfo(skb)->nr_frags;
948 for (i = 0; i < frag_num; i++) {
949 frag = &skb_shinfo(skb)->frags[i];
950 size = skb_frag_size(frag);
951 bdnum_for_frag =
952 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
953 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
954 return -ENOMEM;
955
956 buf_num += bdnum_for_frag;
957 }
958
959 if (buf_num > ring_space(ring))
960 return -EBUSY;
961
962 *bnum = buf_num;
963 return 0;
964}
965
966static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
967 struct hns3_enet_ring *ring)
968{
969 struct sk_buff *skb = *out_skb;
970 int buf_num;
971
972 /* No. of segments (plus a header) */
973 buf_num = skb_shinfo(skb)->nr_frags + 1;
974
975 if (buf_num > ring_space(ring))
976 return -EBUSY;
977
978 *bnum = buf_num;
979
980 return 0;
981}
982
983static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
984{
985 struct device *dev = ring_to_dev(ring);
986 unsigned int i;
987
988 for (i = 0; i < ring->desc_num; i++) {
989 /* check if this is where we started */
990 if (ring->next_to_use == next_to_use_orig)
991 break;
992
993 /* unmap the descriptor dma address */
994 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
995 dma_unmap_single(dev,
996 ring->desc_cb[ring->next_to_use].dma,
997 ring->desc_cb[ring->next_to_use].length,
998 DMA_TO_DEVICE);
999 else
1000 dma_unmap_page(dev,
1001 ring->desc_cb[ring->next_to_use].dma,
1002 ring->desc_cb[ring->next_to_use].length,
1003 DMA_TO_DEVICE);
1004
1005 /* rollback one */
1006 ring_ptr_move_bw(ring, next_to_use);
1007 }
1008}
1009
d43e5aca 1010netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 1011{
1012 struct hns3_nic_priv *priv = netdev_priv(netdev);
1013 struct hns3_nic_ring_data *ring_data =
1014 &tx_ring_data(priv, skb->queue_mapping);
1015 struct hns3_enet_ring *ring = ring_data->ring;
1016 struct device *dev = priv->dev;
1017 struct netdev_queue *dev_queue;
1018 struct skb_frag_struct *frag;
1019 int next_to_use_head;
1020 int next_to_use_frag;
1021 dma_addr_t dma;
1022 int buf_num;
1023 int seg_num;
1024 int size;
1025 int ret;
1026 int i;
1027
1028 /* Prefetch the data used later */
1029 prefetch(skb->data);
1030
1031 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1032 case -EBUSY:
1033 u64_stats_update_begin(&ring->syncp);
1034 ring->stats.tx_busy++;
1035 u64_stats_update_end(&ring->syncp);
1036
1037 goto out_net_tx_busy;
1038 case -ENOMEM:
1039 u64_stats_update_begin(&ring->syncp);
1040 ring->stats.sw_err_cnt++;
1041 u64_stats_update_end(&ring->syncp);
1042 netdev_err(netdev, "no memory to xmit!\n");
1043
1044 goto out_err_tx_ok;
1045 default:
1046 break;
1047 }
1048
1049 /* No. of segments (plus a header) */
1050 seg_num = skb_shinfo(skb)->nr_frags + 1;
1051 /* Fill the first part */
1052 size = skb_headlen(skb);
1053
1054 next_to_use_head = ring->next_to_use;
1055
1056 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1057 if (dma_mapping_error(dev, dma)) {
1058 netdev_err(netdev, "TX head DMA map failed\n");
1059 ring->stats.sw_err_cnt++;
1060 goto out_err_tx_ok;
1061 }
1062
1063 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
1064 DESC_TYPE_SKB);
1065 if (ret)
1066 goto head_dma_map_err;
1067
1068 next_to_use_frag = ring->next_to_use;
1069 /* Fill the fragments */
1070 for (i = 1; i < seg_num; i++) {
1071 frag = &skb_shinfo(skb)->frags[i - 1];
1072 size = skb_frag_size(frag);
1073 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1074 if (dma_mapping_error(dev, dma)) {
1075 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
1076 ring->stats.sw_err_cnt++;
1077 goto frag_dma_map_err;
1078 }
1079 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
1080 seg_num - 1 == i ? 1 : 0,
1081 DESC_TYPE_PAGE);
1082
1083 if (ret)
1084 goto frag_dma_map_err;
1085 }
1086
1087 /* Complete translate all packets */
1088 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1089 netdev_tx_sent_queue(dev_queue, skb->len);
1090
1091 wmb(); /* Commit all data before submit */
1092
1093 hnae_queue_xmit(ring->tqp, buf_num);
1094
1095 return NETDEV_TX_OK;
1096
1097frag_dma_map_err:
1098 hns_nic_dma_unmap(ring, next_to_use_frag);
1099
1100head_dma_map_err:
1101 hns_nic_dma_unmap(ring, next_to_use_head);
1102
1103out_err_tx_ok:
1104 dev_kfree_skb_any(skb);
1105 return NETDEV_TX_OK;
1106
1107out_net_tx_busy:
1108 netif_stop_subqueue(netdev, ring_data->queue_index);
1109 smp_mb(); /* Commit all data before submit */
1110
1111 return NETDEV_TX_BUSY;
1112}
1113
1114static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1115{
9780cb97 1116 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1117 struct sockaddr *mac_addr = p;
1118 int ret;
1119
1120 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1121 return -EADDRNOTAVAIL;
1122
1123 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
1124 if (ret) {
1125 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1126 return ret;
1127 }
1128
1129 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1130
1131 return 0;
1132}
1133
1134static int hns3_nic_set_features(struct net_device *netdev,
1135 netdev_features_t features)
1136{
181d454b 1137 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 1138 struct hns3_nic_priv *priv = netdev_priv(netdev);
052ece6d 1139 struct hnae3_handle *h = priv->ae_handle;
052ece6d 1140 int ret;
76ad4f0e 1141
181d454b
JS		 1142 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1143 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1144 priv->ops.fill_desc = hns3_fill_desc_tso;
1145 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1146 } else {
1147 priv->ops.fill_desc = hns3_fill_desc;
1148 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1149 }
76ad4f0e
S		 1150 }
1151
bd368416
JS		 1152 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1153 h->ae_algo->ops->enable_vlan_filter) {
181d454b
JS		 1154 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1155 h->ae_algo->ops->enable_vlan_filter(h, true);
1156 else
1157 h->ae_algo->ops->enable_vlan_filter(h, false);
1158 }
391b5e93 1159
bd368416
JS		 1160 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1161 h->ae_algo->ops->enable_hw_strip_rxvtag) {
052ece6d
PL		 1162 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1163 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1164 else
1165 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1166
1167 if (ret)
1168 return ret;
1169 }
1170
76ad4f0e
S		 1171 netdev->features = features;
1172 return 0;
1173}
1174
6c88d9d7
PL		 1175static void hns3_nic_get_stats64(struct net_device *netdev,
1176 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1177{
1178 struct hns3_nic_priv *priv = netdev_priv(netdev);
1179 int queue_num = priv->ae_handle->kinfo.num_tqps;
c5f65480 1180 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1181 struct hns3_enet_ring *ring;
1182 unsigned int start;
1183 unsigned int idx;
1184 u64 tx_bytes = 0;
1185 u64 rx_bytes = 0;
1186 u64 tx_pkts = 0;
1187 u64 rx_pkts = 0;
d2a5dca8
JS		 1188 u64 tx_drop = 0;
1189 u64 rx_drop = 0;
76ad4f0e 1190
b875cc37
JS		 1191 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1192 return;
1193
c5f65480
JS		 1194 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1195
76ad4f0e
S		 1196 for (idx = 0; idx < queue_num; idx++) {
1197 /* fetch the tx stats */
1198 ring = priv->ring_data[idx].ring;
1199 do {
d36d36ce 1200 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1201 tx_bytes += ring->stats.tx_bytes;
1202 tx_pkts += ring->stats.tx_pkts;
d2a5dca8
JS		 1203 tx_drop += ring->stats.tx_busy;
1204 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1205 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1206
1207 /* fetch the rx stats */
1208 ring = priv->ring_data[idx + queue_num].ring;
1209 do {
d36d36ce 1210 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1211 rx_bytes += ring->stats.rx_bytes;
1212 rx_pkts += ring->stats.rx_pkts;
d2a5dca8
JS		 1213 rx_drop += ring->stats.non_vld_descs;
1214 rx_drop += ring->stats.err_pkt_len;
1215 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1216 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1217 }
1218
1219 stats->tx_bytes = tx_bytes;
1220 stats->tx_packets = tx_pkts;
1221 stats->rx_bytes = rx_bytes;
1222 stats->rx_packets = rx_pkts;
1223
1224 stats->rx_errors = netdev->stats.rx_errors;
1225 stats->multicast = netdev->stats.multicast;
1226 stats->rx_length_errors = netdev->stats.rx_length_errors;
1227 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1228 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1229
1230 stats->tx_errors = netdev->stats.tx_errors;
d2a5dca8
JS		 1231 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1232 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1233 stats->collisions = netdev->stats.collisions;
1234 stats->rx_over_errors = netdev->stats.rx_over_errors;
1235 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1236 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1237 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1238 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1239 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1240 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1241 stats->tx_window_errors = netdev->stats.tx_window_errors;
1242 stats->rx_compressed = netdev->stats.rx_compressed;
1243 stats->tx_compressed = netdev->stats.tx_compressed;
1244}
1245
1246static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1247 enum hns3_udp_tnl_type type)
1248{
1249 struct hns3_nic_priv *priv = netdev_priv(netdev);
1250 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1251 struct hnae3_handle *h = priv->ae_handle;
1252
1253 if (udp_tnl->used && udp_tnl->dst_port == port) {
1254 udp_tnl->used++;
1255 return;
1256 }
1257
1258 if (udp_tnl->used) {
1259 netdev_warn(netdev,
1260 "UDP tunnel [%d], port [%d] offload\n", type, port);
1261 return;
1262 }
1263
1264 udp_tnl->dst_port = port;
1265 udp_tnl->used = 1;
1266 /* TBD send command to hardware to add port */
1267 if (h->ae_algo->ops->add_tunnel_udp)
1268 h->ae_algo->ops->add_tunnel_udp(h, port);
1269}
1270
1271static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1272 enum hns3_udp_tnl_type type)
1273{
1274 struct hns3_nic_priv *priv = netdev_priv(netdev);
1275 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1276 struct hnae3_handle *h = priv->ae_handle;
1277
1278 if (!udp_tnl->used || udp_tnl->dst_port != port) {
1279 netdev_warn(netdev,
1280 "Invalid UDP tunnel port %d\n", port);
1281 return;
1282 }
1283
1284 udp_tnl->used--;
1285 if (udp_tnl->used)
1286 return;
1287
1288 udp_tnl->dst_port = 0;
1289 /* TBD send command to hardware to del port */
1290 if (h->ae_algo->ops->del_tunnel_udp)
9537e7cb 1291 h->ae_algo->ops->del_tunnel_udp(h, port);
76ad4f0e
S		 1292}
1293
1294/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1295 * @netdev: This physical ports's netdev
1296 * @ti: Tunnel information
1297 */
1298static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1299 struct udp_tunnel_info *ti)
1300{
1301 u16 port_n = ntohs(ti->port);
1302
1303 switch (ti->type) {
1304 case UDP_TUNNEL_TYPE_VXLAN:
1305 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1306 break;
1307 case UDP_TUNNEL_TYPE_GENEVE:
1308 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1309 break;
1310 default:
1311 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1312 break;
1313 }
1314}
1315
1316static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1317 struct udp_tunnel_info *ti)
1318{
1319 u16 port_n = ntohs(ti->port);
1320
1321 switch (ti->type) {
1322 case UDP_TUNNEL_TYPE_VXLAN:
1323 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1324 break;
1325 case UDP_TUNNEL_TYPE_GENEVE:
1326 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1327 break;
1328 default:
1329 break;
1330 }
1331}
1332
30d240df 1333static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1334{
30d240df 1335 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1336 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1337 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1338 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1339 u8 tc = mqprio_qopt->qopt.num_tc;
1340 u16 mode = mqprio_qopt->mode;
1341 u8 hw = mqprio_qopt->qopt.hw;
1342 bool if_running;
76ad4f0e
S		 1343 unsigned int i;
1344 int ret;
1345
30d240df
YL		 1346 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1347 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1348 return -EOPNOTSUPP;
1349
76ad4f0e
S		 1350 if (tc > HNAE3_MAX_TC)
1351 return -EINVAL;
1352
76ad4f0e
S		 1353 if (!netdev)
1354 return -EINVAL;
1355
30d240df
YL		 1356 if_running = netif_running(netdev);
1357 if (if_running) {
1358 hns3_nic_net_stop(netdev);
1359 msleep(100);
76ad4f0e
S		 1360 }
1361
30d240df
YL		 1362 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1363 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1364 if (ret)
30d240df
YL		 1365 goto out;
1366
1367 if (tc <= 1) {
1368 netdev_reset_tc(netdev);
1369 } else {
1370 ret = netdev_set_num_tc(netdev, tc);
1371 if (ret)
1372 goto out;
1373
1374 for (i = 0; i < HNAE3_MAX_TC; i++) {
1375 if (!kinfo->tc_info[i].enable)
1376 continue;
76ad4f0e 1377
76ad4f0e
S		 1378 netdev_set_tc_queue(netdev,
1379 kinfo->tc_info[i].tc,
1380 kinfo->tc_info[i].tqp_count,
1381 kinfo->tc_info[i].tqp_offset);
30d240df 1382 }
76ad4f0e
S		 1383 }
1384
30d240df
YL		 1385 ret = hns3_nic_set_real_num_queue(netdev);
1386
1387out:
1388 if (if_running)
1389 hns3_nic_net_open(netdev);
1390
1391 return ret;
76ad4f0e
S		 1392}
1393
2572ac53 1394static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1395 void *type_data)
76ad4f0e 1396{
575ed7d3 1397 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1398 return -EOPNOTSUPP;
76ad4f0e 1399
30d240df 1400 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1401}
1402
1403static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1404 __be16 proto, u16 vid)
1405{
9780cb97 1406 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1407 int ret = -EIO;
1408
1409 if (h->ae_algo->ops->set_vlan_filter)
1410 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1411
1412 return ret;
1413}
1414
1415static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1416 __be16 proto, u16 vid)
1417{
9780cb97 1418 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1419 int ret = -EIO;
1420
1421 if (h->ae_algo->ops->set_vlan_filter)
1422 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1423
1424 return ret;
1425}
1426
1427static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1428 u8 qos, __be16 vlan_proto)
1429{
9780cb97 1430 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1431 int ret = -EIO;
1432
1433 if (h->ae_algo->ops->set_vf_vlan_filter)
1434 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1435 qos, vlan_proto);
1436
1437 return ret;
1438}
1439
a8e8b7ff
S		 1440static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1441{
9780cb97 1442 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1443 bool if_running = netif_running(netdev);
1444 int ret;
1445
1446 if (!h->ae_algo->ops->set_mtu)
1447 return -EOPNOTSUPP;
1448
1449 /* if this was called with netdev up then bring netdevice down */
1450 if (if_running) {
1451 (void)hns3_nic_net_stop(netdev);
1452 msleep(100);
1453 }
1454
1455 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1456 if (ret) {
1457 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1458 ret);
1459 return ret;
1460 }
1461
5bad95a1
FL		 1462 netdev->mtu = new_mtu;
1463
a8e8b7ff
S		 1464 /* if the netdev was running earlier, bring it up again */
1465 if (if_running && hns3_nic_net_open(netdev))
1466 ret = -EINVAL;
1467
1468 return ret;
1469}
1470
f8fa222c
L		 1471static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1472{
1473 struct hns3_nic_priv *priv = netdev_priv(ndev);
1474 struct hns3_enet_ring *tx_ring = NULL;
1475 int timeout_queue = 0;
1476 int hw_head, hw_tail;
1477 int i;
1478
1479 /* Find the stopped queue the same way the stack does */
1480 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1481 struct netdev_queue *q;
1482 unsigned long trans_start;
1483
1484 q = netdev_get_tx_queue(ndev, i);
1485 trans_start = q->trans_start;
1486 if (netif_xmit_stopped(q) &&
1487 time_after(jiffies,
1488 (trans_start + ndev->watchdog_timeo))) {
1489 timeout_queue = i;
1490 break;
1491 }
1492 }
1493
1494 if (i == ndev->num_tx_queues) {
1495 netdev_info(ndev,
1496 "no netdev TX timeout queue found, timeout count: %llu\n",
1497 priv->tx_timeout_count);
1498 return false;
1499 }
1500
1501 tx_ring = priv->ring_data[timeout_queue].ring;
1502
1503 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1504 HNS3_RING_TX_RING_HEAD_REG);
1505 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1506 HNS3_RING_TX_RING_TAIL_REG);
1507 netdev_info(ndev,
1508 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1509 priv->tx_timeout_count,
1510 timeout_queue,
1511 tx_ring->next_to_use,
1512 tx_ring->next_to_clean,
1513 hw_head,
1514 hw_tail,
1515 readl(tx_ring->tqp_vector->mask_addr));
1516
1517 return true;
1518}
1519
1520static void hns3_nic_net_timeout(struct net_device *ndev)
1521{
1522 struct hns3_nic_priv *priv = netdev_priv(ndev);
1523 unsigned long last_reset_time = priv->last_reset_time;
1524 struct hnae3_handle *h = priv->ae_handle;
1525
1526 if (!hns3_get_tx_timeo_queue_info(ndev))
1527 return;
1528
1529 priv->tx_timeout_count++;
1530
1531 /* This timeout is far away enough from last timeout,
1532 * if timeout again,set the reset type to PF reset
1533 */
1534 if (time_after(jiffies, (last_reset_time + 20 * HZ)))
1535 priv->reset_level = HNAE3_FUNC_RESET;
1536
1537 /* Don't do any new action before the next timeout */
1538 else if (time_before(jiffies, (last_reset_time + ndev->watchdog_timeo)))
1539 return;
1540
1541 priv->last_reset_time = jiffies;
1542
1543 if (h->ae_algo->ops->reset_event)
1544 h->ae_algo->ops->reset_event(h, priv->reset_level);
1545
1546 priv->reset_level++;
1547 if (priv->reset_level > HNAE3_GLOBAL_RESET)
1548 priv->reset_level = HNAE3_GLOBAL_RESET;
1549}
1550
76ad4f0e
S		 1551static const struct net_device_ops hns3_nic_netdev_ops = {
1552 .ndo_open = hns3_nic_net_open,
1553 .ndo_stop = hns3_nic_net_stop,
1554 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1555 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1556 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a8e8b7ff 1557 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1558 .ndo_set_features = hns3_nic_set_features,
1559 .ndo_get_stats64 = hns3_nic_get_stats64,
1560 .ndo_setup_tc = hns3_nic_setup_tc,
1561 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
1562 .ndo_udp_tunnel_add = hns3_nic_udp_tunnel_add,
1563 .ndo_udp_tunnel_del = hns3_nic_udp_tunnel_del,
1564 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1565 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1566 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1567};
1568
1569/* hns3_probe - Device initialization routine
1570 * @pdev: PCI device information struct
1571 * @ent: entry in hns3_pci_tbl
1572 *
1573 * hns3_probe initializes a PF identified by a pci_dev structure.
1574 * The OS initialization, configuring of the PF private structure,
1575 * and a hardware reset occur.
1576 *
1577 * Returns 0 on success, negative on failure
1578 */
1579static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1580{
1581 struct hnae3_ae_dev *ae_dev;
1582 int ret;
1583
1584 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1585 GFP_KERNEL);
1586 if (!ae_dev) {
1587 ret = -ENOMEM;
1588 return ret;
1589 }
1590
1591 ae_dev->pdev = pdev;
e92a0843 1592 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 1593 ae_dev->dev_type = HNAE3_DEV_KNIC;
1594 pci_set_drvdata(pdev, ae_dev);
1595
1596 return hnae3_register_ae_dev(ae_dev);
1597}
1598
1599/* hns3_remove - Device removal routine
1600 * @pdev: PCI device information struct
1601 */
1602static void hns3_remove(struct pci_dev *pdev)
1603{
1604 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1605
1606 hnae3_unregister_ae_dev(ae_dev);
1607
1608 devm_kfree(&pdev->dev, ae_dev);
1609
1610 pci_set_drvdata(pdev, NULL);
1611}
1612
1613static struct pci_driver hns3_driver = {
1614 .name = hns3_driver_name,
1615 .id_table = hns3_pci_tbl,
1616 .probe = hns3_probe,
1617 .remove = hns3_remove,
1618};
1619
1620/* set default feature to hns3 */
1621static void hns3_set_default_feature(struct net_device *netdev)
1622{
391b5e93
JS		 1623 struct hnae3_handle *h = hns3_get_handle(netdev);
1624
76ad4f0e
S		 1625 netdev->priv_flags |= IFF_UNICAST_FLT;
1626
1627 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1628 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1629 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1630 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1631 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1632
1633 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1634
1635 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1636
1637 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1638 NETIF_F_HW_VLAN_CTAG_FILTER |
052ece6d 1639 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1640 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1641 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1642 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1643 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1644
1645 netdev->vlan_features |=
1646 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1647 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1648 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1649 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1650 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1651
1652 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
30ba2ab9 1653 NETIF_F_HW_VLAN_CTAG_TX |
76ad4f0e
S		 1654 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1655 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1656 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1657 NETIF_F_GSO_UDP_TUNNEL_CSUM;
391b5e93
JS		 1658
1659 if (!(h->flags & HNAE3_SUPPORT_VF))
30ba2ab9
JS		 1660 netdev->hw_features |=
1661 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
76ad4f0e
S		 1662}
1663
1664static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1665 struct hns3_desc_cb *cb)
1666{
1667 unsigned int order = hnae_page_order(ring);
1668 struct page *p;
1669
1670 p = dev_alloc_pages(order);
1671 if (!p)
1672 return -ENOMEM;
1673
1674 cb->priv = p;
1675 cb->page_offset = 0;
1676 cb->reuse_flag = 0;
1677 cb->buf = page_address(p);
1678 cb->length = hnae_page_size(ring);
1679 cb->type = DESC_TYPE_PAGE;
1680
76ad4f0e
S		 1681 return 0;
1682}
1683
1684static void hns3_free_buffer(struct hns3_enet_ring *ring,
1685 struct hns3_desc_cb *cb)
1686{
1687 if (cb->type == DESC_TYPE_SKB)
1688 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1689 else if (!HNAE3_IS_TX_RING(ring))
1690 put_page((struct page *)cb->priv);
1691 memset(cb, 0, sizeof(*cb));
1692}
1693
1694static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1695{
1696 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1697 cb->length, ring_to_dma_dir(ring));
1698
1699 if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1700 return -EIO;
1701
1702 return 0;
1703}
1704
1705static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1706 struct hns3_desc_cb *cb)
1707{
1708 if (cb->type == DESC_TYPE_SKB)
1709 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1710 ring_to_dma_dir(ring));
1711 else
1712 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1713 ring_to_dma_dir(ring));
1714}
1715
1716static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1717{
1718 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1719 ring->desc[i].addr = 0;
1720}
1721
1722static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1723{
1724 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1725
1726 if (!ring->desc_cb[i].dma)
1727 return;
1728
1729 hns3_buffer_detach(ring, i);
1730 hns3_free_buffer(ring, cb);
1731}
1732
1733static void hns3_free_buffers(struct hns3_enet_ring *ring)
1734{
1735 int i;
1736
1737 for (i = 0; i < ring->desc_num; i++)
1738 hns3_free_buffer_detach(ring, i);
1739}
1740
1741/* free desc along with its attached buffer */
1742static void hns3_free_desc(struct hns3_enet_ring *ring)
1743{
1744 hns3_free_buffers(ring);
1745
1746 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1747 ring->desc_num * sizeof(ring->desc[0]),
1748 DMA_BIDIRECTIONAL);
1749 ring->desc_dma_addr = 0;
1750 kfree(ring->desc);
1751 ring->desc = NULL;
1752}
1753
1754static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1755{
1756 int size = ring->desc_num * sizeof(ring->desc[0]);
1757
1758 ring->desc = kzalloc(size, GFP_KERNEL);
1759 if (!ring->desc)
1760 return -ENOMEM;
1761
1762 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1763 size, DMA_BIDIRECTIONAL);
1764 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1765 ring->desc_dma_addr = 0;
1766 kfree(ring->desc);
1767 ring->desc = NULL;
1768 return -ENOMEM;
1769 }
1770
1771 return 0;
1772}
1773
1774static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1775 struct hns3_desc_cb *cb)
1776{
1777 int ret;
1778
1779 ret = hns3_alloc_buffer(ring, cb);
1780 if (ret)
1781 goto out;
1782
1783 ret = hns3_map_buffer(ring, cb);
1784 if (ret)
1785 goto out_with_buf;
1786
1787 return 0;
1788
1789out_with_buf:
564883bb 1790 hns3_free_buffer(ring, cb);
76ad4f0e
S		 1791out:
1792 return ret;
1793}
1794
1795static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1796{
1797 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1798
1799 if (ret)
1800 return ret;
1801
1802 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1803
1804 return 0;
1805}
1806
1807/* Allocate memory for raw pkg, and map with dma */
1808static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1809{
1810 int i, j, ret;
1811
1812 for (i = 0; i < ring->desc_num; i++) {
1813 ret = hns3_alloc_buffer_attach(ring, i);
1814 if (ret)
1815 goto out_buffer_fail;
1816 }
1817
1818 return 0;
1819
1820out_buffer_fail:
1821 for (j = i - 1; j >= 0; j--)
1822 hns3_free_buffer_detach(ring, j);
1823 return ret;
1824}
1825
1826/* detach a in-used buffer and replace with a reserved one */
1827static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1828 struct hns3_desc_cb *res_cb)
1829{
b9077428 1830 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 1831 ring->desc_cb[i] = *res_cb;
1832 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1833}
1834
1835static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1836{
1837 ring->desc_cb[i].reuse_flag = 0;
1838 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1839 + ring->desc_cb[i].page_offset);
1840}
1841
1842static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1843 int *pkts)
1844{
1845 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1846
1847 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1848 (*bytes) += desc_cb->length;
1849 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1850 hns3_free_buffer_detach(ring, ring->next_to_clean);
1851
1852 ring_ptr_move_fw(ring, next_to_clean);
1853}
1854
1855static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1856{
1857 int u = ring->next_to_use;
1858 int c = ring->next_to_clean;
1859
1860 if (unlikely(h > ring->desc_num))
1861 return 0;
1862
1863 return u > c ? (h > c && h <= u) : (h > c || h <= u);
1864}
1865
24e750c4 1866bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e
S		 1867{
1868 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1869 struct netdev_queue *dev_queue;
1870 int bytes, pkts;
1871 int head;
1872
1873 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1874 rmb(); /* Make sure head is ready before touch any data */
1875
1876 if (is_ring_empty(ring) || head == ring->next_to_clean)
24e750c4 1877 return true; /* no data to poll */
76ad4f0e
S		 1878
1879 if (!is_valid_clean_head(ring, head)) {
1880 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1881 ring->next_to_use, ring->next_to_clean);
1882
1883 u64_stats_update_begin(&ring->syncp);
1884 ring->stats.io_err_cnt++;
1885 u64_stats_update_end(&ring->syncp);
24e750c4 1886 return true;
76ad4f0e
S		 1887 }
1888
1889 bytes = 0;
1890 pkts = 0;
1891 while (head != ring->next_to_clean && budget) {
1892 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1893 /* Issue prefetch for next Tx descriptor */
1894 prefetch(&ring->desc_cb[ring->next_to_clean]);
1895 budget--;
1896 }
1897
1898 ring->tqp_vector->tx_group.total_bytes += bytes;
1899 ring->tqp_vector->tx_group.total_packets += pkts;
1900
1901 u64_stats_update_begin(&ring->syncp);
1902 ring->stats.tx_bytes += bytes;
1903 ring->stats.tx_pkts += pkts;
1904 u64_stats_update_end(&ring->syncp);
1905
1906 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1907 netdev_tx_completed_queue(dev_queue, pkts, bytes);
1908
1909 if (unlikely(pkts && netif_carrier_ok(netdev) &&
1910 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1911 /* Make sure that anybody stopping the queue after this
1912 * sees the new next_to_clean.
1913 */
1914 smp_mb();
1915 if (netif_tx_queue_stopped(dev_queue)) {
1916 netif_tx_wake_queue(dev_queue);
1917 ring->stats.restart_queue++;
1918 }
1919 }
1920
1921 return !!budget;
1922}
1923
1924static int hns3_desc_unused(struct hns3_enet_ring *ring)
1925{
1926 int ntc = ring->next_to_clean;
1927 int ntu = ring->next_to_use;
1928
1929 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1930}
1931
1932static void
1933hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1934{
1935 struct hns3_desc_cb *desc_cb;
1936 struct hns3_desc_cb res_cbs;
1937 int i, ret;
1938
1939 for (i = 0; i < cleand_count; i++) {
1940 desc_cb = &ring->desc_cb[ring->next_to_use];
1941 if (desc_cb->reuse_flag) {
1942 u64_stats_update_begin(&ring->syncp);
1943 ring->stats.reuse_pg_cnt++;
1944 u64_stats_update_end(&ring->syncp);
1945
1946 hns3_reuse_buffer(ring, ring->next_to_use);
1947 } else {
1948 ret = hns3_reserve_buffer_map(ring, &res_cbs);
1949 if (ret) {
1950 u64_stats_update_begin(&ring->syncp);
1951 ring->stats.sw_err_cnt++;
1952 u64_stats_update_end(&ring->syncp);
1953
1954 netdev_err(ring->tqp->handle->kinfo.netdev,
1955 "hnae reserve buffer map failed.\n");
1956 break;
1957 }
1958 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1959 }
1960
1961 ring_ptr_move_fw(ring, next_to_use);
1962 }
1963
1964 wmb(); /* Make all data has been write before submit */
1965 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1966}
1967
1968/* hns3_nic_get_headlen - determine size of header for LRO/GRO
1969 * @data: pointer to the start of the headers
1970 * @max: total length of section to find headers in
1971 *
1972 * This function is meant to determine the length of headers that will
1973 * be recognized by hardware for LRO, GRO, and RSC offloads. The main
1974 * motivation of doing this is to only perform one pull for IPv4 TCP
1975 * packets so that we can do basic things like calculating the gso_size
1976 * based on the average data per packet.
1977 */
1978static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1979 unsigned int max_size)
1980{
1981 unsigned char *network;
1982 u8 hlen;
1983
1984 /* This should never happen, but better safe than sorry */
1985 if (max_size < ETH_HLEN)
1986 return max_size;
1987
1988 /* Initialize network frame pointer */
1989 network = data;
1990
1991 /* Set first protocol and move network header forward */
1992 network += ETH_HLEN;
1993
1994 /* Handle any vlan tag if present */
1995 if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1996 == HNS3_RX_FLAG_VLAN_PRESENT) {
1997 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1998 return max_size;
1999
2000 network += VLAN_HLEN;
2001 }
2002
2003 /* Handle L3 protocols */
2004 if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
2005 == HNS3_RX_FLAG_L3ID_IPV4) {
2006 if ((typeof(max_size))(network - data) >
2007 (max_size - sizeof(struct iphdr)))
2008 return max_size;
2009
2010 /* Access ihl as a u8 to avoid unaligned access on ia64 */
2011 hlen = (network[0] & 0x0F) << 2;
2012
2013 /* Verify hlen meets minimum size requirements */
2014 if (hlen < sizeof(struct iphdr))
2015 return network - data;
2016
2017 /* Record next protocol if header is present */
2018 } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
2019 == HNS3_RX_FLAG_L3ID_IPV6) {
2020 if ((typeof(max_size))(network - data) >
2021 (max_size - sizeof(struct ipv6hdr)))
2022 return max_size;
2023
2024 /* Record next protocol */
2025 hlen = sizeof(struct ipv6hdr);
2026 } else {
2027 return network - data;
2028 }
2029
2030 /* Relocate pointer to start of L4 header */
2031 network += hlen;
2032
2033 /* Finally sort out TCP/UDP */
2034 if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
2035 == HNS3_RX_FLAG_L4ID_TCP) {
2036 if ((typeof(max_size))(network - data) >
2037 (max_size - sizeof(struct tcphdr)))
2038 return max_size;
2039
2040 /* Access doff as a u8 to avoid unaligned access on ia64 */
2041 hlen = (network[12] & 0xF0) >> 2;
2042
2043 /* Verify hlen meets minimum size requirements */
2044 if (hlen < sizeof(struct tcphdr))
2045 return network - data;
2046
2047 network += hlen;
2048 } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
2049 == HNS3_RX_FLAG_L4ID_UDP) {
2050 if ((typeof(max_size))(network - data) >
2051 (max_size - sizeof(struct udphdr)))
2052 return max_size;
2053
2054 network += sizeof(struct udphdr);
2055 }
2056
2057 /* If everything has gone correctly network should be the
2058 * data section of the packet and will be the end of the header.
2059 * If not then it probably represents the end of the last recognized
2060 * header.
2061 */
2062 if ((typeof(max_size))(network - data) < max_size)
2063 return network - data;
2064 else
2065 return max_size;
2066}
2067
2068static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2069 struct hns3_enet_ring *ring, int pull_len,
2070 struct hns3_desc_cb *desc_cb)
2071{
2072 struct hns3_desc *desc;
2073 int truesize, size;
2074 int last_offset;
2075 bool twobufs;
2076
2077 twobufs = ((PAGE_SIZE < 8192) &&
2078 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2079
2080 desc = &ring->desc[ring->next_to_clean];
2081 size = le16_to_cpu(desc->rx.size);
2082
2083 if (twobufs) {
2084 truesize = hnae_buf_size(ring);
2085 } else {
2086 truesize = ALIGN(size, L1_CACHE_BYTES);
2087 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
2088 }
2089
2090 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2091 size - pull_len, truesize - pull_len);
2092
2093 /* Avoid re-using remote pages,flag default unreuse */
2094 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2095 return;
2096
2097 if (twobufs) {
2098 /* If we are only owner of page we can reuse it */
2099 if (likely(page_count(desc_cb->priv) == 1)) {
2100 /* Flip page offset to other buffer */
2101 desc_cb->page_offset ^= truesize;
2102
2103 desc_cb->reuse_flag = 1;
2104 /* bump ref count on page before it is given*/
2105 get_page(desc_cb->priv);
2106 }
2107 return;
2108 }
2109
2110 /* Move offset up to the next cache line */
2111 desc_cb->page_offset += truesize;
2112
2113 if (desc_cb->page_offset <= last_offset) {
2114 desc_cb->reuse_flag = 1;
2115 /* Bump ref count on page before it is given*/
2116 get_page(desc_cb->priv);
2117 }
2118}
2119
2120static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2121 struct hns3_desc *desc)
2122{
2123 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2124 int l3_type, l4_type;
2125 u32 bd_base_info;
2126 int ol4_type;
2127 u32 l234info;
2128
2129 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2130 l234info = le32_to_cpu(desc->rx.l234_info);
2131
2132 skb->ip_summed = CHECKSUM_NONE;
2133
2134 skb_checksum_none_assert(skb);
2135
2136 if (!(netdev->features & NETIF_F_RXCSUM))
2137 return;
2138
2139 /* check if hardware has done checksum */
2140 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2141 return;
2142
2143 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
2144 hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
2145 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2146 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2147 netdev_err(netdev, "L3/L4 error pkt\n");
2148 u64_stats_update_begin(&ring->syncp);
2149 ring->stats.l3l4_csum_err++;
2150 u64_stats_update_end(&ring->syncp);
2151
2152 return;
2153 }
2154
2155 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
2156 HNS3_RXD_L3ID_S);
2157 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
2158 HNS3_RXD_L4ID_S);
2159
2160 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
2161 switch (ol4_type) {
2162 case HNS3_OL4_TYPE_MAC_IN_UDP:
2163 case HNS3_OL4_TYPE_NVGRE:
2164 skb->csum_level = 1;
2165 case HNS3_OL4_TYPE_NO_TUN:
2166 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2167 if (l3_type == HNS3_L3_TYPE_IPV4 ||
2168 (l3_type == HNS3_L3_TYPE_IPV6 &&
2169 (l4_type == HNS3_L4_TYPE_UDP ||
2170 l4_type == HNS3_L4_TYPE_TCP ||
2171 l4_type == HNS3_L4_TYPE_SCTP)))
2172 skb->ip_summed = CHECKSUM_UNNECESSARY;
2173 break;
2174 }
2175}
2176
d43e5aca
YL		 2177static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2178{
2179 napi_gro_receive(&ring->tqp_vector->napi, skb);
2180}
2181
76ad4f0e
S		 2182static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2183 struct sk_buff **out_skb, int *out_bnum)
2184{
2185 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2186 struct hns3_desc_cb *desc_cb;
2187 struct hns3_desc *desc;
2188 struct sk_buff *skb;
2189 unsigned char *va;
2190 u32 bd_base_info;
2191 int pull_len;
2192 u32 l234info;
2193 int length;
2194 int bnum;
2195
2196 desc = &ring->desc[ring->next_to_clean];
2197 desc_cb = &ring->desc_cb[ring->next_to_clean];
2198
2199 prefetch(desc);
2200
2201 length = le16_to_cpu(desc->rx.pkt_len);
2202 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2203 l234info = le32_to_cpu(desc->rx.l234_info);
2204
2205 /* Check valid BD */
2206 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2207 return -EFAULT;
2208
2209 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2210
2211 /* Prefetch first cache line of first page
2212 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2213 * line size is 64B so need to prefetch twice to make it 128B. But in
2214 * actual we can have greater size of caches with 128B Level 1 cache
2215 * lines. In such a case, single fetch would suffice to cache in the
2216 * relevant part of the header.
2217 */
2218 prefetch(va);
2219#if L1_CACHE_BYTES < 128
2220 prefetch(va + L1_CACHE_BYTES);
2221#endif
2222
2223 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2224 HNS3_RX_HEAD_SIZE);
2225 if (unlikely(!skb)) {
2226 netdev_err(netdev, "alloc rx skb fail\n");
2227
2228 u64_stats_update_begin(&ring->syncp);
2229 ring->stats.sw_err_cnt++;
2230 u64_stats_update_end(&ring->syncp);
2231
2232 return -ENOMEM;
2233 }
2234
2235 prefetchw(skb->data);
2236
9699cffe
PL		 2237 /* Based on hw strategy, the tag offloaded will be stored at
2238 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2239 * in one layer tag case.
2240 */
2241 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2242 u16 vlan_tag;
2243
2244 vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2245 if (!(vlan_tag & VLAN_VID_MASK))
2246 vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2247 if (vlan_tag & VLAN_VID_MASK)
2248 __vlan_hwaccel_put_tag(skb,
2249 htons(ETH_P_8021Q),
2250 vlan_tag);
2251 }
2252
76ad4f0e
S		 2253 bnum = 1;
2254 if (length <= HNS3_RX_HEAD_SIZE) {
2255 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2256
2257 /* We can reuse buffer as-is, just make sure it is local */
2258 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2259 desc_cb->reuse_flag = 1;
2260 else /* This page cannot be reused so discard it */
2261 put_page(desc_cb->priv);
2262
2263 ring_ptr_move_fw(ring, next_to_clean);
2264 } else {
2265 u64_stats_update_begin(&ring->syncp);
2266 ring->stats.seg_pkt_cnt++;
2267 u64_stats_update_end(&ring->syncp);
2268
2269 pull_len = hns3_nic_get_headlen(va, l234info,
2270 HNS3_RX_HEAD_SIZE);
2271 memcpy(__skb_put(skb, pull_len), va,
2272 ALIGN(pull_len, sizeof(long)));
2273
2274 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2275 ring_ptr_move_fw(ring, next_to_clean);
2276
2277 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2278 desc = &ring->desc[ring->next_to_clean];
2279 desc_cb = &ring->desc_cb[ring->next_to_clean];
2280 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2281 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2282 ring_ptr_move_fw(ring, next_to_clean);
2283 bnum++;
2284 }
2285 }
2286
2287 *out_bnum = bnum;
2288
2289 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2290 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2291 ((u64 *)desc)[0], ((u64 *)desc)[1]);
2292 u64_stats_update_begin(&ring->syncp);
2293 ring->stats.non_vld_descs++;
2294 u64_stats_update_end(&ring->syncp);
2295
2296 dev_kfree_skb_any(skb);
2297 return -EINVAL;
2298 }
2299
2300 if (unlikely((!desc->rx.pkt_len) ||
2301 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2302 netdev_err(netdev, "truncated pkt\n");
2303 u64_stats_update_begin(&ring->syncp);
2304 ring->stats.err_pkt_len++;
2305 u64_stats_update_end(&ring->syncp);
2306
2307 dev_kfree_skb_any(skb);
2308 return -EFAULT;
2309 }
2310
2311 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2312 netdev_err(netdev, "L2 error pkt\n");
2313 u64_stats_update_begin(&ring->syncp);
2314 ring->stats.l2_err++;
2315 u64_stats_update_end(&ring->syncp);
2316
2317 dev_kfree_skb_any(skb);
2318 return -EFAULT;
2319 }
2320
2321 u64_stats_update_begin(&ring->syncp);
2322 ring->stats.rx_pkts++;
2323 ring->stats.rx_bytes += skb->len;
2324 u64_stats_update_end(&ring->syncp);
2325
2326 ring->tqp_vector->rx_group.total_bytes += skb->len;
2327
2328 hns3_rx_checksum(ring, skb, desc);
2329 return 0;
2330}
2331
d43e5aca
YL		 2332int hns3_clean_rx_ring(
2333 struct hns3_enet_ring *ring, int budget,
2334 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2335{
2336#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2337 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2338 int recv_pkts, recv_bds, clean_count, err;
2339 int unused_count = hns3_desc_unused(ring);
2340 struct sk_buff *skb = NULL;
2341 int num, bnum = 0;
2342
2343 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2344 rmb(); /* Make sure num taken effect before the other data is touched */
2345
2346 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2347 num -= unused_count;
2348
2349 while (recv_pkts < budget && recv_bds < num) {
2350 /* Reuse or realloc buffers */
2351 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2352 hns3_nic_alloc_rx_buffers(ring,
2353 clean_count + unused_count);
2354 clean_count = 0;
2355 unused_count = hns3_desc_unused(ring);
2356 }
2357
2358 /* Poll one pkt */
2359 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2360 if (unlikely(!skb)) /* This fault cannot be repaired */
2361 goto out;
2362
2363 recv_bds += bnum;
2364 clean_count += bnum;
2365 if (unlikely(err)) { /* Do jump the err */
2366 recv_pkts++;
2367 continue;
2368 }
2369
2370 /* Do update ip stack process */
2371 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2372 rx_fn(ring, skb);
76ad4f0e
S		 2373
2374 recv_pkts++;
2375 }
2376
2377out:
2378 /* Make all data has been write before submit */
2379 if (clean_count + unused_count > 0)
2380 hns3_nic_alloc_rx_buffers(ring,
2381 clean_count + unused_count);
2382
2383 return recv_pkts;
2384}
2385
2386static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2387{
2388#define HNS3_RX_ULTRA_PACKET_RATE 40000
2389 enum hns3_flow_level_range new_flow_level;
2390 struct hns3_enet_tqp_vector *tqp_vector;
2391 int packets_per_secs;
2392 int bytes_per_usecs;
2393 u16 new_int_gl;
2394 int usecs;
2395
9bc727a9 2396 if (!ring_group->coal.int_gl)
76ad4f0e
S		 2397 return false;
2398
2399 if (ring_group->total_packets == 0) {
9bc727a9
YL		 2400 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2401 ring_group->coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 2402 return true;
2403 }
2404
2405 /* Simple throttlerate management
2406 * 0-10MB/s lower (50000 ints/s)
2407 * 10-20MB/s middle (20000 ints/s)
2408 * 20-1249MB/s high (18000 ints/s)
2409 * > 40000pps ultra (8000 ints/s)
2410 */
9bc727a9
YL		 2411 new_flow_level = ring_group->coal.flow_level;
2412 new_int_gl = ring_group->coal.int_gl;
76ad4f0e 2413 tqp_vector = ring_group->ring->tqp_vector;
9bc727a9 2414 usecs = (ring_group->coal.int_gl << 1);
76ad4f0e
S		 2415 bytes_per_usecs = ring_group->total_bytes / usecs;
2416 /* 1000000 microseconds */
2417 packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2418
2419 switch (new_flow_level) {
2420 case HNS3_FLOW_LOW:
2421 if (bytes_per_usecs > 10)
2422 new_flow_level = HNS3_FLOW_MID;
2423 break;
2424 case HNS3_FLOW_MID:
2425 if (bytes_per_usecs > 20)
2426 new_flow_level = HNS3_FLOW_HIGH;
2427 else if (bytes_per_usecs <= 10)
2428 new_flow_level = HNS3_FLOW_LOW;
2429 break;
2430 case HNS3_FLOW_HIGH:
2431 case HNS3_FLOW_ULTRA:
2432 default:
2433 if (bytes_per_usecs <= 20)
2434 new_flow_level = HNS3_FLOW_MID;
2435 break;
2436 }
2437
2438 if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2439 (&tqp_vector->rx_group == ring_group))
2440 new_flow_level = HNS3_FLOW_ULTRA;
2441
2442 switch (new_flow_level) {
2443 case HNS3_FLOW_LOW:
2444 new_int_gl = HNS3_INT_GL_50K;
2445 break;
2446 case HNS3_FLOW_MID:
2447 new_int_gl = HNS3_INT_GL_20K;
2448 break;
2449 case HNS3_FLOW_HIGH:
2450 new_int_gl = HNS3_INT_GL_18K;
2451 break;
2452 case HNS3_FLOW_ULTRA:
2453 new_int_gl = HNS3_INT_GL_8K;
2454 break;
2455 default:
2456 break;
2457 }
2458
2459 ring_group->total_bytes = 0;
2460 ring_group->total_packets = 0;
9bc727a9
YL		 2461 ring_group->coal.flow_level = new_flow_level;
2462 if (new_int_gl != ring_group->coal.int_gl) {
2463 ring_group->coal.int_gl = new_int_gl;
76ad4f0e
S		 2464 return true;
2465 }
2466 return false;
2467}
2468
2469static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2470{
8b1ff1ea
FL		 2471 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2472 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2473 bool rx_update, tx_update;
2474
9bc727a9 2475 if (rx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2476 rx_update = hns3_get_new_int_gl(rx_group);
2477 if (rx_update)
2478 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 2479 rx_group->coal.int_gl);
8b1ff1ea
FL		 2480 }
2481
9bc727a9 2482 if (tx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2483 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2484 if (tx_update)
2485 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 2486 tx_group->coal.int_gl);
76ad4f0e
S		 2487 }
2488}
2489
2490static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2491{
2492 struct hns3_enet_ring *ring;
2493 int rx_pkt_total = 0;
2494
2495 struct hns3_enet_tqp_vector *tqp_vector =
2496 container_of(napi, struct hns3_enet_tqp_vector, napi);
2497 bool clean_complete = true;
2498 int rx_budget;
2499
2500 /* Since the actual Tx work is minimal, we can give the Tx a larger
2501 * budget and be more aggressive about cleaning up the Tx descriptors.
2502 */
2503 hns3_for_each_ring(ring, tqp_vector->tx_group) {
2504 if (!hns3_clean_tx_ring(ring, budget))
2505 clean_complete = false;
2506 }
2507
2508 /* make sure rx ring budget not smaller than 1 */
2509 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2510
2511 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2512 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2513 hns3_rx_skb);
76ad4f0e
S		 2514
2515 if (rx_cleaned >= rx_budget)
2516 clean_complete = false;
2517
2518 rx_pkt_total += rx_cleaned;
2519 }
2520
2521 tqp_vector->rx_group.total_packets += rx_pkt_total;
2522
2523 if (!clean_complete)
2524 return budget;
2525
2526 napi_complete(napi);
2527 hns3_update_new_int_gl(tqp_vector);
2528 hns3_mask_vector_irq(tqp_vector, 1);
2529
2530 return rx_pkt_total;
2531}
2532
2533static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2534 struct hnae3_ring_chain_node *head)
2535{
2536 struct pci_dev *pdev = tqp_vector->handle->pdev;
2537 struct hnae3_ring_chain_node *cur_chain = head;
2538 struct hnae3_ring_chain_node *chain;
2539 struct hns3_enet_ring *tx_ring;
2540 struct hns3_enet_ring *rx_ring;
2541
2542 tx_ring = tqp_vector->tx_group.ring;
2543 if (tx_ring) {
2544 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2545 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2546 HNAE3_RING_TYPE_TX);
11af96a4
FL		 2547 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2548 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 2549
2550 cur_chain->next = NULL;
2551
2552 while (tx_ring->next) {
2553 tx_ring = tx_ring->next;
2554
2555 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2556 GFP_KERNEL);
2557 if (!chain)
2558 return -ENOMEM;
2559
2560 cur_chain->next = chain;
2561 chain->tqp_index = tx_ring->tqp->tqp_index;
2562 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2563 HNAE3_RING_TYPE_TX);
11af96a4
FL		 2564 hnae_set_field(chain->int_gl_idx,
2565 HNAE3_RING_GL_IDX_M,
2566 HNAE3_RING_GL_IDX_S,
2567 HNAE3_RING_GL_TX);
76ad4f0e
S		 2568
2569 cur_chain = chain;
2570 }
2571 }
2572
2573 rx_ring = tqp_vector->rx_group.ring;
2574 if (!tx_ring && rx_ring) {
2575 cur_chain->next = NULL;
2576 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2577 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2578 HNAE3_RING_TYPE_RX);
11af96a4
FL		 2579 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2580 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 2581
2582 rx_ring = rx_ring->next;
2583 }
2584
2585 while (rx_ring) {
2586 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2587 if (!chain)
2588 return -ENOMEM;
2589
2590 cur_chain->next = chain;
2591 chain->tqp_index = rx_ring->tqp->tqp_index;
2592 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2593 HNAE3_RING_TYPE_RX);
11af96a4
FL		 2594 hnae_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2595 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2596
76ad4f0e
S		 2597 cur_chain = chain;
2598
2599 rx_ring = rx_ring->next;
2600 }
2601
2602 return 0;
2603}
2604
2605static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2606 struct hnae3_ring_chain_node *head)
2607{
2608 struct pci_dev *pdev = tqp_vector->handle->pdev;
2609 struct hnae3_ring_chain_node *chain_tmp, *chain;
2610
2611 chain = head->next;
2612
2613 while (chain) {
2614 chain_tmp = chain->next;
2615 devm_kfree(&pdev->dev, chain);
2616 chain = chain_tmp;
2617 }
2618}
2619
2620static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2621 struct hns3_enet_ring *ring)
2622{
2623 ring->next = group->ring;
2624 group->ring = ring;
2625
2626 group->count++;
2627}
2628
2629static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2630{
2631 struct hnae3_ring_chain_node vector_ring_chain;
2632 struct hnae3_handle *h = priv->ae_handle;
2633 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 2634 int ret = 0;
2635 u16 i;
2636
dd38c726
YL		 2637 for (i = 0; i < priv->vector_num; i++) {
2638 tqp_vector = &priv->tqp_vector[i];
2639 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2640 tqp_vector->num_tqps = 0;
2641 }
76ad4f0e 2642
dd38c726
YL		 2643 for (i = 0; i < h->kinfo.num_tqps; i++) {
2644 u16 vector_i = i % priv->vector_num;
2645 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 2646
2647 tqp_vector = &priv->tqp_vector[vector_i];
2648
2649 hns3_add_ring_to_group(&tqp_vector->tx_group,
2650 priv->ring_data[i].ring);
2651
2652 hns3_add_ring_to_group(&tqp_vector->rx_group,
2653 priv->ring_data[i + tqp_num].ring);
2654
76ad4f0e
S		 2655 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2656 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
dd38c726 2657 tqp_vector->num_tqps++;
76ad4f0e
S		 2658 }
2659
dd38c726 2660 for (i = 0; i < priv->vector_num; i++) {
76ad4f0e
S		 2661 tqp_vector = &priv->tqp_vector[i];
2662
2663 tqp_vector->rx_group.total_bytes = 0;
2664 tqp_vector->rx_group.total_packets = 0;
2665 tqp_vector->tx_group.total_bytes = 0;
2666 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 2667 tqp_vector->handle = h;
2668
2669 ret = hns3_get_vector_ring_chain(tqp_vector,
2670 &vector_ring_chain);
2671 if (ret)
dd38c726 2672 return ret;
76ad4f0e
S		 2673
2674 ret = h->ae_algo->ops->map_ring_to_vector(h,
2675 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 2676
2677 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2678
dd38c726
YL		 2679 if (ret)
2680 return ret;
2681
76ad4f0e
S		 2682 netif_napi_add(priv->netdev, &tqp_vector->napi,
2683 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2684 }
2685
dd38c726
YL		 2686 return 0;
2687}
2688
2689static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2690{
2691 struct hnae3_handle *h = priv->ae_handle;
2692 struct hns3_enet_tqp_vector *tqp_vector;
2693 struct hnae3_vector_info *vector;
2694 struct pci_dev *pdev = h->pdev;
2695 u16 tqp_num = h->kinfo.num_tqps;
2696 u16 vector_num;
2697 int ret = 0;
2698 u16 i;
2699
2700 /* RSS size, cpu online and vector_num should be the same */
2701 /* Should consider 2p/4p later */
2702 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2703 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2704 GFP_KERNEL);
2705 if (!vector)
2706 return -ENOMEM;
2707
2708 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2709
2710 priv->vector_num = vector_num;
2711 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2712 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2713 GFP_KERNEL);
2714 if (!priv->tqp_vector) {
2715 ret = -ENOMEM;
2716 goto out;
2717 }
2718
2719 for (i = 0; i < priv->vector_num; i++) {
2720 tqp_vector = &priv->tqp_vector[i];
2721 tqp_vector->idx = i;
2722 tqp_vector->mask_addr = vector[i].io_addr;
2723 tqp_vector->vector_irq = vector[i].vector;
2724 hns3_vector_gl_rl_init(tqp_vector, priv);
2725 }
2726
76ad4f0e
S		 2727out:
2728 devm_kfree(&pdev->dev, vector);
2729 return ret;
2730}
2731
dd38c726
YL		 2732static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2733{
2734 group->ring = NULL;
2735 group->count = 0;
2736}
2737
76ad4f0e
S		 2738static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2739{
2740 struct hnae3_ring_chain_node vector_ring_chain;
2741 struct hnae3_handle *h = priv->ae_handle;
2742 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 2743 int i, ret;
2744
2745 for (i = 0; i < priv->vector_num; i++) {
2746 tqp_vector = &priv->tqp_vector[i];
2747
2748 ret = hns3_get_vector_ring_chain(tqp_vector,
2749 &vector_ring_chain);
2750 if (ret)
2751 return ret;
2752
2753 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2754 tqp_vector->vector_irq, &vector_ring_chain);
2755 if (ret)
2756 return ret;
2757
0d3e6631
YL		 2758 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2759 if (ret)
2760 return ret;
2761
76ad4f0e
S		 2762 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2763
2764 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2765 (void)irq_set_affinity_hint(
2766 priv->tqp_vector[i].vector_irq,
2767 NULL);
ae064e61 2768 free_irq(priv->tqp_vector[i].vector_irq,
2769 &priv->tqp_vector[i]);
76ad4f0e
S		 2770 }
2771
2772 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
dd38c726
YL		 2773 hns3_clear_ring_group(&tqp_vector->rx_group);
2774 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 2775 netif_napi_del(&priv->tqp_vector[i].napi);
2776 }
2777
dd38c726
YL		 2778 return 0;
2779}
2780
2781static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2782{
2783 struct hnae3_handle *h = priv->ae_handle;
2784 struct pci_dev *pdev = h->pdev;
2785 int i, ret;
2786
2787 for (i = 0; i < priv->vector_num; i++) {
2788 struct hns3_enet_tqp_vector *tqp_vector;
2789
2790 tqp_vector = &priv->tqp_vector[i];
2791 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2792 if (ret)
2793 return ret;
2794 }
76ad4f0e 2795
dd38c726 2796 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 2797 return 0;
2798}
2799
2800static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2801 int ring_type)
2802{
2803 struct hns3_nic_ring_data *ring_data = priv->ring_data;
2804 int queue_num = priv->ae_handle->kinfo.num_tqps;
2805 struct pci_dev *pdev = priv->ae_handle->pdev;
2806 struct hns3_enet_ring *ring;
2807
2808 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2809 if (!ring)
2810 return -ENOMEM;
2811
2812 if (ring_type == HNAE3_RING_TYPE_TX) {
2813 ring_data[q->tqp_index].ring = ring;
66b44730 2814 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 2815 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2816 } else {
2817 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 2818 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 2819 ring->io_base = q->io_base;
2820 }
2821
2822 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2823
76ad4f0e
S		 2824 ring->tqp = q;
2825 ring->desc = NULL;
2826 ring->desc_cb = NULL;
2827 ring->dev = priv->dev;
2828 ring->desc_dma_addr = 0;
2829 ring->buf_size = q->buf_size;
2830 ring->desc_num = q->desc_num;
2831 ring->next_to_use = 0;
2832 ring->next_to_clean = 0;
2833
2834 return 0;
2835}
2836
2837static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2838 struct hns3_nic_priv *priv)
2839{
2840 int ret;
2841
2842 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2843 if (ret)
2844 return ret;
2845
2846 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2847 if (ret)
2848 return ret;
2849
2850 return 0;
2851}
2852
2853static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2854{
2855 struct hnae3_handle *h = priv->ae_handle;
2856 struct pci_dev *pdev = h->pdev;
2857 int i, ret;
2858
2859 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2860 sizeof(*priv->ring_data) * 2,
2861 GFP_KERNEL);
2862 if (!priv->ring_data)
2863 return -ENOMEM;
2864
2865 for (i = 0; i < h->kinfo.num_tqps; i++) {
2866 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2867 if (ret)
2868 goto err;
2869 }
2870
2871 return 0;
2872err:
2873 devm_kfree(&pdev->dev, priv->ring_data);
2874 return ret;
2875}
2876
09f2af64
PL		 2877static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2878{
2879 struct hnae3_handle *h = priv->ae_handle;
2880 int i;
2881
2882 for (i = 0; i < h->kinfo.num_tqps; i++) {
2883 devm_kfree(priv->dev, priv->ring_data[i].ring);
2884 devm_kfree(priv->dev,
2885 priv->ring_data[i + h->kinfo.num_tqps].ring);
2886 }
2887 devm_kfree(priv->dev, priv->ring_data);
2888}
2889
76ad4f0e
S		 2890static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2891{
2892 int ret;
2893
2894 if (ring->desc_num <= 0 || ring->buf_size <= 0)
2895 return -EINVAL;
2896
2897 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2898 GFP_KERNEL);
2899 if (!ring->desc_cb) {
2900 ret = -ENOMEM;
2901 goto out;
2902 }
2903
2904 ret = hns3_alloc_desc(ring);
2905 if (ret)
2906 goto out_with_desc_cb;
2907
2908 if (!HNAE3_IS_TX_RING(ring)) {
2909 ret = hns3_alloc_ring_buffers(ring);
2910 if (ret)
2911 goto out_with_desc;
2912 }
2913
2914 return 0;
2915
2916out_with_desc:
2917 hns3_free_desc(ring);
2918out_with_desc_cb:
2919 kfree(ring->desc_cb);
2920 ring->desc_cb = NULL;
2921out:
2922 return ret;
2923}
2924
2925static void hns3_fini_ring(struct hns3_enet_ring *ring)
2926{
2927 hns3_free_desc(ring);
2928 kfree(ring->desc_cb);
2929 ring->desc_cb = NULL;
2930 ring->next_to_clean = 0;
2931 ring->next_to_use = 0;
2932}
2933
1db9b1bf 2934static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 2935{
2936 int bd_size_type;
2937
2938 switch (buf_size) {
2939 case 512:
2940 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2941 break;
2942 case 1024:
2943 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2944 break;
2945 case 2048:
2946 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2947 break;
2948 case 4096:
2949 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2950 break;
2951 default:
2952 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2953 }
2954
2955 return bd_size_type;
2956}
2957
2958static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2959{
2960 dma_addr_t dma = ring->desc_dma_addr;
2961 struct hnae3_queue *q = ring->tqp;
2962
2963 if (!HNAE3_IS_TX_RING(ring)) {
2964 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2965 (u32)dma);
2966 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2967 (u32)((dma >> 31) >> 1));
2968
2969 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2970 hns3_buf_size2type(ring->buf_size));
2971 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2972 ring->desc_num / 8 - 1);
2973
2974 } else {
2975 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2976 (u32)dma);
2977 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2978 (u32)((dma >> 31) >> 1));
2979
2980 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2981 hns3_buf_size2type(ring->buf_size));
2982 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2983 ring->desc_num / 8 - 1);
2984 }
2985}
2986
5668abda 2987int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2988{
2989 struct hnae3_handle *h = priv->ae_handle;
2990 int ring_num = h->kinfo.num_tqps * 2;
2991 int i, j;
2992 int ret;
2993
2994 for (i = 0; i < ring_num; i++) {
2995 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2996 if (ret) {
2997 dev_err(priv->dev,
2998 "Alloc ring memory fail! ret=%d\n", ret);
2999 goto out_when_alloc_ring_memory;
3000 }
3001
3002 hns3_init_ring_hw(priv->ring_data[i].ring);
3003
3004 u64_stats_init(&priv->ring_data[i].ring->syncp);
3005 }
3006
3007 return 0;
3008
3009out_when_alloc_ring_memory:
3010 for (j = i - 1; j >= 0; j--)
ee83f776 3011 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 3012
3013 return -ENOMEM;
3014}
3015
5668abda 3016int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3017{
3018 struct hnae3_handle *h = priv->ae_handle;
3019 int i;
3020
3021 for (i = 0; i < h->kinfo.num_tqps; i++) {
3022 if (h->ae_algo->ops->reset_queue)
3023 h->ae_algo->ops->reset_queue(h, i);
3024
3025 hns3_fini_ring(priv->ring_data[i].ring);
3026 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3027 }
76ad4f0e
S		 3028 return 0;
3029}
3030
3031/* Set mac addr if it is configured. or leave it to the AE driver */
3032static void hns3_init_mac_addr(struct net_device *netdev)
3033{
3034 struct hns3_nic_priv *priv = netdev_priv(netdev);
3035 struct hnae3_handle *h = priv->ae_handle;
3036 u8 mac_addr_temp[ETH_ALEN];
3037
3038 if (h->ae_algo->ops->get_mac_addr) {
3039 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3040 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3041 }
3042
3043 /* Check if the MAC address is valid, if not get a random one */
3044 if (!is_valid_ether_addr(netdev->dev_addr)) {
3045 eth_hw_addr_random(netdev);
3046 dev_warn(priv->dev, "using random MAC address %pM\n",
3047 netdev->dev_addr);
76ad4f0e 3048 }
139e8792
L		 3049
3050 if (h->ae_algo->ops->set_mac_addr)
3051 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
3052
76ad4f0e
S		 3053}
3054
3055static void hns3_nic_set_priv_ops(struct net_device *netdev)
3056{
3057 struct hns3_nic_priv *priv = netdev_priv(netdev);
3058
3059 if ((netdev->features & NETIF_F_TSO) ||
3060 (netdev->features & NETIF_F_TSO6)) {
3061 priv->ops.fill_desc = hns3_fill_desc_tso;
3062 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3063 } else {
3064 priv->ops.fill_desc = hns3_fill_desc;
3065 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3066 }
3067}
3068
3069static int hns3_client_init(struct hnae3_handle *handle)
3070{
3071 struct pci_dev *pdev = handle->pdev;
3072 struct hns3_nic_priv *priv;
3073 struct net_device *netdev;
3074 int ret;
3075
3076 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
678335a1 3077 hns3_get_max_available_channels(handle));
76ad4f0e
S		 3078 if (!netdev)
3079 return -ENOMEM;
3080
3081 priv = netdev_priv(netdev);
3082 priv->dev = &pdev->dev;
3083 priv->netdev = netdev;
3084 priv->ae_handle = handle;
f8fa222c
L		 3085 priv->last_reset_time = jiffies;
3086 priv->reset_level = HNAE3_FUNC_RESET;
3087 priv->tx_timeout_count = 0;
76ad4f0e
S		 3088
3089 handle->kinfo.netdev = netdev;
3090 handle->priv = (void *)priv;
3091
3092 hns3_init_mac_addr(netdev);
3093
3094 hns3_set_default_feature(netdev);
3095
3096 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3097 netdev->priv_flags |= IFF_UNICAST_FLT;
3098 netdev->netdev_ops = &hns3_nic_netdev_ops;
3099 SET_NETDEV_DEV(netdev, &pdev->dev);
3100 hns3_ethtool_set_ops(netdev);
3101 hns3_nic_set_priv_ops(netdev);
3102
3103 /* Carrier off reporting is important to ethtool even BEFORE open */
3104 netif_carrier_off(netdev);
3105
3106 ret = hns3_get_ring_config(priv);
3107 if (ret) {
3108 ret = -ENOMEM;
3109 goto out_get_ring_cfg;
3110 }
3111
dd38c726
YL		 3112 ret = hns3_nic_alloc_vector_data(priv);
3113 if (ret) {
3114 ret = -ENOMEM;
3115 goto out_alloc_vector_data;
3116 }
3117
76ad4f0e
S		 3118 ret = hns3_nic_init_vector_data(priv);
3119 if (ret) {
3120 ret = -ENOMEM;
3121 goto out_init_vector_data;
3122 }
3123
3124 ret = hns3_init_all_ring(priv);
3125 if (ret) {
3126 ret = -ENOMEM;
3127 goto out_init_ring_data;
3128 }
3129
3130 ret = register_netdev(netdev);
3131 if (ret) {
3132 dev_err(priv->dev, "probe register netdev fail!\n");
3133 goto out_reg_netdev_fail;
3134 }
3135
986743db
YL		 3136 hns3_dcbnl_setup(handle);
3137
a8e8b7ff
S		 3138 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3139 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3140
76ad4f0e
S		 3141 return ret;
3142
3143out_reg_netdev_fail:
3144out_init_ring_data:
3145 (void)hns3_nic_uninit_vector_data(priv);
76ad4f0e 3146out_init_vector_data:
dd38c726
YL		 3147 hns3_nic_dealloc_vector_data(priv);
3148out_alloc_vector_data:
3149 priv->ring_data = NULL;
76ad4f0e
S		 3150out_get_ring_cfg:
3151 priv->ae_handle = NULL;
3152 free_netdev(netdev);
3153 return ret;
3154}
3155
3156static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3157{
3158 struct net_device *netdev = handle->kinfo.netdev;
3159 struct hns3_nic_priv *priv = netdev_priv(netdev);
3160 int ret;
3161
3162 if (netdev->reg_state != NETREG_UNINITIALIZED)
3163 unregister_netdev(netdev);
3164
3165 ret = hns3_nic_uninit_vector_data(priv);
3166 if (ret)
3167 netdev_err(netdev, "uninit vector error\n");
3168
dd38c726
YL		 3169 ret = hns3_nic_dealloc_vector_data(priv);
3170 if (ret)
3171 netdev_err(netdev, "dealloc vector error\n");
3172
76ad4f0e
S		 3173 ret = hns3_uninit_all_ring(priv);
3174 if (ret)
3175 netdev_err(netdev, "uninit ring error\n");
3176
ec777890
YL		 3177 hns3_put_ring_config(priv);
3178
76ad4f0e
S		 3179 priv->ring_data = NULL;
3180
3181 free_netdev(netdev);
3182}
3183
3184static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3185{
3186 struct net_device *netdev = handle->kinfo.netdev;
3187
3188 if (!netdev)
3189 return;
3190
3191 if (linkup) {
3192 netif_carrier_on(netdev);
3193 netif_tx_wake_all_queues(netdev);
3194 netdev_info(netdev, "link up\n");
3195 } else {
3196 netif_carrier_off(netdev);
3197 netif_tx_stop_all_queues(netdev);
3198 netdev_info(netdev, "link down\n");
3199 }
3200}
3201
9df8f79a
YL		 3202static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3203{
3204 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3205 struct net_device *ndev = kinfo->netdev;
075cfdd6 3206 bool if_running;
9df8f79a
YL		 3207 int ret;
3208 u8 i;
3209
3210 if (tc > HNAE3_MAX_TC)
3211 return -EINVAL;
3212
3213 if (!ndev)
3214 return -ENODEV;
3215
075cfdd6
CIK		 3216 if_running = netif_running(ndev);
3217
9df8f79a
YL		 3218 ret = netdev_set_num_tc(ndev, tc);
3219 if (ret)
3220 return ret;
3221
3222 if (if_running) {
3223 (void)hns3_nic_net_stop(ndev);
3224 msleep(100);
3225 }
3226
3227 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3228 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3229 if (ret)
3230 goto err_out;
3231
3232 if (tc <= 1) {
3233 netdev_reset_tc(ndev);
3234 goto out;
3235 }
3236
3237 for (i = 0; i < HNAE3_MAX_TC; i++) {
3238 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3239
3240 if (tc_info->enable)
3241 netdev_set_tc_queue(ndev,
3242 tc_info->tc,
3243 tc_info->tqp_count,
3244 tc_info->tqp_offset);
3245 }
3246
3247 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
3248 netdev_set_prio_tc_map(ndev, i,
3249 kinfo->prio_tc[i]);
3250 }
3251
3252out:
3253 ret = hns3_nic_set_real_num_queue(ndev);
3254
3255err_out:
3256 if (if_running)
3257 (void)hns3_nic_net_open(ndev);
3258
3259 return ret;
3260}
3261
bb6b94a8
L		 3262static void hns3_recover_hw_addr(struct net_device *ndev)
3263{
3264 struct netdev_hw_addr_list *list;
3265 struct netdev_hw_addr *ha, *tmp;
3266
3267 /* go through and sync uc_addr entries to the device */
3268 list = &ndev->uc;
3269 list_for_each_entry_safe(ha, tmp, &list->list, list)
3270 hns3_nic_uc_sync(ndev, ha->addr);
3271
3272 /* go through and sync mc_addr entries to the device */
3273 list = &ndev->mc;
3274 list_for_each_entry_safe(ha, tmp, &list->list, list)
3275 hns3_nic_mc_sync(ndev, ha->addr);
3276}
3277
3278static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb)
3279{
3280 dev_kfree_skb_any(skb);
3281}
3282
3283static void hns3_clear_all_ring(struct hnae3_handle *h)
3284{
3285 struct net_device *ndev = h->kinfo.netdev;
3286 struct hns3_nic_priv *priv = netdev_priv(ndev);
3287 u32 i;
3288
3289 for (i = 0; i < h->kinfo.num_tqps; i++) {
3290 struct netdev_queue *dev_queue;
3291 struct hns3_enet_ring *ring;
3292
3293 ring = priv->ring_data[i].ring;
3294 hns3_clean_tx_ring(ring, ring->desc_num);
3295 dev_queue = netdev_get_tx_queue(ndev,
3296 priv->ring_data[i].queue_index);
3297 netdev_tx_reset_queue(dev_queue);
3298
3299 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3300 hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data);
3301 }
3302}
3303
3304static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3305{
3306 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3307 struct net_device *ndev = kinfo->netdev;
3308
3309 if (!netif_running(ndev))
3310 return -EIO;
3311
3312 return hns3_nic_net_stop(ndev);
3313}
3314
3315static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3316{
3317 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3318 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
3319 int ret = 0;
3320
3321 if (netif_running(kinfo->netdev)) {
3322 ret = hns3_nic_net_up(kinfo->netdev);
3323 if (ret) {
3324 netdev_err(kinfo->netdev,
3325 "hns net up fail, ret=%d!\n", ret);
3326 return ret;
3327 }
3328
3329 priv->last_reset_time = jiffies;
3330 }
3331
3332 return ret;
3333}
3334
3335static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3336{
3337 struct net_device *netdev = handle->kinfo.netdev;
3338 struct hns3_nic_priv *priv = netdev_priv(netdev);
3339 int ret;
3340
3341 priv->reset_level = 1;
3342 hns3_init_mac_addr(netdev);
3343 hns3_nic_set_rx_mode(netdev);
3344 hns3_recover_hw_addr(netdev);
3345
3346 /* Carrier off reporting is important to ethtool even BEFORE open */
3347 netif_carrier_off(netdev);
3348
3349 ret = hns3_get_ring_config(priv);
3350 if (ret)
3351 return ret;
3352
3353 ret = hns3_nic_init_vector_data(priv);
3354 if (ret)
3355 return ret;
3356
3357 ret = hns3_init_all_ring(priv);
3358 if (ret) {
3359 hns3_nic_uninit_vector_data(priv);
3360 priv->ring_data = NULL;
3361 }
3362
3363 return ret;
3364}
3365
3366static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3367{
3368 struct net_device *netdev = handle->kinfo.netdev;
3369 struct hns3_nic_priv *priv = netdev_priv(netdev);
3370 int ret;
3371
3372 hns3_clear_all_ring(handle);
3373
3374 ret = hns3_nic_uninit_vector_data(priv);
3375 if (ret) {
3376 netdev_err(netdev, "uninit vector error\n");
3377 return ret;
3378 }
3379
3380 ret = hns3_uninit_all_ring(priv);
3381 if (ret)
3382 netdev_err(netdev, "uninit ring error\n");
3383
ec777890
YL		 3384 hns3_put_ring_config(priv);
3385
bb6b94a8
L		 3386 priv->ring_data = NULL;
3387
3388 return ret;
3389}
3390
3391static int hns3_reset_notify(struct hnae3_handle *handle,
3392 enum hnae3_reset_notify_type type)
3393{
3394 int ret = 0;
3395
3396 switch (type) {
3397 case HNAE3_UP_CLIENT:
e1586241
SM		 3398 ret = hns3_reset_notify_up_enet(handle);
3399 break;
bb6b94a8
L		 3400 case HNAE3_DOWN_CLIENT:
3401 ret = hns3_reset_notify_down_enet(handle);
3402 break;
3403 case HNAE3_INIT_CLIENT:
3404 ret = hns3_reset_notify_init_enet(handle);
3405 break;
3406 case HNAE3_UNINIT_CLIENT:
3407 ret = hns3_reset_notify_uninit_enet(handle);
3408 break;
3409 default:
3410 break;
3411 }
3412
3413 return ret;
3414}
3415
7a242b23
YL		 3416static void hns3_restore_coal(struct hns3_nic_priv *priv,
3417 struct hns3_enet_coalesce *tx,
3418 struct hns3_enet_coalesce *rx)
3419{
3420 u16 vector_num = priv->vector_num;
3421 int i;
3422
3423 for (i = 0; i < vector_num; i++) {
3424 memcpy(&priv->tqp_vector[i].tx_group.coal, tx,
3425 sizeof(struct hns3_enet_coalesce));
3426 memcpy(&priv->tqp_vector[i].rx_group.coal, rx,
3427 sizeof(struct hns3_enet_coalesce));
3428 }
3429}
3430
3431static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num,
3432 struct hns3_enet_coalesce *tx,
3433 struct hns3_enet_coalesce *rx)
09f2af64
PL		 3434{
3435 struct hns3_nic_priv *priv = netdev_priv(netdev);
3436 struct hnae3_handle *h = hns3_get_handle(netdev);
3437 int ret;
3438
3439 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3440 if (ret)
3441 return ret;
3442
3443 ret = hns3_get_ring_config(priv);
3444 if (ret)
3445 return ret;
3446
dd38c726
YL		 3447 ret = hns3_nic_alloc_vector_data(priv);
3448 if (ret)
3449 goto err_alloc_vector;
3450
7a242b23
YL		 3451 hns3_restore_coal(priv, tx, rx);
3452
09f2af64
PL		 3453 ret = hns3_nic_init_vector_data(priv);
3454 if (ret)
3455 goto err_uninit_vector;
3456
3457 ret = hns3_init_all_ring(priv);
3458 if (ret)
3459 goto err_put_ring;
3460
3461 return 0;
3462
3463err_put_ring:
3464 hns3_put_ring_config(priv);
3465err_uninit_vector:
3466 hns3_nic_uninit_vector_data(priv);
dd38c726
YL		 3467err_alloc_vector:
3468 hns3_nic_dealloc_vector_data(priv);
09f2af64
PL		 3469 return ret;
3470}
3471
3472static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3473{
3474 return (new_tqp_num / num_tc) * num_tc;
3475}
3476
3477int hns3_set_channels(struct net_device *netdev,
3478 struct ethtool_channels *ch)
3479{
3480 struct hns3_nic_priv *priv = netdev_priv(netdev);
3481 struct hnae3_handle *h = hns3_get_handle(netdev);
3482 struct hnae3_knic_private_info *kinfo = &h->kinfo;
7a242b23 3483 struct hns3_enet_coalesce tx_coal, rx_coal;
09f2af64
PL		 3484 bool if_running = netif_running(netdev);
3485 u32 new_tqp_num = ch->combined_count;
3486 u16 org_tqp_num;
3487 int ret;
3488
3489 if (ch->rx_count || ch->tx_count)
3490 return -EINVAL;
3491
678335a1 3492 if (new_tqp_num > hns3_get_max_available_channels(h) ||
09f2af64
PL		 3493 new_tqp_num < kinfo->num_tc) {
3494 dev_err(&netdev->dev,
3495 "Change tqps fail, the tqp range is from %d to %d",
3496 kinfo->num_tc,
678335a1 3497 hns3_get_max_available_channels(h));
09f2af64
PL		 3498 return -EINVAL;
3499 }
3500
3501 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3502 if (kinfo->num_tqps == new_tqp_num)
3503 return 0;
3504
3505 if (if_running)
3506 dev_close(netdev);
3507
3508 hns3_clear_all_ring(h);
3509
3510 ret = hns3_nic_uninit_vector_data(priv);
3511 if (ret) {
3512 dev_err(&netdev->dev,
3513 "Unbind vector with tqp fail, nothing is changed");
3514 goto open_netdev;
3515 }
3516
7a242b23
YL		 3517 /* Changing the tqp num may also change the vector num,
3518 * ethtool only support setting and querying one coal
3519 * configuation for now, so save the vector 0' coal
3520 * configuation here in order to restore it.
3521 */
3522 memcpy(&tx_coal, &priv->tqp_vector[0].tx_group.coal,
3523 sizeof(struct hns3_enet_coalesce));
3524 memcpy(&rx_coal, &priv->tqp_vector[0].rx_group.coal,
3525 sizeof(struct hns3_enet_coalesce));
3526
dd38c726
YL		 3527 hns3_nic_dealloc_vector_data(priv);
3528
09f2af64 3529 hns3_uninit_all_ring(priv);
ec777890 3530 hns3_put_ring_config(priv);
09f2af64
PL		 3531
3532 org_tqp_num = h->kinfo.num_tqps;
7a242b23 3533 ret = hns3_modify_tqp_num(netdev, new_tqp_num, &tx_coal, &rx_coal);
09f2af64 3534 if (ret) {
7a242b23
YL		 3535 ret = hns3_modify_tqp_num(netdev, org_tqp_num,
3536 &tx_coal, &rx_coal);
09f2af64
PL		 3537 if (ret) {
3538 /* If revert to old tqp failed, fatal error occurred */
3539 dev_err(&netdev->dev,
3540 "Revert to old tqp num fail, ret=%d", ret);
3541 return ret;
3542 }
3543 dev_info(&netdev->dev,
3544 "Change tqp num fail, Revert to old tqp num");
3545 }
3546
3547open_netdev:
3548 if (if_running)
3549 dev_open(netdev);
3550
3551 return ret;
3552}
3553
1db9b1bf 3554static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 3555 .init_instance = hns3_client_init,
3556 .uninit_instance = hns3_client_uninit,
3557 .link_status_change = hns3_link_status_change,
9df8f79a 3558 .setup_tc = hns3_client_setup_tc,
bb6b94a8 3559 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 3560};
3561
3562/* hns3_init_module - Driver registration routine
3563 * hns3_init_module is the first routine called when the driver is
3564 * loaded. All it does is register with the PCI subsystem.
3565 */
3566static int __init hns3_init_module(void)
3567{
3568 int ret;
3569
3570 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3571 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3572
3573 client.type = HNAE3_CLIENT_KNIC;
3574 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3575 hns3_driver_name);
3576
3577 client.ops = &client_ops;
3578
3579 ret = hnae3_register_client(&client);
3580 if (ret)
3581 return ret;
3582
3583 ret = pci_register_driver(&hns3_driver);
3584 if (ret)
3585 hnae3_unregister_client(&client);
3586
3587 return ret;
3588}
3589module_init(hns3_init_module);
3590
3591/* hns3_exit_module - Driver exit cleanup routine
3592 * hns3_exit_module is called just before the driver is removed
3593 * from memory.
3594 */
3595static void __exit hns3_exit_module(void)
3596{
3597 pci_unregister_driver(&hns3_driver);
3598 hnae3_unregister_client(&client);
3599}
3600module_exit(hns3_exit_module);
3601
3602MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3603MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3604MODULE_LICENSE("GPL");
3605MODULE_ALIAS("pci:hns-nic");
Ubuntu Focal Linux kernel mirror