]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: fix the VF queue reset flow error
[mirror_ubuntu-bionic-kernel.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
76ad4f0e
S		 1/*
2 * Copyright (c) 2016~2017 Hisilicon Limited.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/etherdevice.h>
12#include <linux/interrupt.h>
13#include <linux/if_vlan.h>
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/skbuff.h>
19#include <linux/sctp.h>
20#include <linux/vermagic.h>
21#include <net/gre.h>
30d240df 22#include <net/pkt_cls.h>
76ad4f0e
S		 23#include <net/vxlan.h>
24
25#include "hnae3.h"
26#include "hns3_enet.h"
27
1db9b1bf 28static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 29const char hns3_driver_version[] = VERMAGIC_STRING;
30static const char hns3_driver_string[] =
31 "Hisilicon Ethernet Network Driver for Hip08 Family";
32static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33static struct hnae3_client client;
34
35/* hns3_pci_tbl - PCI Device ID Table
36 *
37 * Last entry must be all 0s
38 *
39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40 * Class, Class Mask, private data (not used) }
41 */
42static const struct pci_device_id hns3_pci_tbl[] = {
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
a9c89a3f
SM		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62static irqreturn_t hns3_irq_handle(int irq, void *dev)
63{
64 struct hns3_enet_tqp_vector *tqp_vector = dev;
65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
71static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
72{
73 struct hns3_enet_tqp_vector *tqp_vectors;
74 unsigned int i;
75
76 for (i = 0; i < priv->vector_num; i++) {
77 tqp_vectors = &priv->tqp_vector[i];
78
79 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
80 continue;
81
82 /* release the irq resource */
83 free_irq(tqp_vectors->vector_irq, tqp_vectors);
84 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
85 }
86}
87
88static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 int txrx_int_idx = 0;
92 int rx_int_idx = 0;
93 int tx_int_idx = 0;
94 unsigned int i;
95 int ret;
96
97 for (i = 0; i < priv->vector_num; i++) {
98 tqp_vectors = &priv->tqp_vector[i];
99
100 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
101 continue;
102
103 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
104 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
105 "%s-%s-%d", priv->netdev->name, "TxRx",
106 txrx_int_idx++);
107 txrx_int_idx++;
108 } else if (tqp_vectors->rx_group.ring) {
109 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
110 "%s-%s-%d", priv->netdev->name, "Rx",
111 rx_int_idx++);
112 } else if (tqp_vectors->tx_group.ring) {
113 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
114 "%s-%s-%d", priv->netdev->name, "Tx",
115 tx_int_idx++);
116 } else {
117 /* Skip this unused q_vector */
118 continue;
119 }
120
121 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
122
123 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
124 tqp_vectors->name,
125 tqp_vectors);
126 if (ret) {
127 netdev_err(priv->netdev, "request irq(%d) fail\n",
128 tqp_vectors->vector_irq);
129 return ret;
130 }
131
132 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
133 }
134
135 return 0;
136}
137
138static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
139 u32 mask_en)
140{
141 writel(mask_en, tqp_vector->mask_addr);
142}
143
144static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
145{
146 napi_enable(&tqp_vector->napi);
147
148 /* enable vector */
149 hns3_mask_vector_irq(tqp_vector, 1);
150}
151
152static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
153{
154 /* disable vector */
155 hns3_mask_vector_irq(tqp_vector, 0);
156
157 disable_irq(tqp_vector->vector_irq);
158 napi_disable(&tqp_vector->napi);
159}
160
5acd0356
FL		 161void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
162 u32 rl_value)
76ad4f0e 163{
5acd0356
FL		 164 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
165
76ad4f0e
S		 166 /* this defines the configuration for RL (Interrupt Rate Limiter).
167 * Rl defines rate of interrupts i.e. number of interrupts-per-second
168 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
169 */
5acd0356 170
d420d2de
YL		 171 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
172 !tqp_vector->rx_group.coal.gl_adapt_enable)
5acd0356
FL		 173 /* According to the hardware, the range of rl_reg is
174 * 0-59 and the unit is 4.
175 */
176 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
177
178 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
179}
180
181void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
182 u32 gl_value)
183{
184 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
185
186 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
187}
188
189void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
190 u32 gl_value)
191{
192 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
193
194 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 195}
196
2b27decc
FL		 197static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
198 struct hns3_nic_priv *priv)
76ad4f0e 199{
2b27decc
FL		 200 struct hnae3_handle *h = priv->ae_handle;
201
76ad4f0e
S		 202 /* initialize the configuration for interrupt coalescing.
203 * 1. GL (Interrupt Gap Limiter)
204 * 2. RL (Interrupt Rate Limiter)
205 */
206
2b27decc 207 /* Default: enable interrupt coalescing self-adaptive and GL */
d420d2de
YL		 208 tqp_vector->tx_group.coal.gl_adapt_enable = 1;
209 tqp_vector->rx_group.coal.gl_adapt_enable = 1;
2b27decc 210
d420d2de
YL		 211 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
212 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
2b27decc 213
2b27decc
FL		 214 /* Default: disable RL */
215 h->kinfo.int_rl_setting = 0;
2b27decc 216
d420d2de
YL		 217 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
218 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 219}
220
6cbd6d33
YL		 221static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
222 struct hns3_nic_priv *priv)
223{
224 struct hnae3_handle *h = priv->ae_handle;
225
226 hns3_set_vector_coalesce_tx_gl(tqp_vector,
d420d2de 227 tqp_vector->tx_group.coal.int_gl);
6cbd6d33 228 hns3_set_vector_coalesce_rx_gl(tqp_vector,
d420d2de 229 tqp_vector->rx_group.coal.int_gl);
6cbd6d33
YL		 230 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
231}
232
9df8f79a
YL		 233static int hns3_nic_set_real_num_queue(struct net_device *netdev)
234{
9780cb97 235 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 236 struct hnae3_knic_private_info *kinfo = &h->kinfo;
237 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
238 int ret;
239
240 ret = netif_set_real_num_tx_queues(netdev, queue_size);
241 if (ret) {
242 netdev_err(netdev,
243 "netif_set_real_num_tx_queues fail, ret=%d!\n",
244 ret);
245 return ret;
246 }
247
248 ret = netif_set_real_num_rx_queues(netdev, queue_size);
249 if (ret) {
250 netdev_err(netdev,
251 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
252 return ret;
253 }
254
255 return 0;
256}
257
2d7187ce
PL		 258static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
259{
260 u16 free_tqps, max_rss_size, max_tqps;
261
262 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
263 max_tqps = h->kinfo.num_tc * max_rss_size;
264
265 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
266}
267
76ad4f0e
S		 268static int hns3_nic_net_up(struct net_device *netdev)
269{
270 struct hns3_nic_priv *priv = netdev_priv(netdev);
271 struct hnae3_handle *h = priv->ae_handle;
272 int i, j;
273 int ret;
274
275 /* get irq resource for all vectors */
276 ret = hns3_nic_init_irq(priv);
277 if (ret) {
278 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
279 return ret;
280 }
281
282 /* enable the vectors */
283 for (i = 0; i < priv->vector_num; i++)
284 hns3_vector_enable(&priv->tqp_vector[i]);
285
286 /* start the ae_dev */
287 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
288 if (ret)
289 goto out_start_err;
290
a3083abb
JS		 291 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
292
76ad4f0e
S		 293 return 0;
294
295out_start_err:
296 for (j = i - 1; j >= 0; j--)
297 hns3_vector_disable(&priv->tqp_vector[j]);
298
299 hns3_nic_uninit_irq(priv);
300
301 return ret;
302}
303
304static int hns3_nic_net_open(struct net_device *netdev)
305{
f8fa222c 306 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 307 int ret;
308
309 netif_carrier_off(netdev);
310
9df8f79a
YL		 311 ret = hns3_nic_set_real_num_queue(netdev);
312 if (ret)
76ad4f0e 313 return ret;
76ad4f0e
S		 314
315 ret = hns3_nic_net_up(netdev);
316 if (ret) {
317 netdev_err(netdev,
318 "hns net up fail, ret=%d!\n", ret);
319 return ret;
320 }
321
f8fa222c 322 priv->last_reset_time = jiffies;
76ad4f0e
S		 323 return 0;
324}
325
326static void hns3_nic_net_down(struct net_device *netdev)
327{
328 struct hns3_nic_priv *priv = netdev_priv(netdev);
329 const struct hnae3_ae_ops *ops;
330 int i;
331
a3083abb
JS		 332 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
333 return;
334
76ad4f0e
S		 335 /* stop ae_dev */
336 ops = priv->ae_handle->ae_algo->ops;
337 if (ops->stop)
338 ops->stop(priv->ae_handle);
339
340 /* disable vectors */
341 for (i = 0; i < priv->vector_num; i++)
342 hns3_vector_disable(&priv->tqp_vector[i]);
343
344 /* free irq resources */
345 hns3_nic_uninit_irq(priv);
346}
347
348static int hns3_nic_net_stop(struct net_device *netdev)
349{
350 netif_tx_stop_all_queues(netdev);
351 netif_carrier_off(netdev);
352
353 hns3_nic_net_down(netdev);
354
355 return 0;
356}
357
76ad4f0e
S		 358static int hns3_nic_uc_sync(struct net_device *netdev,
359 const unsigned char *addr)
360{
9780cb97 361 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 362
363 if (h->ae_algo->ops->add_uc_addr)
364 return h->ae_algo->ops->add_uc_addr(h, addr);
365
366 return 0;
367}
368
369static int hns3_nic_uc_unsync(struct net_device *netdev,
370 const unsigned char *addr)
371{
9780cb97 372 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 373
374 if (h->ae_algo->ops->rm_uc_addr)
375 return h->ae_algo->ops->rm_uc_addr(h, addr);
376
377 return 0;
378}
379
380static int hns3_nic_mc_sync(struct net_device *netdev,
381 const unsigned char *addr)
382{
9780cb97 383 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 384
720a8478 385 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 386 return h->ae_algo->ops->add_mc_addr(h, addr);
387
388 return 0;
389}
390
391static int hns3_nic_mc_unsync(struct net_device *netdev,
392 const unsigned char *addr)
393{
9780cb97 394 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 395
720a8478 396 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 397 return h->ae_algo->ops->rm_mc_addr(h, addr);
398
399 return 0;
400}
401
1db9b1bf 402static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 403{
9780cb97 404 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 405
406 if (h->ae_algo->ops->set_promisc_mode) {
407 if (netdev->flags & IFF_PROMISC)
408 h->ae_algo->ops->set_promisc_mode(h, 1);
409 else
410 h->ae_algo->ops->set_promisc_mode(h, 0);
411 }
412 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
413 netdev_err(netdev, "sync uc address fail\n");
414 if (netdev->flags & IFF_MULTICAST)
415 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
416 netdev_err(netdev, "sync mc address fail\n");
417}
418
419static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
420 u16 *mss, u32 *type_cs_vlan_tso)
421{
422 u32 l4_offset, hdr_len;
423 union l3_hdr_info l3;
424 union l4_hdr_info l4;
425 u32 l4_paylen;
426 int ret;
427
428 if (!skb_is_gso(skb))
429 return 0;
430
431 ret = skb_cow_head(skb, 0);
432 if (ret)
433 return ret;
434
435 l3.hdr = skb_network_header(skb);
436 l4.hdr = skb_transport_header(skb);
437
438 /* Software should clear the IPv4's checksum field when tso is
439 * needed.
440 */
441 if (l3.v4->version == 4)
442 l3.v4->check = 0;
443
444 /* tunnel packet.*/
445 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
446 SKB_GSO_GRE_CSUM |
447 SKB_GSO_UDP_TUNNEL |
448 SKB_GSO_UDP_TUNNEL_CSUM)) {
449 if ((!(skb_shinfo(skb)->gso_type &
450 SKB_GSO_PARTIAL)) &&
451 (skb_shinfo(skb)->gso_type &
452 SKB_GSO_UDP_TUNNEL_CSUM)) {
453 /* Software should clear the udp's checksum
454 * field when tso is needed.
455 */
456 l4.udp->check = 0;
457 }
458 /* reset l3&l4 pointers from outer to inner headers */
459 l3.hdr = skb_inner_network_header(skb);
460 l4.hdr = skb_inner_transport_header(skb);
461
462 /* Software should clear the IPv4's checksum field when
463 * tso is needed.
464 */
465 if (l3.v4->version == 4)
466 l3.v4->check = 0;
467 }
468
469 /* normal or tunnel packet*/
470 l4_offset = l4.hdr - skb->data;
471 hdr_len = (l4.tcp->doff * 4) + l4_offset;
472
473 /* remove payload length from inner pseudo checksum when tso*/
474 l4_paylen = skb->len - l4_offset;
475 csum_replace_by_diff(&l4.tcp->check,
476 (__force __wsum)htonl(l4_paylen));
477
478 /* find the txbd field values */
479 *paylen = skb->len - hdr_len;
480 hnae_set_bit(*type_cs_vlan_tso,
481 HNS3_TXD_TSO_B, 1);
482
483 /* get MSS for TSO */
484 *mss = skb_shinfo(skb)->gso_size;
485
486 return 0;
487}
488
1898d4e4
S		 489static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
490 u8 *il4_proto)
76ad4f0e
S		 491{
492 union {
493 struct iphdr *v4;
494 struct ipv6hdr *v6;
495 unsigned char *hdr;
496 } l3;
497 unsigned char *l4_hdr;
498 unsigned char *exthdr;
499 u8 l4_proto_tmp;
500 __be16 frag_off;
501
502 /* find outer header point */
503 l3.hdr = skb_network_header(skb);
504 l4_hdr = skb_inner_transport_header(skb);
505
506 if (skb->protocol == htons(ETH_P_IPV6)) {
507 exthdr = l3.hdr + sizeof(*l3.v6);
508 l4_proto_tmp = l3.v6->nexthdr;
509 if (l4_hdr != exthdr)
510 ipv6_skip_exthdr(skb, exthdr - skb->data,
511 &l4_proto_tmp, &frag_off);
512 } else if (skb->protocol == htons(ETH_P_IP)) {
513 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 514 } else {
515 return -EINVAL;
76ad4f0e
S		 516 }
517
518 *ol4_proto = l4_proto_tmp;
519
520 /* tunnel packet */
521 if (!skb->encapsulation) {
522 *il4_proto = 0;
1898d4e4 523 return 0;
76ad4f0e
S		 524 }
525
526 /* find inner header point */
527 l3.hdr = skb_inner_network_header(skb);
528 l4_hdr = skb_inner_transport_header(skb);
529
530 if (l3.v6->version == 6) {
531 exthdr = l3.hdr + sizeof(*l3.v6);
532 l4_proto_tmp = l3.v6->nexthdr;
533 if (l4_hdr != exthdr)
534 ipv6_skip_exthdr(skb, exthdr - skb->data,
535 &l4_proto_tmp, &frag_off);
536 } else if (l3.v4->version == 4) {
537 l4_proto_tmp = l3.v4->protocol;
538 }
539
540 *il4_proto = l4_proto_tmp;
1898d4e4
S		 541
542 return 0;
76ad4f0e
S		 543}
544
545static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
546 u8 il4_proto, u32 *type_cs_vlan_tso,
547 u32 *ol_type_vlan_len_msec)
548{
549 union {
550 struct iphdr *v4;
551 struct ipv6hdr *v6;
552 unsigned char *hdr;
553 } l3;
554 union {
555 struct tcphdr *tcp;
556 struct udphdr *udp;
557 struct gre_base_hdr *gre;
558 unsigned char *hdr;
559 } l4;
560 unsigned char *l2_hdr;
561 u8 l4_proto = ol4_proto;
562 u32 ol2_len;
563 u32 ol3_len;
564 u32 ol4_len;
565 u32 l2_len;
566 u32 l3_len;
567
568 l3.hdr = skb_network_header(skb);
569 l4.hdr = skb_transport_header(skb);
570
571 /* compute L2 header size for normal packet, defined in 2 Bytes */
572 l2_len = l3.hdr - skb->data;
573 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
574 HNS3_TXD_L2LEN_S, l2_len >> 1);
575
576 /* tunnel packet*/
577 if (skb->encapsulation) {
578 /* compute OL2 header size, defined in 2 Bytes */
579 ol2_len = l2_len;
580 hnae_set_field(*ol_type_vlan_len_msec,
581 HNS3_TXD_L2LEN_M,
582 HNS3_TXD_L2LEN_S, ol2_len >> 1);
583
584 /* compute OL3 header size, defined in 4 Bytes */
585 ol3_len = l4.hdr - l3.hdr;
586 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
587 HNS3_TXD_L3LEN_S, ol3_len >> 2);
588
589 /* MAC in UDP, MAC in GRE (0x6558)*/
590 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
591 /* switch MAC header ptr from outer to inner header.*/
592 l2_hdr = skb_inner_mac_header(skb);
593
594 /* compute OL4 header size, defined in 4 Bytes. */
595 ol4_len = l2_hdr - l4.hdr;
596 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
597 HNS3_TXD_L4LEN_S, ol4_len >> 2);
598
599 /* switch IP header ptr from outer to inner header */
600 l3.hdr = skb_inner_network_header(skb);
601
602 /* compute inner l2 header size, defined in 2 Bytes. */
603 l2_len = l3.hdr - l2_hdr;
604 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
605 HNS3_TXD_L2LEN_S, l2_len >> 1);
606 } else {
607 /* skb packet types not supported by hardware,
608 * txbd len fild doesn't be filled.
609 */
610 return;
611 }
612
613 /* switch L4 header pointer from outer to inner */
614 l4.hdr = skb_inner_transport_header(skb);
615
616 l4_proto = il4_proto;
617 }
618
619 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
620 l3_len = l4.hdr - l3.hdr;
621 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
622 HNS3_TXD_L3LEN_S, l3_len >> 2);
623
624 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
625 switch (l4_proto) {
626 case IPPROTO_TCP:
627 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
628 HNS3_TXD_L4LEN_S, l4.tcp->doff);
629 break;
630 case IPPROTO_SCTP:
631 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
632 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
633 break;
634 case IPPROTO_UDP:
635 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
636 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
637 break;
638 default:
639 /* skb packet types not supported by hardware,
640 * txbd len fild doesn't be filled.
641 */
642 return;
643 }
644}
645
646static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
647 u8 il4_proto, u32 *type_cs_vlan_tso,
648 u32 *ol_type_vlan_len_msec)
649{
650 union {
651 struct iphdr *v4;
652 struct ipv6hdr *v6;
653 unsigned char *hdr;
654 } l3;
655 u32 l4_proto = ol4_proto;
656
657 l3.hdr = skb_network_header(skb);
658
659 /* define OL3 type and tunnel type(OL4).*/
660 if (skb->encapsulation) {
661 /* define outer network header type.*/
662 if (skb->protocol == htons(ETH_P_IP)) {
663 if (skb_is_gso(skb))
664 hnae_set_field(*ol_type_vlan_len_msec,
665 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
666 HNS3_OL3T_IPV4_CSUM);
667 else
668 hnae_set_field(*ol_type_vlan_len_msec,
669 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
670 HNS3_OL3T_IPV4_NO_CSUM);
671
672 } else if (skb->protocol == htons(ETH_P_IPV6)) {
673 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
674 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
675 }
676
677 /* define tunnel type(OL4).*/
678 switch (l4_proto) {
679 case IPPROTO_UDP:
680 hnae_set_field(*ol_type_vlan_len_msec,
681 HNS3_TXD_TUNTYPE_M,
682 HNS3_TXD_TUNTYPE_S,
683 HNS3_TUN_MAC_IN_UDP);
684 break;
685 case IPPROTO_GRE:
686 hnae_set_field(*ol_type_vlan_len_msec,
687 HNS3_TXD_TUNTYPE_M,
688 HNS3_TXD_TUNTYPE_S,
689 HNS3_TUN_NVGRE);
690 break;
691 default:
692 /* drop the skb tunnel packet if hardware don't support,
693 * because hardware can't calculate csum when TSO.
694 */
695 if (skb_is_gso(skb))
696 return -EDOM;
697
698 /* the stack computes the IP header already,
699 * driver calculate l4 checksum when not TSO.
700 */
701 skb_checksum_help(skb);
702 return 0;
703 }
704
705 l3.hdr = skb_inner_network_header(skb);
706 l4_proto = il4_proto;
707 }
708
709 if (l3.v4->version == 4) {
710 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
711 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
712
713 /* the stack computes the IP header already, the only time we
714 * need the hardware to recompute it is in the case of TSO.
715 */
716 if (skb_is_gso(skb))
717 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
718
719 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
720 } else if (l3.v6->version == 6) {
721 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
722 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
723 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
724 }
725
726 switch (l4_proto) {
727 case IPPROTO_TCP:
728 hnae_set_field(*type_cs_vlan_tso,
729 HNS3_TXD_L4T_M,
730 HNS3_TXD_L4T_S,
731 HNS3_L4T_TCP);
732 break;
733 case IPPROTO_UDP:
734 hnae_set_field(*type_cs_vlan_tso,
735 HNS3_TXD_L4T_M,
736 HNS3_TXD_L4T_S,
737 HNS3_L4T_UDP);
738 break;
739 case IPPROTO_SCTP:
740 hnae_set_field(*type_cs_vlan_tso,
741 HNS3_TXD_L4T_M,
742 HNS3_TXD_L4T_S,
743 HNS3_L4T_SCTP);
744 break;
745 default:
746 /* drop the skb tunnel packet if hardware don't support,
747 * because hardware can't calculate csum when TSO.
748 */
749 if (skb_is_gso(skb))
750 return -EDOM;
751
752 /* the stack computes the IP header already,
753 * driver calculate l4 checksum when not TSO.
754 */
755 skb_checksum_help(skb);
756 return 0;
757 }
758
759 return 0;
760}
761
762static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
763{
764 /* Config bd buffer end */
765 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
766 HNS3_TXD_BDTYPE_M, 0);
767 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
768 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
7036d26f 769 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 770}
771
1fdd8dc5
PL		 772static int hns3_fill_desc_vtags(struct sk_buff *skb,
773 struct hns3_enet_ring *tx_ring,
774 u32 *inner_vlan_flag,
775 u32 *out_vlan_flag,
776 u16 *inner_vtag,
777 u16 *out_vtag)
778{
779#define HNS3_TX_VLAN_PRIO_SHIFT 13
780
781 if (skb->protocol == htons(ETH_P_8021Q) &&
782 !(tx_ring->tqp->handle->kinfo.netdev->features &
783 NETIF_F_HW_VLAN_CTAG_TX)) {
784 /* When HW VLAN acceleration is turned off, and the stack
785 * sets the protocol to 802.1q, the driver just need to
786 * set the protocol to the encapsulated ethertype.
787 */
788 skb->protocol = vlan_get_protocol(skb);
789 return 0;
790 }
791
792 if (skb_vlan_tag_present(skb)) {
793 u16 vlan_tag;
794
795 vlan_tag = skb_vlan_tag_get(skb);
796 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
797
798 /* Based on hw strategy, use out_vtag in two layer tag case,
799 * and use inner_vtag in one tag case.
800 */
801 if (skb->protocol == htons(ETH_P_8021Q)) {
802 hnae_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
803 *out_vtag = vlan_tag;
804 } else {
805 hnae_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
806 *inner_vtag = vlan_tag;
807 }
808 } else if (skb->protocol == htons(ETH_P_8021Q)) {
809 struct vlan_ethhdr *vhdr;
810 int rc;
811
812 rc = skb_cow_head(skb, 0);
813 if (rc < 0)
814 return rc;
815 vhdr = (struct vlan_ethhdr *)skb->data;
816 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
817 << HNS3_TX_VLAN_PRIO_SHIFT);
818 }
819
820 skb->protocol = vlan_get_protocol(skb);
821 return 0;
822}
823
76ad4f0e
S		 824static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
825 int size, dma_addr_t dma, int frag_end,
826 enum hns_desc_type type)
827{
828 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
829 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
830 u32 ol_type_vlan_len_msec = 0;
831 u16 bdtp_fe_sc_vld_ra_ri = 0;
832 u32 type_cs_vlan_tso = 0;
833 struct sk_buff *skb;
1fdd8dc5
PL		 834 u16 inner_vtag = 0;
835 u16 out_vtag = 0;
76ad4f0e
S		 836 u32 paylen = 0;
837 u16 mss = 0;
838 __be16 protocol;
839 u8 ol4_proto;
840 u8 il4_proto;
841 int ret;
842
843 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
844 desc_cb->priv = priv;
845 desc_cb->length = size;
846 desc_cb->dma = dma;
847 desc_cb->type = type;
848
849 /* now, fill the descriptor */
850 desc->addr = cpu_to_le64(dma);
851 desc->tx.send_size = cpu_to_le16((u16)size);
852 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
853 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
854
855 if (type == DESC_TYPE_SKB) {
856 skb = (struct sk_buff *)priv;
a90bb9a5 857 paylen = skb->len;
76ad4f0e 858
1fdd8dc5
PL		 859 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
860 &ol_type_vlan_len_msec,
861 &inner_vtag, &out_vtag);
862 if (unlikely(ret))
863 return ret;
864
76ad4f0e
S		 865 if (skb->ip_summed == CHECKSUM_PARTIAL) {
866 skb_reset_mac_len(skb);
867 protocol = skb->protocol;
868
1898d4e4
S		 869 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
870 if (ret)
871 return ret;
76ad4f0e
S		 872 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
873 &type_cs_vlan_tso,
874 &ol_type_vlan_len_msec);
875 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
876 &type_cs_vlan_tso,
877 &ol_type_vlan_len_msec);
878 if (ret)
879 return ret;
880
881 ret = hns3_set_tso(skb, &paylen, &mss,
882 &type_cs_vlan_tso);
883 if (ret)
884 return ret;
885 }
886
887 /* Set txbd */
888 desc->tx.ol_type_vlan_len_msec =
889 cpu_to_le32(ol_type_vlan_len_msec);
890 desc->tx.type_cs_vlan_tso_len =
891 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 892 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 893 desc->tx.mss = cpu_to_le16(mss);
1fdd8dc5
PL		 894 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
895 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
76ad4f0e
S		 896 }
897
898 /* move ring pointer to next.*/
899 ring_ptr_move_fw(ring, next_to_use);
900
901 return 0;
902}
903
904static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
905 int size, dma_addr_t dma, int frag_end,
906 enum hns_desc_type type)
907{
908 unsigned int frag_buf_num;
909 unsigned int k;
910 int sizeoflast;
911 int ret;
912
913 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
914 sizeoflast = size % HNS3_MAX_BD_SIZE;
915 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
916
917 /* When the frag size is bigger than hardware, split this frag */
918 for (k = 0; k < frag_buf_num; k++) {
919 ret = hns3_fill_desc(ring, priv,
920 (k == frag_buf_num - 1) ?
921 sizeoflast : HNS3_MAX_BD_SIZE,
922 dma + HNS3_MAX_BD_SIZE * k,
923 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
924 (type == DESC_TYPE_SKB && !k) ?
925 DESC_TYPE_SKB : DESC_TYPE_PAGE);
926 if (ret)
927 return ret;
928 }
929
930 return 0;
931}
932
933static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
934 struct hns3_enet_ring *ring)
935{
936 struct sk_buff *skb = *out_skb;
937 struct skb_frag_struct *frag;
938 int bdnum_for_frag;
939 int frag_num;
940 int buf_num;
941 int size;
942 int i;
943
944 size = skb_headlen(skb);
945 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
946
947 frag_num = skb_shinfo(skb)->nr_frags;
948 for (i = 0; i < frag_num; i++) {
949 frag = &skb_shinfo(skb)->frags[i];
950 size = skb_frag_size(frag);
951 bdnum_for_frag =
952 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
953 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
954 return -ENOMEM;
955
956 buf_num += bdnum_for_frag;
957 }
958
959 if (buf_num > ring_space(ring))
960 return -EBUSY;
961
962 *bnum = buf_num;
963 return 0;
964}
965
966static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
967 struct hns3_enet_ring *ring)
968{
969 struct sk_buff *skb = *out_skb;
970 int buf_num;
971
972 /* No. of segments (plus a header) */
973 buf_num = skb_shinfo(skb)->nr_frags + 1;
974
975 if (buf_num > ring_space(ring))
976 return -EBUSY;
977
978 *bnum = buf_num;
979
980 return 0;
981}
982
983static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
984{
985 struct device *dev = ring_to_dev(ring);
986 unsigned int i;
987
988 for (i = 0; i < ring->desc_num; i++) {
989 /* check if this is where we started */
990 if (ring->next_to_use == next_to_use_orig)
991 break;
992
993 /* unmap the descriptor dma address */
994 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
995 dma_unmap_single(dev,
996 ring->desc_cb[ring->next_to_use].dma,
997 ring->desc_cb[ring->next_to_use].length,
998 DMA_TO_DEVICE);
999 else
1000 dma_unmap_page(dev,
1001 ring->desc_cb[ring->next_to_use].dma,
1002 ring->desc_cb[ring->next_to_use].length,
1003 DMA_TO_DEVICE);
1004
1005 /* rollback one */
1006 ring_ptr_move_bw(ring, next_to_use);
1007 }
1008}
1009
d43e5aca 1010netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 1011{
1012 struct hns3_nic_priv *priv = netdev_priv(netdev);
1013 struct hns3_nic_ring_data *ring_data =
1014 &tx_ring_data(priv, skb->queue_mapping);
1015 struct hns3_enet_ring *ring = ring_data->ring;
1016 struct device *dev = priv->dev;
1017 struct netdev_queue *dev_queue;
1018 struct skb_frag_struct *frag;
1019 int next_to_use_head;
1020 int next_to_use_frag;
1021 dma_addr_t dma;
1022 int buf_num;
1023 int seg_num;
1024 int size;
1025 int ret;
1026 int i;
1027
1028 /* Prefetch the data used later */
1029 prefetch(skb->data);
1030
1031 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1032 case -EBUSY:
1033 u64_stats_update_begin(&ring->syncp);
1034 ring->stats.tx_busy++;
1035 u64_stats_update_end(&ring->syncp);
1036
1037 goto out_net_tx_busy;
1038 case -ENOMEM:
1039 u64_stats_update_begin(&ring->syncp);
1040 ring->stats.sw_err_cnt++;
1041 u64_stats_update_end(&ring->syncp);
1042 netdev_err(netdev, "no memory to xmit!\n");
1043
1044 goto out_err_tx_ok;
1045 default:
1046 break;
1047 }
1048
1049 /* No. of segments (plus a header) */
1050 seg_num = skb_shinfo(skb)->nr_frags + 1;
1051 /* Fill the first part */
1052 size = skb_headlen(skb);
1053
1054 next_to_use_head = ring->next_to_use;
1055
1056 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1057 if (dma_mapping_error(dev, dma)) {
1058 netdev_err(netdev, "TX head DMA map failed\n");
1059 ring->stats.sw_err_cnt++;
1060 goto out_err_tx_ok;
1061 }
1062
1063 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
1064 DESC_TYPE_SKB);
1065 if (ret)
1066 goto head_dma_map_err;
1067
1068 next_to_use_frag = ring->next_to_use;
1069 /* Fill the fragments */
1070 for (i = 1; i < seg_num; i++) {
1071 frag = &skb_shinfo(skb)->frags[i - 1];
1072 size = skb_frag_size(frag);
1073 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1074 if (dma_mapping_error(dev, dma)) {
1075 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
1076 ring->stats.sw_err_cnt++;
1077 goto frag_dma_map_err;
1078 }
1079 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
1080 seg_num - 1 == i ? 1 : 0,
1081 DESC_TYPE_PAGE);
1082
1083 if (ret)
1084 goto frag_dma_map_err;
1085 }
1086
1087 /* Complete translate all packets */
1088 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1089 netdev_tx_sent_queue(dev_queue, skb->len);
1090
1091 wmb(); /* Commit all data before submit */
1092
1093 hnae_queue_xmit(ring->tqp, buf_num);
1094
1095 return NETDEV_TX_OK;
1096
1097frag_dma_map_err:
1098 hns_nic_dma_unmap(ring, next_to_use_frag);
1099
1100head_dma_map_err:
1101 hns_nic_dma_unmap(ring, next_to_use_head);
1102
1103out_err_tx_ok:
1104 dev_kfree_skb_any(skb);
1105 return NETDEV_TX_OK;
1106
1107out_net_tx_busy:
1108 netif_stop_subqueue(netdev, ring_data->queue_index);
1109 smp_mb(); /* Commit all data before submit */
1110
1111 return NETDEV_TX_BUSY;
1112}
1113
1114static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1115{
9780cb97 1116 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1117 struct sockaddr *mac_addr = p;
1118 int ret;
1119
1120 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1121 return -EADDRNOTAVAIL;
1122
3cbf5e2d 1123 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
76ad4f0e
S		 1124 if (ret) {
1125 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1126 return ret;
1127 }
1128
1129 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1130
1131 return 0;
1132}
1133
1134static int hns3_nic_set_features(struct net_device *netdev,
1135 netdev_features_t features)
1136{
21b6fd34 1137 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 1138 struct hns3_nic_priv *priv = netdev_priv(netdev);
5f9a7732 1139 struct hnae3_handle *h = priv->ae_handle;
5f9a7732 1140 int ret;
76ad4f0e 1141
21b6fd34
JS		 1142 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1143 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1144 priv->ops.fill_desc = hns3_fill_desc_tso;
1145 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1146 } else {
1147 priv->ops.fill_desc = hns3_fill_desc;
1148 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1149 }
76ad4f0e
S		 1150 }
1151
88576b4b
JS		 1152 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1153 h->ae_algo->ops->enable_vlan_filter) {
21b6fd34
JS		 1154 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1155 h->ae_algo->ops->enable_vlan_filter(h, true);
1156 else
1157 h->ae_algo->ops->enable_vlan_filter(h, false);
1158 }
d818396d 1159
88576b4b
JS		 1160 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1161 h->ae_algo->ops->enable_hw_strip_rxvtag) {
5f9a7732
PL		 1162 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1163 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1164 else
1165 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1166
1167 if (ret)
1168 return ret;
1169 }
1170
76ad4f0e
S		 1171 netdev->features = features;
1172 return 0;
1173}
1174
9596f6f0
PL		 1175static void hns3_nic_get_stats64(struct net_device *netdev,
1176 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1177{
1178 struct hns3_nic_priv *priv = netdev_priv(netdev);
1179 int queue_num = priv->ae_handle->kinfo.num_tqps;
7a5d2a39 1180 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1181 struct hns3_enet_ring *ring;
1182 unsigned int start;
1183 unsigned int idx;
1184 u64 tx_bytes = 0;
1185 u64 rx_bytes = 0;
1186 u64 tx_pkts = 0;
1187 u64 rx_pkts = 0;
0a83231f
JS		 1188 u64 tx_drop = 0;
1189 u64 rx_drop = 0;
76ad4f0e 1190
a3083abb
JS		 1191 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1192 return;
1193
7a5d2a39
JS		 1194 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1195
76ad4f0e
S		 1196 for (idx = 0; idx < queue_num; idx++) {
1197 /* fetch the tx stats */
1198 ring = priv->ring_data[idx].ring;
1199 do {
d36d36ce 1200 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1201 tx_bytes += ring->stats.tx_bytes;
1202 tx_pkts += ring->stats.tx_pkts;
0a83231f
JS		 1203 tx_drop += ring->stats.tx_busy;
1204 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1205 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1206
1207 /* fetch the rx stats */
1208 ring = priv->ring_data[idx + queue_num].ring;
1209 do {
d36d36ce 1210 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1211 rx_bytes += ring->stats.rx_bytes;
1212 rx_pkts += ring->stats.rx_pkts;
0a83231f
JS		 1213 rx_drop += ring->stats.non_vld_descs;
1214 rx_drop += ring->stats.err_pkt_len;
1215 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1216 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1217 }
1218
1219 stats->tx_bytes = tx_bytes;
1220 stats->tx_packets = tx_pkts;
1221 stats->rx_bytes = rx_bytes;
1222 stats->rx_packets = rx_pkts;
1223
1224 stats->rx_errors = netdev->stats.rx_errors;
1225 stats->multicast = netdev->stats.multicast;
1226 stats->rx_length_errors = netdev->stats.rx_length_errors;
1227 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1228 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1229
1230 stats->tx_errors = netdev->stats.tx_errors;
0a83231f
JS		 1231 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1232 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1233 stats->collisions = netdev->stats.collisions;
1234 stats->rx_over_errors = netdev->stats.rx_over_errors;
1235 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1236 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1237 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1238 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1239 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1240 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1241 stats->tx_window_errors = netdev->stats.tx_window_errors;
1242 stats->rx_compressed = netdev->stats.rx_compressed;
1243 stats->tx_compressed = netdev->stats.tx_compressed;
1244}
1245
1246static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1247 enum hns3_udp_tnl_type type)
1248{
1249 struct hns3_nic_priv *priv = netdev_priv(netdev);
1250 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1251 struct hnae3_handle *h = priv->ae_handle;
1252
1253 if (udp_tnl->used && udp_tnl->dst_port == port) {
1254 udp_tnl->used++;
1255 return;
1256 }
1257
1258 if (udp_tnl->used) {
1259 netdev_warn(netdev,
1260 "UDP tunnel [%d], port [%d] offload\n", type, port);
1261 return;
1262 }
1263
1264 udp_tnl->dst_port = port;
1265 udp_tnl->used = 1;
1266 /* TBD send command to hardware to add port */
1267 if (h->ae_algo->ops->add_tunnel_udp)
1268 h->ae_algo->ops->add_tunnel_udp(h, port);
1269}
1270
1271static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1272 enum hns3_udp_tnl_type type)
1273{
1274 struct hns3_nic_priv *priv = netdev_priv(netdev);
1275 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1276 struct hnae3_handle *h = priv->ae_handle;
1277
1278 if (!udp_tnl->used || udp_tnl->dst_port != port) {
1279 netdev_warn(netdev,
1280 "Invalid UDP tunnel port %d\n", port);
1281 return;
1282 }
1283
1284 udp_tnl->used--;
1285 if (udp_tnl->used)
1286 return;
1287
1288 udp_tnl->dst_port = 0;
1289 /* TBD send command to hardware to del port */
1290 if (h->ae_algo->ops->del_tunnel_udp)
9537e7cb 1291 h->ae_algo->ops->del_tunnel_udp(h, port);
76ad4f0e
S		 1292}
1293
1294/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1295 * @netdev: This physical ports's netdev
1296 * @ti: Tunnel information
1297 */
1298static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1299 struct udp_tunnel_info *ti)
1300{
1301 u16 port_n = ntohs(ti->port);
1302
1303 switch (ti->type) {
1304 case UDP_TUNNEL_TYPE_VXLAN:
1305 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1306 break;
1307 case UDP_TUNNEL_TYPE_GENEVE:
1308 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1309 break;
1310 default:
1311 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1312 break;
1313 }
1314}
1315
1316static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1317 struct udp_tunnel_info *ti)
1318{
1319 u16 port_n = ntohs(ti->port);
1320
1321 switch (ti->type) {
1322 case UDP_TUNNEL_TYPE_VXLAN:
1323 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1324 break;
1325 case UDP_TUNNEL_TYPE_GENEVE:
1326 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1327 break;
1328 default:
1329 break;
1330 }
1331}
1332
30d240df 1333static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1334{
30d240df 1335 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1336 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1337 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1338 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1339 u8 tc = mqprio_qopt->qopt.num_tc;
1340 u16 mode = mqprio_qopt->mode;
1341 u8 hw = mqprio_qopt->qopt.hw;
1342 bool if_running;
76ad4f0e
S		 1343 unsigned int i;
1344 int ret;
1345
30d240df
YL		 1346 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1347 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1348 return -EOPNOTSUPP;
1349
76ad4f0e
S		 1350 if (tc > HNAE3_MAX_TC)
1351 return -EINVAL;
1352
76ad4f0e
S		 1353 if (!netdev)
1354 return -EINVAL;
1355
30d240df
YL		 1356 if_running = netif_running(netdev);
1357 if (if_running) {
1358 hns3_nic_net_stop(netdev);
1359 msleep(100);
76ad4f0e
S		 1360 }
1361
30d240df
YL		 1362 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1363 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1364 if (ret)
30d240df
YL		 1365 goto out;
1366
1367 if (tc <= 1) {
1368 netdev_reset_tc(netdev);
1369 } else {
1370 ret = netdev_set_num_tc(netdev, tc);
1371 if (ret)
1372 goto out;
1373
1374 for (i = 0; i < HNAE3_MAX_TC; i++) {
1375 if (!kinfo->tc_info[i].enable)
1376 continue;
76ad4f0e 1377
76ad4f0e
S		 1378 netdev_set_tc_queue(netdev,
1379 kinfo->tc_info[i].tc,
1380 kinfo->tc_info[i].tqp_count,
1381 kinfo->tc_info[i].tqp_offset);
30d240df 1382 }
76ad4f0e
S		 1383 }
1384
30d240df
YL		 1385 ret = hns3_nic_set_real_num_queue(netdev);
1386
1387out:
1388 if (if_running)
1389 hns3_nic_net_open(netdev);
1390
1391 return ret;
76ad4f0e
S		 1392}
1393
2572ac53 1394static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1395 void *type_data)
76ad4f0e 1396{
575ed7d3 1397 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1398 return -EOPNOTSUPP;
76ad4f0e 1399
30d240df 1400 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1401}
1402
1403static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1404 __be16 proto, u16 vid)
1405{
9780cb97 1406 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1407 int ret = -EIO;
1408
1409 if (h->ae_algo->ops->set_vlan_filter)
1410 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1411
1412 return ret;
1413}
1414
1415static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1416 __be16 proto, u16 vid)
1417{
9780cb97 1418 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1419 int ret = -EIO;
1420
1421 if (h->ae_algo->ops->set_vlan_filter)
1422 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1423
1424 return ret;
1425}
1426
1427static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1428 u8 qos, __be16 vlan_proto)
1429{
9780cb97 1430 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1431 int ret = -EIO;
1432
1433 if (h->ae_algo->ops->set_vf_vlan_filter)
1434 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1435 qos, vlan_proto);
1436
1437 return ret;
1438}
1439
a8e8b7ff
S		 1440static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1441{
9780cb97 1442 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1443 bool if_running = netif_running(netdev);
1444 int ret;
1445
1446 if (!h->ae_algo->ops->set_mtu)
1447 return -EOPNOTSUPP;
1448
1449 /* if this was called with netdev up then bring netdevice down */
1450 if (if_running) {
1451 (void)hns3_nic_net_stop(netdev);
1452 msleep(100);
1453 }
1454
1455 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1456 if (ret) {
1457 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1458 ret);
1459 return ret;
1460 }
1461
fe6362f9
FL		 1462 netdev->mtu = new_mtu;
1463
a8e8b7ff
S		 1464 /* if the netdev was running earlier, bring it up again */
1465 if (if_running && hns3_nic_net_open(netdev))
1466 ret = -EINVAL;
1467
1468 return ret;
1469}
1470
f8fa222c
L		 1471static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1472{
1473 struct hns3_nic_priv *priv = netdev_priv(ndev);
1474 struct hns3_enet_ring *tx_ring = NULL;
1475 int timeout_queue = 0;
1476 int hw_head, hw_tail;
1477 int i;
1478
1479 /* Find the stopped queue the same way the stack does */
1480 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1481 struct netdev_queue *q;
1482 unsigned long trans_start;
1483
1484 q = netdev_get_tx_queue(ndev, i);
1485 trans_start = q->trans_start;
1486 if (netif_xmit_stopped(q) &&
1487 time_after(jiffies,
1488 (trans_start + ndev->watchdog_timeo))) {
1489 timeout_queue = i;
1490 break;
1491 }
1492 }
1493
1494 if (i == ndev->num_tx_queues) {
1495 netdev_info(ndev,
1496 "no netdev TX timeout queue found, timeout count: %llu\n",
1497 priv->tx_timeout_count);
1498 return false;
1499 }
1500
1501 tx_ring = priv->ring_data[timeout_queue].ring;
1502
1503 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1504 HNS3_RING_TX_RING_HEAD_REG);
1505 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1506 HNS3_RING_TX_RING_TAIL_REG);
1507 netdev_info(ndev,
1508 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1509 priv->tx_timeout_count,
1510 timeout_queue,
1511 tx_ring->next_to_use,
1512 tx_ring->next_to_clean,
1513 hw_head,
1514 hw_tail,
1515 readl(tx_ring->tqp_vector->mask_addr));
1516
1517 return true;
1518}
1519
1520static void hns3_nic_net_timeout(struct net_device *ndev)
1521{
1522 struct hns3_nic_priv *priv = netdev_priv(ndev);
1523 unsigned long last_reset_time = priv->last_reset_time;
1524 struct hnae3_handle *h = priv->ae_handle;
1525
1526 if (!hns3_get_tx_timeo_queue_info(ndev))
1527 return;
1528
1529 priv->tx_timeout_count++;
1530
1531 /* This timeout is far away enough from last timeout,
1532 * if timeout again,set the reset type to PF reset
1533 */
1534 if (time_after(jiffies, (last_reset_time + 20 * HZ)))
1535 priv->reset_level = HNAE3_FUNC_RESET;
1536
1537 /* Don't do any new action before the next timeout */
1538 else if (time_before(jiffies, (last_reset_time + ndev->watchdog_timeo)))
1539 return;
1540
1541 priv->last_reset_time = jiffies;
1542
1543 if (h->ae_algo->ops->reset_event)
1544 h->ae_algo->ops->reset_event(h, priv->reset_level);
1545
1546 priv->reset_level++;
1547 if (priv->reset_level > HNAE3_GLOBAL_RESET)
1548 priv->reset_level = HNAE3_GLOBAL_RESET;
1549}
1550
76ad4f0e
S		 1551static const struct net_device_ops hns3_nic_netdev_ops = {
1552 .ndo_open = hns3_nic_net_open,
1553 .ndo_stop = hns3_nic_net_stop,
1554 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1555 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1556 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a8e8b7ff 1557 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1558 .ndo_set_features = hns3_nic_set_features,
1559 .ndo_get_stats64 = hns3_nic_get_stats64,
1560 .ndo_setup_tc = hns3_nic_setup_tc,
1561 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
1562 .ndo_udp_tunnel_add = hns3_nic_udp_tunnel_add,
1563 .ndo_udp_tunnel_del = hns3_nic_udp_tunnel_del,
1564 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1565 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1566 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1567};
1568
1569/* hns3_probe - Device initialization routine
1570 * @pdev: PCI device information struct
1571 * @ent: entry in hns3_pci_tbl
1572 *
1573 * hns3_probe initializes a PF identified by a pci_dev structure.
1574 * The OS initialization, configuring of the PF private structure,
1575 * and a hardware reset occur.
1576 *
1577 * Returns 0 on success, negative on failure
1578 */
1579static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1580{
1581 struct hnae3_ae_dev *ae_dev;
1582 int ret;
1583
1584 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1585 GFP_KERNEL);
1586 if (!ae_dev) {
1587 ret = -ENOMEM;
1588 return ret;
1589 }
1590
1591 ae_dev->pdev = pdev;
e92a0843 1592 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 1593 ae_dev->dev_type = HNAE3_DEV_KNIC;
1594 pci_set_drvdata(pdev, ae_dev);
1595
1596 return hnae3_register_ae_dev(ae_dev);
1597}
1598
1599/* hns3_remove - Device removal routine
1600 * @pdev: PCI device information struct
1601 */
1602static void hns3_remove(struct pci_dev *pdev)
1603{
1604 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1605
1606 hnae3_unregister_ae_dev(ae_dev);
1607
1608 devm_kfree(&pdev->dev, ae_dev);
1609
1610 pci_set_drvdata(pdev, NULL);
1611}
1612
1613static struct pci_driver hns3_driver = {
1614 .name = hns3_driver_name,
1615 .id_table = hns3_pci_tbl,
1616 .probe = hns3_probe,
1617 .remove = hns3_remove,
1618};
1619
1620/* set default feature to hns3 */
1621static void hns3_set_default_feature(struct net_device *netdev)
1622{
d818396d
JS		 1623 struct hnae3_handle *h = hns3_get_handle(netdev);
1624
76ad4f0e
S		 1625 netdev->priv_flags |= IFF_UNICAST_FLT;
1626
1627 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1628 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1629 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1630 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1631 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1632
1633 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1634
1635 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1636
1637 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1638 NETIF_F_HW_VLAN_CTAG_FILTER |
5f9a7732 1639 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1640 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1641 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1642 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1643 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1644
1645 netdev->vlan_features |=
1646 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1647 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1648 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1649 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1650 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1651
1652 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
0ba96f84 1653 NETIF_F_HW_VLAN_CTAG_TX |
76ad4f0e
S		 1654 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1655 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1656 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1657 NETIF_F_GSO_UDP_TUNNEL_CSUM;
d818396d
JS		 1658
1659 if (!(h->flags & HNAE3_SUPPORT_VF))
0ba96f84
JS		 1660 netdev->hw_features |=
1661 NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
76ad4f0e
S		 1662}
1663
1664static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1665 struct hns3_desc_cb *cb)
1666{
1667 unsigned int order = hnae_page_order(ring);
1668 struct page *p;
1669
1670 p = dev_alloc_pages(order);
1671 if (!p)
1672 return -ENOMEM;
1673
1674 cb->priv = p;
1675 cb->page_offset = 0;
1676 cb->reuse_flag = 0;
1677 cb->buf = page_address(p);
1678 cb->length = hnae_page_size(ring);
1679 cb->type = DESC_TYPE_PAGE;
1680
76ad4f0e
S		 1681 return 0;
1682}
1683
1684static void hns3_free_buffer(struct hns3_enet_ring *ring,
1685 struct hns3_desc_cb *cb)
1686{
1687 if (cb->type == DESC_TYPE_SKB)
1688 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1689 else if (!HNAE3_IS_TX_RING(ring))
1690 put_page((struct page *)cb->priv);
1691 memset(cb, 0, sizeof(*cb));
1692}
1693
1694static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1695{
1696 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1697 cb->length, ring_to_dma_dir(ring));
1698
1699 if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1700 return -EIO;
1701
1702 return 0;
1703}
1704
1705static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1706 struct hns3_desc_cb *cb)
1707{
1708 if (cb->type == DESC_TYPE_SKB)
1709 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1710 ring_to_dma_dir(ring));
1711 else
1712 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1713 ring_to_dma_dir(ring));
1714}
1715
1716static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1717{
1718 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1719 ring->desc[i].addr = 0;
1720}
1721
1722static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1723{
1724 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1725
1726 if (!ring->desc_cb[i].dma)
1727 return;
1728
1729 hns3_buffer_detach(ring, i);
1730 hns3_free_buffer(ring, cb);
1731}
1732
1733static void hns3_free_buffers(struct hns3_enet_ring *ring)
1734{
1735 int i;
1736
1737 for (i = 0; i < ring->desc_num; i++)
1738 hns3_free_buffer_detach(ring, i);
1739}
1740
1741/* free desc along with its attached buffer */
1742static void hns3_free_desc(struct hns3_enet_ring *ring)
1743{
1744 hns3_free_buffers(ring);
1745
1746 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1747 ring->desc_num * sizeof(ring->desc[0]),
1748 DMA_BIDIRECTIONAL);
1749 ring->desc_dma_addr = 0;
1750 kfree(ring->desc);
1751 ring->desc = NULL;
1752}
1753
1754static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1755{
1756 int size = ring->desc_num * sizeof(ring->desc[0]);
1757
1758 ring->desc = kzalloc(size, GFP_KERNEL);
1759 if (!ring->desc)
1760 return -ENOMEM;
1761
1762 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1763 size, DMA_BIDIRECTIONAL);
1764 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1765 ring->desc_dma_addr = 0;
1766 kfree(ring->desc);
1767 ring->desc = NULL;
1768 return -ENOMEM;
1769 }
1770
1771 return 0;
1772}
1773
1774static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1775 struct hns3_desc_cb *cb)
1776{
1777 int ret;
1778
1779 ret = hns3_alloc_buffer(ring, cb);
1780 if (ret)
1781 goto out;
1782
1783 ret = hns3_map_buffer(ring, cb);
1784 if (ret)
1785 goto out_with_buf;
1786
1787 return 0;
1788
1789out_with_buf:
564883bb 1790 hns3_free_buffer(ring, cb);
76ad4f0e
S		 1791out:
1792 return ret;
1793}
1794
1795static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1796{
1797 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1798
1799 if (ret)
1800 return ret;
1801
1802 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1803
1804 return 0;
1805}
1806
1807/* Allocate memory for raw pkg, and map with dma */
1808static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1809{
1810 int i, j, ret;
1811
1812 for (i = 0; i < ring->desc_num; i++) {
1813 ret = hns3_alloc_buffer_attach(ring, i);
1814 if (ret)
1815 goto out_buffer_fail;
1816 }
1817
1818 return 0;
1819
1820out_buffer_fail:
1821 for (j = i - 1; j >= 0; j--)
1822 hns3_free_buffer_detach(ring, j);
1823 return ret;
1824}
1825
1826/* detach a in-used buffer and replace with a reserved one */
1827static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1828 struct hns3_desc_cb *res_cb)
1829{
b9077428 1830 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 1831 ring->desc_cb[i] = *res_cb;
1832 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1833}
1834
1835static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1836{
1837 ring->desc_cb[i].reuse_flag = 0;
1838 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1839 + ring->desc_cb[i].page_offset);
1840}
1841
1842static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1843 int *pkts)
1844{
1845 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1846
1847 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1848 (*bytes) += desc_cb->length;
1849 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1850 hns3_free_buffer_detach(ring, ring->next_to_clean);
1851
1852 ring_ptr_move_fw(ring, next_to_clean);
1853}
1854
1855static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1856{
1857 int u = ring->next_to_use;
1858 int c = ring->next_to_clean;
1859
1860 if (unlikely(h > ring->desc_num))
1861 return 0;
1862
1863 return u > c ? (h > c && h <= u) : (h > c || h <= u);
1864}
1865
24e750c4 1866bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e
S		 1867{
1868 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1869 struct netdev_queue *dev_queue;
1870 int bytes, pkts;
1871 int head;
1872
1873 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1874 rmb(); /* Make sure head is ready before touch any data */
1875
1876 if (is_ring_empty(ring) || head == ring->next_to_clean)
24e750c4 1877 return true; /* no data to poll */
76ad4f0e
S		 1878
1879 if (!is_valid_clean_head(ring, head)) {
1880 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1881 ring->next_to_use, ring->next_to_clean);
1882
1883 u64_stats_update_begin(&ring->syncp);
1884 ring->stats.io_err_cnt++;
1885 u64_stats_update_end(&ring->syncp);
24e750c4 1886 return true;
76ad4f0e
S		 1887 }
1888
1889 bytes = 0;
1890 pkts = 0;
1891 while (head != ring->next_to_clean && budget) {
1892 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1893 /* Issue prefetch for next Tx descriptor */
1894 prefetch(&ring->desc_cb[ring->next_to_clean]);
1895 budget--;
1896 }
1897
1898 ring->tqp_vector->tx_group.total_bytes += bytes;
1899 ring->tqp_vector->tx_group.total_packets += pkts;
1900
1901 u64_stats_update_begin(&ring->syncp);
1902 ring->stats.tx_bytes += bytes;
1903 ring->stats.tx_pkts += pkts;
1904 u64_stats_update_end(&ring->syncp);
1905
1906 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1907 netdev_tx_completed_queue(dev_queue, pkts, bytes);
1908
1909 if (unlikely(pkts && netif_carrier_ok(netdev) &&
1910 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1911 /* Make sure that anybody stopping the queue after this
1912 * sees the new next_to_clean.
1913 */
1914 smp_mb();
1915 if (netif_tx_queue_stopped(dev_queue)) {
1916 netif_tx_wake_queue(dev_queue);
1917 ring->stats.restart_queue++;
1918 }
1919 }
1920
1921 return !!budget;
1922}
1923
1924static int hns3_desc_unused(struct hns3_enet_ring *ring)
1925{
1926 int ntc = ring->next_to_clean;
1927 int ntu = ring->next_to_use;
1928
1929 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1930}
1931
1932static void
1933hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1934{
1935 struct hns3_desc_cb *desc_cb;
1936 struct hns3_desc_cb res_cbs;
1937 int i, ret;
1938
1939 for (i = 0; i < cleand_count; i++) {
1940 desc_cb = &ring->desc_cb[ring->next_to_use];
1941 if (desc_cb->reuse_flag) {
1942 u64_stats_update_begin(&ring->syncp);
1943 ring->stats.reuse_pg_cnt++;
1944 u64_stats_update_end(&ring->syncp);
1945
1946 hns3_reuse_buffer(ring, ring->next_to_use);
1947 } else {
1948 ret = hns3_reserve_buffer_map(ring, &res_cbs);
1949 if (ret) {
1950 u64_stats_update_begin(&ring->syncp);
1951 ring->stats.sw_err_cnt++;
1952 u64_stats_update_end(&ring->syncp);
1953
1954 netdev_err(ring->tqp->handle->kinfo.netdev,
1955 "hnae reserve buffer map failed.\n");
1956 break;
1957 }
1958 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1959 }
1960
1961 ring_ptr_move_fw(ring, next_to_use);
1962 }
1963
1964 wmb(); /* Make all data has been write before submit */
1965 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1966}
1967
1968/* hns3_nic_get_headlen - determine size of header for LRO/GRO
1969 * @data: pointer to the start of the headers
1970 * @max: total length of section to find headers in
1971 *
1972 * This function is meant to determine the length of headers that will
1973 * be recognized by hardware for LRO, GRO, and RSC offloads. The main
1974 * motivation of doing this is to only perform one pull for IPv4 TCP
1975 * packets so that we can do basic things like calculating the gso_size
1976 * based on the average data per packet.
1977 */
1978static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1979 unsigned int max_size)
1980{
1981 unsigned char *network;
1982 u8 hlen;
1983
1984 /* This should never happen, but better safe than sorry */
1985 if (max_size < ETH_HLEN)
1986 return max_size;
1987
1988 /* Initialize network frame pointer */
1989 network = data;
1990
1991 /* Set first protocol and move network header forward */
1992 network += ETH_HLEN;
1993
1994 /* Handle any vlan tag if present */
1995 if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1996 == HNS3_RX_FLAG_VLAN_PRESENT) {
1997 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1998 return max_size;
1999
2000 network += VLAN_HLEN;
2001 }
2002
2003 /* Handle L3 protocols */
2004 if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
2005 == HNS3_RX_FLAG_L3ID_IPV4) {
2006 if ((typeof(max_size))(network - data) >
2007 (max_size - sizeof(struct iphdr)))
2008 return max_size;
2009
2010 /* Access ihl as a u8 to avoid unaligned access on ia64 */
2011 hlen = (network[0] & 0x0F) << 2;
2012
2013 /* Verify hlen meets minimum size requirements */
2014 if (hlen < sizeof(struct iphdr))
2015 return network - data;
2016
2017 /* Record next protocol if header is present */
2018 } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
2019 == HNS3_RX_FLAG_L3ID_IPV6) {
2020 if ((typeof(max_size))(network - data) >
2021 (max_size - sizeof(struct ipv6hdr)))
2022 return max_size;
2023
2024 /* Record next protocol */
2025 hlen = sizeof(struct ipv6hdr);
2026 } else {
2027 return network - data;
2028 }
2029
2030 /* Relocate pointer to start of L4 header */
2031 network += hlen;
2032
2033 /* Finally sort out TCP/UDP */
2034 if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
2035 == HNS3_RX_FLAG_L4ID_TCP) {
2036 if ((typeof(max_size))(network - data) >
2037 (max_size - sizeof(struct tcphdr)))
2038 return max_size;
2039
2040 /* Access doff as a u8 to avoid unaligned access on ia64 */
2041 hlen = (network[12] & 0xF0) >> 2;
2042
2043 /* Verify hlen meets minimum size requirements */
2044 if (hlen < sizeof(struct tcphdr))
2045 return network - data;
2046
2047 network += hlen;
2048 } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
2049 == HNS3_RX_FLAG_L4ID_UDP) {
2050 if ((typeof(max_size))(network - data) >
2051 (max_size - sizeof(struct udphdr)))
2052 return max_size;
2053
2054 network += sizeof(struct udphdr);
2055 }
2056
2057 /* If everything has gone correctly network should be the
2058 * data section of the packet and will be the end of the header.
2059 * If not then it probably represents the end of the last recognized
2060 * header.
2061 */
2062 if ((typeof(max_size))(network - data) < max_size)
2063 return network - data;
2064 else
2065 return max_size;
2066}
2067
2068static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2069 struct hns3_enet_ring *ring, int pull_len,
2070 struct hns3_desc_cb *desc_cb)
2071{
2072 struct hns3_desc *desc;
2073 int truesize, size;
2074 int last_offset;
2075 bool twobufs;
2076
2077 twobufs = ((PAGE_SIZE < 8192) &&
2078 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2079
2080 desc = &ring->desc[ring->next_to_clean];
2081 size = le16_to_cpu(desc->rx.size);
2082
885a882a
PL		 2083 truesize = hnae_buf_size(ring);
2084
2085 if (!twobufs)
76ad4f0e 2086 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
76ad4f0e
S		 2087
2088 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
885a882a 2089 size - pull_len, truesize);
76ad4f0e
S		 2090
2091 /* Avoid re-using remote pages,flag default unreuse */
2092 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2093 return;
2094
2095 if (twobufs) {
2096 /* If we are only owner of page we can reuse it */
2097 if (likely(page_count(desc_cb->priv) == 1)) {
2098 /* Flip page offset to other buffer */
2099 desc_cb->page_offset ^= truesize;
2100
2101 desc_cb->reuse_flag = 1;
2102 /* bump ref count on page before it is given*/
2103 get_page(desc_cb->priv);
2104 }
2105 return;
2106 }
2107
2108 /* Move offset up to the next cache line */
2109 desc_cb->page_offset += truesize;
2110
2111 if (desc_cb->page_offset <= last_offset) {
2112 desc_cb->reuse_flag = 1;
2113 /* Bump ref count on page before it is given*/
2114 get_page(desc_cb->priv);
2115 }
2116}
2117
2118static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2119 struct hns3_desc *desc)
2120{
2121 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2122 int l3_type, l4_type;
2123 u32 bd_base_info;
2124 int ol4_type;
2125 u32 l234info;
2126
2127 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2128 l234info = le32_to_cpu(desc->rx.l234_info);
2129
2130 skb->ip_summed = CHECKSUM_NONE;
2131
2132 skb_checksum_none_assert(skb);
2133
2134 if (!(netdev->features & NETIF_F_RXCSUM))
2135 return;
2136
2137 /* check if hardware has done checksum */
2138 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2139 return;
2140
2141 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
2142 hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
2143 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2144 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2145 netdev_err(netdev, "L3/L4 error pkt\n");
2146 u64_stats_update_begin(&ring->syncp);
2147 ring->stats.l3l4_csum_err++;
2148 u64_stats_update_end(&ring->syncp);
2149
2150 return;
2151 }
2152
2153 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
2154 HNS3_RXD_L3ID_S);
2155 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
2156 HNS3_RXD_L4ID_S);
2157
2158 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
2159 switch (ol4_type) {
2160 case HNS3_OL4_TYPE_MAC_IN_UDP:
2161 case HNS3_OL4_TYPE_NVGRE:
2162 skb->csum_level = 1;
2163 case HNS3_OL4_TYPE_NO_TUN:
2164 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2165 if (l3_type == HNS3_L3_TYPE_IPV4 ||
2166 (l3_type == HNS3_L3_TYPE_IPV6 &&
2167 (l4_type == HNS3_L4_TYPE_UDP ||
2168 l4_type == HNS3_L4_TYPE_TCP ||
2169 l4_type == HNS3_L4_TYPE_SCTP)))
2170 skb->ip_summed = CHECKSUM_UNNECESSARY;
2171 break;
2172 }
2173}
2174
d43e5aca
YL		 2175static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2176{
2177 napi_gro_receive(&ring->tqp_vector->napi, skb);
2178}
2179
76ad4f0e
S		 2180static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2181 struct sk_buff **out_skb, int *out_bnum)
2182{
2183 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2184 struct hns3_desc_cb *desc_cb;
2185 struct hns3_desc *desc;
2186 struct sk_buff *skb;
2187 unsigned char *va;
2188 u32 bd_base_info;
2189 int pull_len;
2190 u32 l234info;
2191 int length;
2192 int bnum;
2193
2194 desc = &ring->desc[ring->next_to_clean];
2195 desc_cb = &ring->desc_cb[ring->next_to_clean];
2196
2197 prefetch(desc);
2198
2199 length = le16_to_cpu(desc->rx.pkt_len);
2200 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2201 l234info = le32_to_cpu(desc->rx.l234_info);
2202
2203 /* Check valid BD */
2204 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2205 return -EFAULT;
2206
2207 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2208
2209 /* Prefetch first cache line of first page
2210 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2211 * line size is 64B so need to prefetch twice to make it 128B. But in
2212 * actual we can have greater size of caches with 128B Level 1 cache
2213 * lines. In such a case, single fetch would suffice to cache in the
2214 * relevant part of the header.
2215 */
2216 prefetch(va);
2217#if L1_CACHE_BYTES < 128
2218 prefetch(va + L1_CACHE_BYTES);
2219#endif
2220
2221 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2222 HNS3_RX_HEAD_SIZE);
2223 if (unlikely(!skb)) {
2224 netdev_err(netdev, "alloc rx skb fail\n");
2225
2226 u64_stats_update_begin(&ring->syncp);
2227 ring->stats.sw_err_cnt++;
2228 u64_stats_update_end(&ring->syncp);
2229
2230 return -ENOMEM;
2231 }
2232
2233 prefetchw(skb->data);
2234
1fdd8dc5
PL		 2235 /* Based on hw strategy, the tag offloaded will be stored at
2236 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2237 * in one layer tag case.
2238 */
2239 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2240 u16 vlan_tag;
2241
2242 vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2243 if (!(vlan_tag & VLAN_VID_MASK))
2244 vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2245 if (vlan_tag & VLAN_VID_MASK)
2246 __vlan_hwaccel_put_tag(skb,
2247 htons(ETH_P_8021Q),
2248 vlan_tag);
2249 }
2250
76ad4f0e
S		 2251 bnum = 1;
2252 if (length <= HNS3_RX_HEAD_SIZE) {
2253 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2254
2255 /* We can reuse buffer as-is, just make sure it is local */
2256 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2257 desc_cb->reuse_flag = 1;
2258 else /* This page cannot be reused so discard it */
2259 put_page(desc_cb->priv);
2260
2261 ring_ptr_move_fw(ring, next_to_clean);
2262 } else {
2263 u64_stats_update_begin(&ring->syncp);
2264 ring->stats.seg_pkt_cnt++;
2265 u64_stats_update_end(&ring->syncp);
2266
2267 pull_len = hns3_nic_get_headlen(va, l234info,
2268 HNS3_RX_HEAD_SIZE);
2269 memcpy(__skb_put(skb, pull_len), va,
2270 ALIGN(pull_len, sizeof(long)));
2271
2272 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2273 ring_ptr_move_fw(ring, next_to_clean);
2274
2275 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2276 desc = &ring->desc[ring->next_to_clean];
2277 desc_cb = &ring->desc_cb[ring->next_to_clean];
2278 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2279 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2280 ring_ptr_move_fw(ring, next_to_clean);
2281 bnum++;
2282 }
2283 }
2284
2285 *out_bnum = bnum;
2286
2287 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2288 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2289 ((u64 *)desc)[0], ((u64 *)desc)[1]);
2290 u64_stats_update_begin(&ring->syncp);
2291 ring->stats.non_vld_descs++;
2292 u64_stats_update_end(&ring->syncp);
2293
2294 dev_kfree_skb_any(skb);
2295 return -EINVAL;
2296 }
2297
2298 if (unlikely((!desc->rx.pkt_len) ||
2299 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2300 netdev_err(netdev, "truncated pkt\n");
2301 u64_stats_update_begin(&ring->syncp);
2302 ring->stats.err_pkt_len++;
2303 u64_stats_update_end(&ring->syncp);
2304
2305 dev_kfree_skb_any(skb);
2306 return -EFAULT;
2307 }
2308
2309 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2310 netdev_err(netdev, "L2 error pkt\n");
2311 u64_stats_update_begin(&ring->syncp);
2312 ring->stats.l2_err++;
2313 u64_stats_update_end(&ring->syncp);
2314
2315 dev_kfree_skb_any(skb);
2316 return -EFAULT;
2317 }
2318
2319 u64_stats_update_begin(&ring->syncp);
2320 ring->stats.rx_pkts++;
2321 ring->stats.rx_bytes += skb->len;
2322 u64_stats_update_end(&ring->syncp);
2323
2324 ring->tqp_vector->rx_group.total_bytes += skb->len;
2325
2326 hns3_rx_checksum(ring, skb, desc);
2327 return 0;
2328}
2329
d43e5aca
YL		 2330int hns3_clean_rx_ring(
2331 struct hns3_enet_ring *ring, int budget,
2332 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2333{
2334#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2335 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2336 int recv_pkts, recv_bds, clean_count, err;
2337 int unused_count = hns3_desc_unused(ring);
2338 struct sk_buff *skb = NULL;
2339 int num, bnum = 0;
2340
2341 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2342 rmb(); /* Make sure num taken effect before the other data is touched */
2343
2344 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2345 num -= unused_count;
2346
2347 while (recv_pkts < budget && recv_bds < num) {
2348 /* Reuse or realloc buffers */
2349 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2350 hns3_nic_alloc_rx_buffers(ring,
2351 clean_count + unused_count);
2352 clean_count = 0;
2353 unused_count = hns3_desc_unused(ring);
2354 }
2355
2356 /* Poll one pkt */
2357 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2358 if (unlikely(!skb)) /* This fault cannot be repaired */
2359 goto out;
2360
2361 recv_bds += bnum;
2362 clean_count += bnum;
2363 if (unlikely(err)) { /* Do jump the err */
2364 recv_pkts++;
2365 continue;
2366 }
2367
2368 /* Do update ip stack process */
2369 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2370 rx_fn(ring, skb);
76ad4f0e
S		 2371
2372 recv_pkts++;
2373 }
2374
2375out:
2376 /* Make all data has been write before submit */
2377 if (clean_count + unused_count > 0)
2378 hns3_nic_alloc_rx_buffers(ring,
2379 clean_count + unused_count);
2380
2381 return recv_pkts;
2382}
2383
2384static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2385{
2386#define HNS3_RX_ULTRA_PACKET_RATE 40000
2387 enum hns3_flow_level_range new_flow_level;
2388 struct hns3_enet_tqp_vector *tqp_vector;
2389 int packets_per_secs;
2390 int bytes_per_usecs;
2391 u16 new_int_gl;
2392 int usecs;
2393
d420d2de 2394 if (!ring_group->coal.int_gl)
76ad4f0e
S		 2395 return false;
2396
2397 if (ring_group->total_packets == 0) {
d420d2de
YL		 2398 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2399 ring_group->coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 2400 return true;
2401 }
2402
2403 /* Simple throttlerate management
2404 * 0-10MB/s lower (50000 ints/s)
2405 * 10-20MB/s middle (20000 ints/s)
2406 * 20-1249MB/s high (18000 ints/s)
2407 * > 40000pps ultra (8000 ints/s)
2408 */
d420d2de
YL		 2409 new_flow_level = ring_group->coal.flow_level;
2410 new_int_gl = ring_group->coal.int_gl;
76ad4f0e 2411 tqp_vector = ring_group->ring->tqp_vector;
d420d2de 2412 usecs = (ring_group->coal.int_gl << 1);
76ad4f0e
S		 2413 bytes_per_usecs = ring_group->total_bytes / usecs;
2414 /* 1000000 microseconds */
2415 packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2416
2417 switch (new_flow_level) {
2418 case HNS3_FLOW_LOW:
2419 if (bytes_per_usecs > 10)
2420 new_flow_level = HNS3_FLOW_MID;
2421 break;
2422 case HNS3_FLOW_MID:
2423 if (bytes_per_usecs > 20)
2424 new_flow_level = HNS3_FLOW_HIGH;
2425 else if (bytes_per_usecs <= 10)
2426 new_flow_level = HNS3_FLOW_LOW;
2427 break;
2428 case HNS3_FLOW_HIGH:
2429 case HNS3_FLOW_ULTRA:
2430 default:
2431 if (bytes_per_usecs <= 20)
2432 new_flow_level = HNS3_FLOW_MID;
2433 break;
2434 }
2435
2436 if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2437 (&tqp_vector->rx_group == ring_group))
2438 new_flow_level = HNS3_FLOW_ULTRA;
2439
2440 switch (new_flow_level) {
2441 case HNS3_FLOW_LOW:
2442 new_int_gl = HNS3_INT_GL_50K;
2443 break;
2444 case HNS3_FLOW_MID:
2445 new_int_gl = HNS3_INT_GL_20K;
2446 break;
2447 case HNS3_FLOW_HIGH:
2448 new_int_gl = HNS3_INT_GL_18K;
2449 break;
2450 case HNS3_FLOW_ULTRA:
2451 new_int_gl = HNS3_INT_GL_8K;
2452 break;
2453 default:
2454 break;
2455 }
2456
2457 ring_group->total_bytes = 0;
2458 ring_group->total_packets = 0;
d420d2de
YL		 2459 ring_group->coal.flow_level = new_flow_level;
2460 if (new_int_gl != ring_group->coal.int_gl) {
2461 ring_group->coal.int_gl = new_int_gl;
76ad4f0e
S		 2462 return true;
2463 }
2464 return false;
2465}
2466
2467static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2468{
dc114fce
FL		 2469 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2470 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2471 bool rx_update, tx_update;
2472
d420d2de 2473 if (rx_group->coal.gl_adapt_enable) {
dc114fce
FL		 2474 rx_update = hns3_get_new_int_gl(rx_group);
2475 if (rx_update)
2476 hns3_set_vector_coalesce_rx_gl(tqp_vector,
d420d2de 2477 rx_group->coal.int_gl);
dc114fce
FL		 2478 }
2479
d420d2de 2480 if (tx_group->coal.gl_adapt_enable) {
dc114fce
FL		 2481 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2482 if (tx_update)
2483 hns3_set_vector_coalesce_tx_gl(tqp_vector,
d420d2de 2484 tx_group->coal.int_gl);
76ad4f0e
S		 2485 }
2486}
2487
2488static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2489{
2490 struct hns3_enet_ring *ring;
2491 int rx_pkt_total = 0;
2492
2493 struct hns3_enet_tqp_vector *tqp_vector =
2494 container_of(napi, struct hns3_enet_tqp_vector, napi);
2495 bool clean_complete = true;
2496 int rx_budget;
2497
2498 /* Since the actual Tx work is minimal, we can give the Tx a larger
2499 * budget and be more aggressive about cleaning up the Tx descriptors.
2500 */
2501 hns3_for_each_ring(ring, tqp_vector->tx_group) {
2502 if (!hns3_clean_tx_ring(ring, budget))
2503 clean_complete = false;
2504 }
2505
2506 /* make sure rx ring budget not smaller than 1 */
2507 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2508
2509 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2510 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2511 hns3_rx_skb);
76ad4f0e
S		 2512
2513 if (rx_cleaned >= rx_budget)
2514 clean_complete = false;
2515
2516 rx_pkt_total += rx_cleaned;
2517 }
2518
2519 tqp_vector->rx_group.total_packets += rx_pkt_total;
2520
2521 if (!clean_complete)
2522 return budget;
2523
2524 napi_complete(napi);
2525 hns3_update_new_int_gl(tqp_vector);
2526 hns3_mask_vector_irq(tqp_vector, 1);
2527
2528 return rx_pkt_total;
2529}
2530
2531static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2532 struct hnae3_ring_chain_node *head)
2533{
2534 struct pci_dev *pdev = tqp_vector->handle->pdev;
2535 struct hnae3_ring_chain_node *cur_chain = head;
2536 struct hnae3_ring_chain_node *chain;
2537 struct hns3_enet_ring *tx_ring;
2538 struct hns3_enet_ring *rx_ring;
2539
2540 tx_ring = tqp_vector->tx_group.ring;
2541 if (tx_ring) {
2542 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2543 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2544 HNAE3_RING_TYPE_TX);
f230c6c5
FL		 2545 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2546 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 2547
2548 cur_chain->next = NULL;
2549
2550 while (tx_ring->next) {
2551 tx_ring = tx_ring->next;
2552
2553 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2554 GFP_KERNEL);
2555 if (!chain)
2556 return -ENOMEM;
2557
2558 cur_chain->next = chain;
2559 chain->tqp_index = tx_ring->tqp->tqp_index;
2560 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2561 HNAE3_RING_TYPE_TX);
f230c6c5
FL		 2562 hnae_set_field(chain->int_gl_idx,
2563 HNAE3_RING_GL_IDX_M,
2564 HNAE3_RING_GL_IDX_S,
2565 HNAE3_RING_GL_TX);
76ad4f0e
S		 2566
2567 cur_chain = chain;
2568 }
2569 }
2570
2571 rx_ring = tqp_vector->rx_group.ring;
2572 if (!tx_ring && rx_ring) {
2573 cur_chain->next = NULL;
2574 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2575 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2576 HNAE3_RING_TYPE_RX);
f230c6c5
FL		 2577 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2578 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 2579
2580 rx_ring = rx_ring->next;
2581 }
2582
2583 while (rx_ring) {
2584 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2585 if (!chain)
2586 return -ENOMEM;
2587
2588 cur_chain->next = chain;
2589 chain->tqp_index = rx_ring->tqp->tqp_index;
2590 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2591 HNAE3_RING_TYPE_RX);
f230c6c5
FL		 2592 hnae_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2593 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2594
76ad4f0e
S		 2595 cur_chain = chain;
2596
2597 rx_ring = rx_ring->next;
2598 }
2599
2600 return 0;
2601}
2602
2603static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2604 struct hnae3_ring_chain_node *head)
2605{
2606 struct pci_dev *pdev = tqp_vector->handle->pdev;
2607 struct hnae3_ring_chain_node *chain_tmp, *chain;
2608
2609 chain = head->next;
2610
2611 while (chain) {
2612 chain_tmp = chain->next;
2613 devm_kfree(&pdev->dev, chain);
2614 chain = chain_tmp;
2615 }
2616}
2617
2618static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2619 struct hns3_enet_ring *ring)
2620{
2621 ring->next = group->ring;
2622 group->ring = ring;
2623
2624 group->count++;
2625}
2626
2627static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2628{
2629 struct hnae3_ring_chain_node vector_ring_chain;
2630 struct hnae3_handle *h = priv->ae_handle;
2631 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 2632 int ret = 0;
2633 u16 i;
2634
6cbd6d33
YL		 2635 for (i = 0; i < priv->vector_num; i++) {
2636 tqp_vector = &priv->tqp_vector[i];
2637 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2638 tqp_vector->num_tqps = 0;
2639 }
76ad4f0e 2640
6cbd6d33
YL		 2641 for (i = 0; i < h->kinfo.num_tqps; i++) {
2642 u16 vector_i = i % priv->vector_num;
2643 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 2644
2645 tqp_vector = &priv->tqp_vector[vector_i];
2646
2647 hns3_add_ring_to_group(&tqp_vector->tx_group,
2648 priv->ring_data[i].ring);
2649
2650 hns3_add_ring_to_group(&tqp_vector->rx_group,
2651 priv->ring_data[i + tqp_num].ring);
2652
76ad4f0e
S		 2653 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2654 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
6cbd6d33 2655 tqp_vector->num_tqps++;
76ad4f0e
S		 2656 }
2657
6cbd6d33 2658 for (i = 0; i < priv->vector_num; i++) {
76ad4f0e
S		 2659 tqp_vector = &priv->tqp_vector[i];
2660
2661 tqp_vector->rx_group.total_bytes = 0;
2662 tqp_vector->rx_group.total_packets = 0;
2663 tqp_vector->tx_group.total_bytes = 0;
2664 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 2665 tqp_vector->handle = h;
2666
2667 ret = hns3_get_vector_ring_chain(tqp_vector,
2668 &vector_ring_chain);
2669 if (ret)
6cbd6d33 2670 return ret;
76ad4f0e
S		 2671
2672 ret = h->ae_algo->ops->map_ring_to_vector(h,
2673 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 2674
2675 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2676
6cbd6d33
YL		 2677 if (ret)
2678 return ret;
2679
76ad4f0e
S		 2680 netif_napi_add(priv->netdev, &tqp_vector->napi,
2681 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2682 }
2683
6cbd6d33
YL		 2684 return 0;
2685}
2686
2687static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2688{
2689 struct hnae3_handle *h = priv->ae_handle;
2690 struct hns3_enet_tqp_vector *tqp_vector;
2691 struct hnae3_vector_info *vector;
2692 struct pci_dev *pdev = h->pdev;
2693 u16 tqp_num = h->kinfo.num_tqps;
2694 u16 vector_num;
2695 int ret = 0;
2696 u16 i;
2697
2698 /* RSS size, cpu online and vector_num should be the same */
2699 /* Should consider 2p/4p later */
2700 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2701 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2702 GFP_KERNEL);
2703 if (!vector)
2704 return -ENOMEM;
2705
2706 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2707
2708 priv->vector_num = vector_num;
2709 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2710 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2711 GFP_KERNEL);
2712 if (!priv->tqp_vector) {
2713 ret = -ENOMEM;
2714 goto out;
2715 }
2716
2717 for (i = 0; i < priv->vector_num; i++) {
2718 tqp_vector = &priv->tqp_vector[i];
2719 tqp_vector->idx = i;
2720 tqp_vector->mask_addr = vector[i].io_addr;
2721 tqp_vector->vector_irq = vector[i].vector;
2722 hns3_vector_gl_rl_init(tqp_vector, priv);
2723 }
2724
76ad4f0e
S		 2725out:
2726 devm_kfree(&pdev->dev, vector);
2727 return ret;
2728}
2729
6cbd6d33
YL		 2730static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2731{
2732 group->ring = NULL;
2733 group->count = 0;
2734}
2735
76ad4f0e
S		 2736static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2737{
2738 struct hnae3_ring_chain_node vector_ring_chain;
2739 struct hnae3_handle *h = priv->ae_handle;
2740 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 2741 int i, ret;
2742
2743 for (i = 0; i < priv->vector_num; i++) {
2744 tqp_vector = &priv->tqp_vector[i];
2745
2746 ret = hns3_get_vector_ring_chain(tqp_vector,
2747 &vector_ring_chain);
2748 if (ret)
2749 return ret;
2750
2751 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2752 tqp_vector->vector_irq, &vector_ring_chain);
2753 if (ret)
2754 return ret;
2755
7412200c
YL		 2756 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2757 if (ret)
2758 return ret;
2759
76ad4f0e
S		 2760 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2761
2762 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2763 (void)irq_set_affinity_hint(
2764 priv->tqp_vector[i].vector_irq,
2765 NULL);
ae064e61 2766 free_irq(priv->tqp_vector[i].vector_irq,
2767 &priv->tqp_vector[i]);
76ad4f0e
S		 2768 }
2769
2770 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
6cbd6d33
YL		 2771 hns3_clear_ring_group(&tqp_vector->rx_group);
2772 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 2773 netif_napi_del(&priv->tqp_vector[i].napi);
2774 }
2775
6cbd6d33
YL		 2776 return 0;
2777}
2778
2779static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2780{
2781 struct hnae3_handle *h = priv->ae_handle;
2782 struct pci_dev *pdev = h->pdev;
2783 int i, ret;
2784
2785 for (i = 0; i < priv->vector_num; i++) {
2786 struct hns3_enet_tqp_vector *tqp_vector;
2787
2788 tqp_vector = &priv->tqp_vector[i];
2789 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2790 if (ret)
2791 return ret;
2792 }
76ad4f0e 2793
6cbd6d33 2794 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 2795 return 0;
2796}
2797
2798static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2799 int ring_type)
2800{
2801 struct hns3_nic_ring_data *ring_data = priv->ring_data;
2802 int queue_num = priv->ae_handle->kinfo.num_tqps;
2803 struct pci_dev *pdev = priv->ae_handle->pdev;
2804 struct hns3_enet_ring *ring;
2805
2806 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2807 if (!ring)
2808 return -ENOMEM;
2809
2810 if (ring_type == HNAE3_RING_TYPE_TX) {
2811 ring_data[q->tqp_index].ring = ring;
66b44730 2812 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 2813 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2814 } else {
2815 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 2816 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 2817 ring->io_base = q->io_base;
2818 }
2819
2820 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2821
76ad4f0e
S		 2822 ring->tqp = q;
2823 ring->desc = NULL;
2824 ring->desc_cb = NULL;
2825 ring->dev = priv->dev;
2826 ring->desc_dma_addr = 0;
2827 ring->buf_size = q->buf_size;
2828 ring->desc_num = q->desc_num;
2829 ring->next_to_use = 0;
2830 ring->next_to_clean = 0;
2831
2832 return 0;
2833}
2834
2835static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2836 struct hns3_nic_priv *priv)
2837{
2838 int ret;
2839
2840 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2841 if (ret)
2842 return ret;
2843
2844 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2845 if (ret)
2846 return ret;
2847
2848 return 0;
2849}
2850
2851static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2852{
2853 struct hnae3_handle *h = priv->ae_handle;
2854 struct pci_dev *pdev = h->pdev;
2855 int i, ret;
2856
2857 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2858 sizeof(*priv->ring_data) * 2,
2859 GFP_KERNEL);
2860 if (!priv->ring_data)
2861 return -ENOMEM;
2862
2863 for (i = 0; i < h->kinfo.num_tqps; i++) {
2864 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2865 if (ret)
2866 goto err;
2867 }
2868
2869 return 0;
2870err:
2871 devm_kfree(&pdev->dev, priv->ring_data);
2872 return ret;
2873}
2874
f1f779ce
PL		 2875static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2876{
2877 struct hnae3_handle *h = priv->ae_handle;
2878 int i;
2879
2880 for (i = 0; i < h->kinfo.num_tqps; i++) {
2881 devm_kfree(priv->dev, priv->ring_data[i].ring);
2882 devm_kfree(priv->dev,
2883 priv->ring_data[i + h->kinfo.num_tqps].ring);
2884 }
2885 devm_kfree(priv->dev, priv->ring_data);
2886}
2887
76ad4f0e
S		 2888static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2889{
2890 int ret;
2891
2892 if (ring->desc_num <= 0 || ring->buf_size <= 0)
2893 return -EINVAL;
2894
2895 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2896 GFP_KERNEL);
2897 if (!ring->desc_cb) {
2898 ret = -ENOMEM;
2899 goto out;
2900 }
2901
2902 ret = hns3_alloc_desc(ring);
2903 if (ret)
2904 goto out_with_desc_cb;
2905
2906 if (!HNAE3_IS_TX_RING(ring)) {
2907 ret = hns3_alloc_ring_buffers(ring);
2908 if (ret)
2909 goto out_with_desc;
2910 }
2911
2912 return 0;
2913
2914out_with_desc:
2915 hns3_free_desc(ring);
2916out_with_desc_cb:
2917 kfree(ring->desc_cb);
2918 ring->desc_cb = NULL;
2919out:
2920 return ret;
2921}
2922
2923static void hns3_fini_ring(struct hns3_enet_ring *ring)
2924{
2925 hns3_free_desc(ring);
2926 kfree(ring->desc_cb);
2927 ring->desc_cb = NULL;
2928 ring->next_to_clean = 0;
2929 ring->next_to_use = 0;
2930}
2931
1db9b1bf 2932static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 2933{
2934 int bd_size_type;
2935
2936 switch (buf_size) {
2937 case 512:
2938 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2939 break;
2940 case 1024:
2941 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2942 break;
2943 case 2048:
2944 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2945 break;
2946 case 4096:
2947 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2948 break;
2949 default:
2950 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2951 }
2952
2953 return bd_size_type;
2954}
2955
2956static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2957{
2958 dma_addr_t dma = ring->desc_dma_addr;
2959 struct hnae3_queue *q = ring->tqp;
2960
2961 if (!HNAE3_IS_TX_RING(ring)) {
2962 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2963 (u32)dma);
2964 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2965 (u32)((dma >> 31) >> 1));
2966
2967 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2968 hns3_buf_size2type(ring->buf_size));
2969 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2970 ring->desc_num / 8 - 1);
2971
2972 } else {
2973 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2974 (u32)dma);
2975 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2976 (u32)((dma >> 31) >> 1));
2977
2978 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2979 hns3_buf_size2type(ring->buf_size));
2980 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2981 ring->desc_num / 8 - 1);
2982 }
2983}
2984
5668abda 2985int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2986{
2987 struct hnae3_handle *h = priv->ae_handle;
2988 int ring_num = h->kinfo.num_tqps * 2;
2989 int i, j;
2990 int ret;
2991
2992 for (i = 0; i < ring_num; i++) {
2993 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2994 if (ret) {
2995 dev_err(priv->dev,
2996 "Alloc ring memory fail! ret=%d\n", ret);
2997 goto out_when_alloc_ring_memory;
2998 }
2999
3000 hns3_init_ring_hw(priv->ring_data[i].ring);
3001
3002 u64_stats_init(&priv->ring_data[i].ring->syncp);
3003 }
3004
3005 return 0;
3006
3007out_when_alloc_ring_memory:
3008 for (j = i - 1; j >= 0; j--)
ee83f776 3009 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 3010
3011 return -ENOMEM;
3012}
3013
5668abda 3014int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3015{
3016 struct hnae3_handle *h = priv->ae_handle;
3017 int i;
3018
3019 for (i = 0; i < h->kinfo.num_tqps; i++) {
3020 if (h->ae_algo->ops->reset_queue)
3021 h->ae_algo->ops->reset_queue(h, i);
3022
3023 hns3_fini_ring(priv->ring_data[i].ring);
3024 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3025 }
76ad4f0e
S		 3026 return 0;
3027}
3028
3029/* Set mac addr if it is configured. or leave it to the AE driver */
3030static void hns3_init_mac_addr(struct net_device *netdev)
3031{
3032 struct hns3_nic_priv *priv = netdev_priv(netdev);
3033 struct hnae3_handle *h = priv->ae_handle;
3034 u8 mac_addr_temp[ETH_ALEN];
3035
3036 if (h->ae_algo->ops->get_mac_addr) {
3037 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3038 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3039 }
3040
3041 /* Check if the MAC address is valid, if not get a random one */
3042 if (!is_valid_ether_addr(netdev->dev_addr)) {
3043 eth_hw_addr_random(netdev);
3044 dev_warn(priv->dev, "using random MAC address %pM\n",
3045 netdev->dev_addr);
76ad4f0e 3046 }
139e8792
L		 3047
3048 if (h->ae_algo->ops->set_mac_addr)
3cbf5e2d 3049 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
139e8792 3050
76ad4f0e
S		 3051}
3052
3053static void hns3_nic_set_priv_ops(struct net_device *netdev)
3054{
3055 struct hns3_nic_priv *priv = netdev_priv(netdev);
3056
3057 if ((netdev->features & NETIF_F_TSO) ||
3058 (netdev->features & NETIF_F_TSO6)) {
3059 priv->ops.fill_desc = hns3_fill_desc_tso;
3060 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3061 } else {
3062 priv->ops.fill_desc = hns3_fill_desc;
3063 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3064 }
3065}
3066
3067static int hns3_client_init(struct hnae3_handle *handle)
3068{
3069 struct pci_dev *pdev = handle->pdev;
3070 struct hns3_nic_priv *priv;
3071 struct net_device *netdev;
3072 int ret;
3073
3074 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
2d7187ce 3075 hns3_get_max_available_channels(handle));
76ad4f0e
S		 3076 if (!netdev)
3077 return -ENOMEM;
3078
3079 priv = netdev_priv(netdev);
3080 priv->dev = &pdev->dev;
3081 priv->netdev = netdev;
3082 priv->ae_handle = handle;
f8fa222c
L		 3083 priv->last_reset_time = jiffies;
3084 priv->reset_level = HNAE3_FUNC_RESET;
3085 priv->tx_timeout_count = 0;
76ad4f0e
S		 3086
3087 handle->kinfo.netdev = netdev;
3088 handle->priv = (void *)priv;
3089
3090 hns3_init_mac_addr(netdev);
3091
3092 hns3_set_default_feature(netdev);
3093
3094 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3095 netdev->priv_flags |= IFF_UNICAST_FLT;
3096 netdev->netdev_ops = &hns3_nic_netdev_ops;
3097 SET_NETDEV_DEV(netdev, &pdev->dev);
3098 hns3_ethtool_set_ops(netdev);
3099 hns3_nic_set_priv_ops(netdev);
3100
3101 /* Carrier off reporting is important to ethtool even BEFORE open */
3102 netif_carrier_off(netdev);
3103
3104 ret = hns3_get_ring_config(priv);
3105 if (ret) {
3106 ret = -ENOMEM;
3107 goto out_get_ring_cfg;
3108 }
3109
6cbd6d33
YL		 3110 ret = hns3_nic_alloc_vector_data(priv);
3111 if (ret) {
3112 ret = -ENOMEM;
3113 goto out_alloc_vector_data;
3114 }
3115
76ad4f0e
S		 3116 ret = hns3_nic_init_vector_data(priv);
3117 if (ret) {
3118 ret = -ENOMEM;
3119 goto out_init_vector_data;
3120 }
3121
3122 ret = hns3_init_all_ring(priv);
3123 if (ret) {
3124 ret = -ENOMEM;
3125 goto out_init_ring_data;
3126 }
3127
3128 ret = register_netdev(netdev);
3129 if (ret) {
3130 dev_err(priv->dev, "probe register netdev fail!\n");
3131 goto out_reg_netdev_fail;
3132 }
3133
986743db
YL		 3134 hns3_dcbnl_setup(handle);
3135
a8e8b7ff
S		 3136 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3137 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3138
76ad4f0e
S		 3139 return ret;
3140
3141out_reg_netdev_fail:
3142out_init_ring_data:
3143 (void)hns3_nic_uninit_vector_data(priv);
76ad4f0e 3144out_init_vector_data:
6cbd6d33
YL		 3145 hns3_nic_dealloc_vector_data(priv);
3146out_alloc_vector_data:
3147 priv->ring_data = NULL;
76ad4f0e
S		 3148out_get_ring_cfg:
3149 priv->ae_handle = NULL;
3150 free_netdev(netdev);
3151 return ret;
3152}
3153
3154static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3155{
3156 struct net_device *netdev = handle->kinfo.netdev;
3157 struct hns3_nic_priv *priv = netdev_priv(netdev);
3158 int ret;
3159
3160 if (netdev->reg_state != NETREG_UNINITIALIZED)
3161 unregister_netdev(netdev);
3162
3163 ret = hns3_nic_uninit_vector_data(priv);
3164 if (ret)
3165 netdev_err(netdev, "uninit vector error\n");
3166
6cbd6d33
YL		 3167 ret = hns3_nic_dealloc_vector_data(priv);
3168 if (ret)
3169 netdev_err(netdev, "dealloc vector error\n");
3170
76ad4f0e
S		 3171 ret = hns3_uninit_all_ring(priv);
3172 if (ret)
3173 netdev_err(netdev, "uninit ring error\n");
3174
a2ddee8c
YL		 3175 hns3_put_ring_config(priv);
3176
76ad4f0e
S		 3177 priv->ring_data = NULL;
3178
3179 free_netdev(netdev);
3180}
3181
3182static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3183{
3184 struct net_device *netdev = handle->kinfo.netdev;
3185
3186 if (!netdev)
3187 return;
3188
3189 if (linkup) {
3190 netif_carrier_on(netdev);
3191 netif_tx_wake_all_queues(netdev);
3192 netdev_info(netdev, "link up\n");
3193 } else {
3194 netif_carrier_off(netdev);
3195 netif_tx_stop_all_queues(netdev);
3196 netdev_info(netdev, "link down\n");
3197 }
3198}
3199
9df8f79a
YL		 3200static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3201{
3202 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3203 struct net_device *ndev = kinfo->netdev;
075cfdd6 3204 bool if_running;
9df8f79a
YL		 3205 int ret;
3206 u8 i;
3207
3208 if (tc > HNAE3_MAX_TC)
3209 return -EINVAL;
3210
3211 if (!ndev)
3212 return -ENODEV;
3213
075cfdd6
CIK		 3214 if_running = netif_running(ndev);
3215
9df8f79a
YL		 3216 ret = netdev_set_num_tc(ndev, tc);
3217 if (ret)
3218 return ret;
3219
3220 if (if_running) {
3221 (void)hns3_nic_net_stop(ndev);
3222 msleep(100);
3223 }
3224
3225 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3226 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3227 if (ret)
3228 goto err_out;
3229
3230 if (tc <= 1) {
3231 netdev_reset_tc(ndev);
3232 goto out;
3233 }
3234
3235 for (i = 0; i < HNAE3_MAX_TC; i++) {
3236 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3237
3238 if (tc_info->enable)
3239 netdev_set_tc_queue(ndev,
3240 tc_info->tc,
3241 tc_info->tqp_count,
3242 tc_info->tqp_offset);
3243 }
3244
3245 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
3246 netdev_set_prio_tc_map(ndev, i,
3247 kinfo->prio_tc[i]);
3248 }
3249
3250out:
3251 ret = hns3_nic_set_real_num_queue(ndev);
3252
3253err_out:
3254 if (if_running)
3255 (void)hns3_nic_net_open(ndev);
3256
3257 return ret;
3258}
3259
bb6b94a8
L		 3260static void hns3_recover_hw_addr(struct net_device *ndev)
3261{
3262 struct netdev_hw_addr_list *list;
3263 struct netdev_hw_addr *ha, *tmp;
3264
3265 /* go through and sync uc_addr entries to the device */
3266 list = &ndev->uc;
3267 list_for_each_entry_safe(ha, tmp, &list->list, list)
3268 hns3_nic_uc_sync(ndev, ha->addr);
3269
3270 /* go through and sync mc_addr entries to the device */
3271 list = &ndev->mc;
3272 list_for_each_entry_safe(ha, tmp, &list->list, list)
3273 hns3_nic_mc_sync(ndev, ha->addr);
3274}
3275
3276static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb)
3277{
3278 dev_kfree_skb_any(skb);
3279}
3280
3281static void hns3_clear_all_ring(struct hnae3_handle *h)
3282{
3283 struct net_device *ndev = h->kinfo.netdev;
3284 struct hns3_nic_priv *priv = netdev_priv(ndev);
3285 u32 i;
3286
3287 for (i = 0; i < h->kinfo.num_tqps; i++) {
3288 struct netdev_queue *dev_queue;
3289 struct hns3_enet_ring *ring;
3290
3291 ring = priv->ring_data[i].ring;
3292 hns3_clean_tx_ring(ring, ring->desc_num);
3293 dev_queue = netdev_get_tx_queue(ndev,
3294 priv->ring_data[i].queue_index);
3295 netdev_tx_reset_queue(dev_queue);
3296
3297 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3298 hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data);
3299 }
3300}
3301
3302static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3303{
3304 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3305 struct net_device *ndev = kinfo->netdev;
3306
3307 if (!netif_running(ndev))
3308 return -EIO;
3309
3310 return hns3_nic_net_stop(ndev);
3311}
3312
3313static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3314{
3315 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3316 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
3317 int ret = 0;
3318
3319 if (netif_running(kinfo->netdev)) {
3320 ret = hns3_nic_net_up(kinfo->netdev);
3321 if (ret) {
3322 netdev_err(kinfo->netdev,
3323 "hns net up fail, ret=%d!\n", ret);
3324 return ret;
3325 }
3326
3327 priv->last_reset_time = jiffies;
3328 }
3329
3330 return ret;
3331}
3332
3333static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3334{
3335 struct net_device *netdev = handle->kinfo.netdev;
3336 struct hns3_nic_priv *priv = netdev_priv(netdev);
3337 int ret;
3338
3339 priv->reset_level = 1;
3340 hns3_init_mac_addr(netdev);
3341 hns3_nic_set_rx_mode(netdev);
3342 hns3_recover_hw_addr(netdev);
3343
3344 /* Carrier off reporting is important to ethtool even BEFORE open */
3345 netif_carrier_off(netdev);
3346
3347 ret = hns3_get_ring_config(priv);
3348 if (ret)
3349 return ret;
3350
3351 ret = hns3_nic_init_vector_data(priv);
3352 if (ret)
3353 return ret;
3354
3355 ret = hns3_init_all_ring(priv);
3356 if (ret) {
3357 hns3_nic_uninit_vector_data(priv);
3358 priv->ring_data = NULL;
3359 }
3360
3361 return ret;
3362}
3363
3364static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3365{
3366 struct net_device *netdev = handle->kinfo.netdev;
3367 struct hns3_nic_priv *priv = netdev_priv(netdev);
3368 int ret;
3369
3370 hns3_clear_all_ring(handle);
3371
3372 ret = hns3_nic_uninit_vector_data(priv);
3373 if (ret) {
3374 netdev_err(netdev, "uninit vector error\n");
3375 return ret;
3376 }
3377
3378 ret = hns3_uninit_all_ring(priv);
3379 if (ret)
3380 netdev_err(netdev, "uninit ring error\n");
3381
a2ddee8c
YL		 3382 hns3_put_ring_config(priv);
3383
bb6b94a8
L		 3384 priv->ring_data = NULL;
3385
3386 return ret;
3387}
3388
3389static int hns3_reset_notify(struct hnae3_handle *handle,
3390 enum hnae3_reset_notify_type type)
3391{
3392 int ret = 0;
3393
3394 switch (type) {
3395 case HNAE3_UP_CLIENT:
741e1778
SM		 3396 ret = hns3_reset_notify_up_enet(handle);
3397 break;
bb6b94a8
L		 3398 case HNAE3_DOWN_CLIENT:
3399 ret = hns3_reset_notify_down_enet(handle);
3400 break;
3401 case HNAE3_INIT_CLIENT:
3402 ret = hns3_reset_notify_init_enet(handle);
3403 break;
3404 case HNAE3_UNINIT_CLIENT:
3405 ret = hns3_reset_notify_uninit_enet(handle);
3406 break;
3407 default:
3408 break;
3409 }
3410
3411 return ret;
3412}
3413
351dad5e
YL		 3414static void hns3_restore_coal(struct hns3_nic_priv *priv,
3415 struct hns3_enet_coalesce *tx,
3416 struct hns3_enet_coalesce *rx)
3417{
3418 u16 vector_num = priv->vector_num;
3419 int i;
3420
3421 for (i = 0; i < vector_num; i++) {
3422 memcpy(&priv->tqp_vector[i].tx_group.coal, tx,
3423 sizeof(struct hns3_enet_coalesce));
3424 memcpy(&priv->tqp_vector[i].rx_group.coal, rx,
3425 sizeof(struct hns3_enet_coalesce));
3426 }
3427}
3428
3429static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num,
3430 struct hns3_enet_coalesce *tx,
3431 struct hns3_enet_coalesce *rx)
f1f779ce
PL		 3432{
3433 struct hns3_nic_priv *priv = netdev_priv(netdev);
3434 struct hnae3_handle *h = hns3_get_handle(netdev);
3435 int ret;
3436
3437 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3438 if (ret)
3439 return ret;
3440
3441 ret = hns3_get_ring_config(priv);
3442 if (ret)
3443 return ret;
3444
6cbd6d33
YL		 3445 ret = hns3_nic_alloc_vector_data(priv);
3446 if (ret)
3447 goto err_alloc_vector;
3448
351dad5e
YL		 3449 hns3_restore_coal(priv, tx, rx);
3450
f1f779ce
PL		 3451 ret = hns3_nic_init_vector_data(priv);
3452 if (ret)
3453 goto err_uninit_vector;
3454
3455 ret = hns3_init_all_ring(priv);
3456 if (ret)
3457 goto err_put_ring;
3458
3459 return 0;
3460
3461err_put_ring:
3462 hns3_put_ring_config(priv);
3463err_uninit_vector:
3464 hns3_nic_uninit_vector_data(priv);
6cbd6d33
YL		 3465err_alloc_vector:
3466 hns3_nic_dealloc_vector_data(priv);
f1f779ce
PL		 3467 return ret;
3468}
3469
3470static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3471{
3472 return (new_tqp_num / num_tc) * num_tc;
3473}
3474
3475int hns3_set_channels(struct net_device *netdev,
3476 struct ethtool_channels *ch)
3477{
3478 struct hns3_nic_priv *priv = netdev_priv(netdev);
3479 struct hnae3_handle *h = hns3_get_handle(netdev);
3480 struct hnae3_knic_private_info *kinfo = &h->kinfo;
351dad5e 3481 struct hns3_enet_coalesce tx_coal, rx_coal;
f1f779ce
PL		 3482 bool if_running = netif_running(netdev);
3483 u32 new_tqp_num = ch->combined_count;
3484 u16 org_tqp_num;
3485 int ret;
3486
3487 if (ch->rx_count || ch->tx_count)
3488 return -EINVAL;
3489
2d7187ce 3490 if (new_tqp_num > hns3_get_max_available_channels(h) ||
f1f779ce
PL		 3491 new_tqp_num < kinfo->num_tc) {
3492 dev_err(&netdev->dev,
3493 "Change tqps fail, the tqp range is from %d to %d",
3494 kinfo->num_tc,
2d7187ce 3495 hns3_get_max_available_channels(h));
f1f779ce
PL		 3496 return -EINVAL;
3497 }
3498
3499 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3500 if (kinfo->num_tqps == new_tqp_num)
3501 return 0;
3502
3503 if (if_running)
41efd6b1 3504 hns3_nic_net_stop(netdev);
f1f779ce
PL		 3505
3506 hns3_clear_all_ring(h);
3507
3508 ret = hns3_nic_uninit_vector_data(priv);
3509 if (ret) {
3510 dev_err(&netdev->dev,
3511 "Unbind vector with tqp fail, nothing is changed");
3512 goto open_netdev;
3513 }
3514
351dad5e
YL		 3515 /* Changing the tqp num may also change the vector num,
3516 * ethtool only support setting and querying one coal
3517 * configuation for now, so save the vector 0' coal
3518 * configuation here in order to restore it.
3519 */
3520 memcpy(&tx_coal, &priv->tqp_vector[0].tx_group.coal,
3521 sizeof(struct hns3_enet_coalesce));
3522 memcpy(&rx_coal, &priv->tqp_vector[0].rx_group.coal,
3523 sizeof(struct hns3_enet_coalesce));
3524
6cbd6d33
YL		 3525 hns3_nic_dealloc_vector_data(priv);
3526
f1f779ce 3527 hns3_uninit_all_ring(priv);
a2ddee8c 3528 hns3_put_ring_config(priv);
f1f779ce
PL		 3529
3530 org_tqp_num = h->kinfo.num_tqps;
351dad5e 3531 ret = hns3_modify_tqp_num(netdev, new_tqp_num, &tx_coal, &rx_coal);
f1f779ce 3532 if (ret) {
351dad5e
YL		 3533 ret = hns3_modify_tqp_num(netdev, org_tqp_num,
3534 &tx_coal, &rx_coal);
f1f779ce
PL		 3535 if (ret) {
3536 /* If revert to old tqp failed, fatal error occurred */
3537 dev_err(&netdev->dev,
3538 "Revert to old tqp num fail, ret=%d", ret);
3539 return ret;
3540 }
3541 dev_info(&netdev->dev,
3542 "Change tqp num fail, Revert to old tqp num");
3543 }
3544
3545open_netdev:
3546 if (if_running)
41efd6b1 3547 hns3_nic_net_open(netdev);
f1f779ce
PL		 3548
3549 return ret;
3550}
3551
1db9b1bf 3552static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 3553 .init_instance = hns3_client_init,
3554 .uninit_instance = hns3_client_uninit,
3555 .link_status_change = hns3_link_status_change,
9df8f79a 3556 .setup_tc = hns3_client_setup_tc,
bb6b94a8 3557 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 3558};
3559
3560/* hns3_init_module - Driver registration routine
3561 * hns3_init_module is the first routine called when the driver is
3562 * loaded. All it does is register with the PCI subsystem.
3563 */
3564static int __init hns3_init_module(void)
3565{
3566 int ret;
3567
3568 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3569 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3570
3571 client.type = HNAE3_CLIENT_KNIC;
3572 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3573 hns3_driver_name);
3574
3575 client.ops = &client_ops;
3576
3577 ret = hnae3_register_client(&client);
3578 if (ret)
3579 return ret;
3580
3581 ret = pci_register_driver(&hns3_driver);
3582 if (ret)
3583 hnae3_unregister_client(&client);
3584
3585 return ret;
3586}
3587module_init(hns3_init_module);
3588
3589/* hns3_exit_module - Driver exit cleanup routine
3590 * hns3_exit_module is called just before the driver is removed
3591 * from memory.
3592 */
3593static void __exit hns3_exit_module(void)
3594{
3595 pci_unregister_driver(&hns3_driver);
3596 hnae3_unregister_client(&client);
3597}
3598module_exit(hns3_exit_module);
3599
3600MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3601MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3602MODULE_LICENSE("GPL");
3603MODULE_ALIAS("pci:hns-nic");
ubuntu-bionic-kernel mirror