]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: devm_kmalloc() -> devm_kmalloc_array()
[mirror_ubuntu-focal-kernel.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
76ad4f0e
S		 1/*
2 * Copyright (c) 2016~2017 Hisilicon Limited.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/etherdevice.h>
12#include <linux/interrupt.h>
13#include <linux/if_vlan.h>
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/skbuff.h>
19#include <linux/sctp.h>
20#include <linux/vermagic.h>
21#include <net/gre.h>
30d240df 22#include <net/pkt_cls.h>
76ad4f0e
S		 23#include <net/vxlan.h>
24
25#include "hnae3.h"
26#include "hns3_enet.h"
27
7b763f3f
FL		 28static void hns3_clear_all_ring(struct hnae3_handle *h);
29static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
30
1db9b1bf 31static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 32const char hns3_driver_version[] = VERMAGIC_STRING;
33static const char hns3_driver_string[] =
34 "Hisilicon Ethernet Network Driver for Hip08 Family";
35static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
36static struct hnae3_client client;
37
38/* hns3_pci_tbl - PCI Device ID Table
39 *
40 * Last entry must be all 0s
41 *
42 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
43 * Class, Class Mask, private data (not used) }
44 */
45static const struct pci_device_id hns3_pci_tbl[] = {
46 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 48 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 49 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 50 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 51 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 52 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 53 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 54 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 55 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 57 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
424eb834
SM		 58 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
59 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
76ad4f0e
S		 60 /* required last entry */
61 {0, }
62};
63MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
64
65static irqreturn_t hns3_irq_handle(int irq, void *dev)
66{
67 struct hns3_enet_tqp_vector *tqp_vector = dev;
68
69 napi_schedule(&tqp_vector->napi);
70
71 return IRQ_HANDLED;
72}
73
74static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
75{
76 struct hns3_enet_tqp_vector *tqp_vectors;
77 unsigned int i;
78
79 for (i = 0; i < priv->vector_num; i++) {
80 tqp_vectors = &priv->tqp_vector[i];
81
82 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
83 continue;
84
85 /* release the irq resource */
86 free_irq(tqp_vectors->vector_irq, tqp_vectors);
87 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
88 }
89}
90
91static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
92{
93 struct hns3_enet_tqp_vector *tqp_vectors;
94 int txrx_int_idx = 0;
95 int rx_int_idx = 0;
96 int tx_int_idx = 0;
97 unsigned int i;
98 int ret;
99
100 for (i = 0; i < priv->vector_num; i++) {
101 tqp_vectors = &priv->tqp_vector[i];
102
103 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
104 continue;
105
106 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
107 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
108 "%s-%s-%d", priv->netdev->name, "TxRx",
109 txrx_int_idx++);
110 txrx_int_idx++;
111 } else if (tqp_vectors->rx_group.ring) {
112 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
113 "%s-%s-%d", priv->netdev->name, "Rx",
114 rx_int_idx++);
115 } else if (tqp_vectors->tx_group.ring) {
116 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
117 "%s-%s-%d", priv->netdev->name, "Tx",
118 tx_int_idx++);
119 } else {
120 /* Skip this unused q_vector */
121 continue;
122 }
123
124 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
125
126 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
127 tqp_vectors->name,
128 tqp_vectors);
129 if (ret) {
130 netdev_err(priv->netdev, "request irq(%d) fail\n",
131 tqp_vectors->vector_irq);
132 return ret;
133 }
134
135 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
136 }
137
138 return 0;
139}
140
141static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
142 u32 mask_en)
143{
144 writel(mask_en, tqp_vector->mask_addr);
145}
146
147static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
148{
149 napi_enable(&tqp_vector->napi);
150
151 /* enable vector */
152 hns3_mask_vector_irq(tqp_vector, 1);
153}
154
155static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
156{
157 /* disable vector */
158 hns3_mask_vector_irq(tqp_vector, 0);
159
160 disable_irq(tqp_vector->vector_irq);
161 napi_disable(&tqp_vector->napi);
162}
163
434776a5
FL		 164void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
165 u32 rl_value)
76ad4f0e 166{
434776a5
FL		 167 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
168
76ad4f0e
S		 169 /* this defines the configuration for RL (Interrupt Rate Limiter).
170 * Rl defines rate of interrupts i.e. number of interrupts-per-second
171 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
172 */
434776a5 173
9bc727a9
YL		 174 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
175 !tqp_vector->rx_group.coal.gl_adapt_enable)
434776a5
FL		 176 /* According to the hardware, the range of rl_reg is
177 * 0-59 and the unit is 4.
178 */
179 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
180
181 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
182}
183
184void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
185 u32 gl_value)
186{
187 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
188
189 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
190}
191
192void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
193 u32 gl_value)
194{
195 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
196
197 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 198}
199
5fd4789a
FL		 200static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
201 struct hns3_nic_priv *priv)
76ad4f0e 202{
5fd4789a
FL		 203 struct hnae3_handle *h = priv->ae_handle;
204
76ad4f0e
S		 205 /* initialize the configuration for interrupt coalescing.
206 * 1. GL (Interrupt Gap Limiter)
207 * 2. RL (Interrupt Rate Limiter)
208 */
209
5fd4789a 210 /* Default: enable interrupt coalescing self-adaptive and GL */
9bc727a9
YL		 211 tqp_vector->tx_group.coal.gl_adapt_enable = 1;
212 tqp_vector->rx_group.coal.gl_adapt_enable = 1;
5fd4789a 213
9bc727a9
YL		 214 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
215 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
5fd4789a 216
5fd4789a
FL		 217 /* Default: disable RL */
218 h->kinfo.int_rl_setting = 0;
5fd4789a 219
cd9d187b 220 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
9bc727a9
YL		 221 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
222 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 223}
224
dd38c726
YL		 225static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
226 struct hns3_nic_priv *priv)
227{
228 struct hnae3_handle *h = priv->ae_handle;
229
230 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 231 tqp_vector->tx_group.coal.int_gl);
dd38c726 232 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 233 tqp_vector->rx_group.coal.int_gl);
dd38c726
YL		 234 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
235}
236
9df8f79a
YL		 237static int hns3_nic_set_real_num_queue(struct net_device *netdev)
238{
9780cb97 239 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 240 struct hnae3_knic_private_info *kinfo = &h->kinfo;
241 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
242 int ret;
243
244 ret = netif_set_real_num_tx_queues(netdev, queue_size);
245 if (ret) {
246 netdev_err(netdev,
247 "netif_set_real_num_tx_queues fail, ret=%d!\n",
248 ret);
249 return ret;
250 }
251
252 ret = netif_set_real_num_rx_queues(netdev, queue_size);
253 if (ret) {
254 netdev_err(netdev,
255 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
256 return ret;
257 }
258
259 return 0;
260}
261
678335a1
PL		 262static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
263{
264 u16 free_tqps, max_rss_size, max_tqps;
265
266 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
267 max_tqps = h->kinfo.num_tc * max_rss_size;
268
269 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
270}
271
76ad4f0e
S		 272static int hns3_nic_net_up(struct net_device *netdev)
273{
274 struct hns3_nic_priv *priv = netdev_priv(netdev);
275 struct hnae3_handle *h = priv->ae_handle;
276 int i, j;
277 int ret;
278
7b763f3f
FL		 279 ret = hns3_nic_reset_all_ring(h);
280 if (ret)
281 return ret;
282
76ad4f0e
S		 283 /* get irq resource for all vectors */
284 ret = hns3_nic_init_irq(priv);
285 if (ret) {
286 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
287 return ret;
288 }
289
290 /* enable the vectors */
291 for (i = 0; i < priv->vector_num; i++)
292 hns3_vector_enable(&priv->tqp_vector[i]);
293
294 /* start the ae_dev */
295 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
296 if (ret)
297 goto out_start_err;
298
b875cc37
JS		 299 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
300
76ad4f0e
S		 301 return 0;
302
303out_start_err:
304 for (j = i - 1; j >= 0; j--)
305 hns3_vector_disable(&priv->tqp_vector[j]);
306
307 hns3_nic_uninit_irq(priv);
308
309 return ret;
310}
311
312static int hns3_nic_net_open(struct net_device *netdev)
313{
f8fa222c 314 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 315 int ret;
316
317 netif_carrier_off(netdev);
318
9df8f79a
YL		 319 ret = hns3_nic_set_real_num_queue(netdev);
320 if (ret)
76ad4f0e 321 return ret;
76ad4f0e
S		 322
323 ret = hns3_nic_net_up(netdev);
324 if (ret) {
325 netdev_err(netdev,
326 "hns net up fail, ret=%d!\n", ret);
327 return ret;
328 }
329
6d4c3981 330 priv->ae_handle->last_reset_time = jiffies;
76ad4f0e
S		 331 return 0;
332}
333
334static void hns3_nic_net_down(struct net_device *netdev)
335{
336 struct hns3_nic_priv *priv = netdev_priv(netdev);
337 const struct hnae3_ae_ops *ops;
338 int i;
339
b875cc37
JS		 340 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
341 return;
342
7b763f3f
FL		 343 /* disable vectors */
344 for (i = 0; i < priv->vector_num; i++)
345 hns3_vector_disable(&priv->tqp_vector[i]);
346
76ad4f0e
S		 347 /* stop ae_dev */
348 ops = priv->ae_handle->ae_algo->ops;
349 if (ops->stop)
350 ops->stop(priv->ae_handle);
351
76ad4f0e
S		 352 /* free irq resources */
353 hns3_nic_uninit_irq(priv);
7b763f3f
FL		 354
355 hns3_clear_all_ring(priv->ae_handle);
76ad4f0e
S		 356}
357
358static int hns3_nic_net_stop(struct net_device *netdev)
359{
360 netif_tx_stop_all_queues(netdev);
361 netif_carrier_off(netdev);
362
363 hns3_nic_net_down(netdev);
364
365 return 0;
366}
367
76ad4f0e
S		 368static int hns3_nic_uc_sync(struct net_device *netdev,
369 const unsigned char *addr)
370{
9780cb97 371 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 372
373 if (h->ae_algo->ops->add_uc_addr)
374 return h->ae_algo->ops->add_uc_addr(h, addr);
375
376 return 0;
377}
378
379static int hns3_nic_uc_unsync(struct net_device *netdev,
380 const unsigned char *addr)
381{
9780cb97 382 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 383
384 if (h->ae_algo->ops->rm_uc_addr)
385 return h->ae_algo->ops->rm_uc_addr(h, addr);
386
387 return 0;
388}
389
390static int hns3_nic_mc_sync(struct net_device *netdev,
391 const unsigned char *addr)
392{
9780cb97 393 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 394
720a8478 395 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 396 return h->ae_algo->ops->add_mc_addr(h, addr);
397
398 return 0;
399}
400
401static int hns3_nic_mc_unsync(struct net_device *netdev,
402 const unsigned char *addr)
403{
9780cb97 404 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 405
720a8478 406 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 407 return h->ae_algo->ops->rm_mc_addr(h, addr);
408
409 return 0;
410}
411
1db9b1bf 412static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 413{
9780cb97 414 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 415
416 if (h->ae_algo->ops->set_promisc_mode) {
417 if (netdev->flags & IFF_PROMISC)
3b75c3df
PL		 418 h->ae_algo->ops->set_promisc_mode(h, true, true);
419 else if (netdev->flags & IFF_ALLMULTI)
420 h->ae_algo->ops->set_promisc_mode(h, false, true);
76ad4f0e 421 else
3b75c3df 422 h->ae_algo->ops->set_promisc_mode(h, false, false);
76ad4f0e
S		 423 }
424 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
425 netdev_err(netdev, "sync uc address fail\n");
40cca1c5 426 if (netdev->flags & IFF_MULTICAST) {
76ad4f0e
S		 427 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
428 netdev_err(netdev, "sync mc address fail\n");
40cca1c5
XW		 429
430 if (h->ae_algo->ops->update_mta_status)
431 h->ae_algo->ops->update_mta_status(h);
432 }
76ad4f0e
S		 433}
434
435static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
436 u16 *mss, u32 *type_cs_vlan_tso)
437{
438 u32 l4_offset, hdr_len;
439 union l3_hdr_info l3;
440 union l4_hdr_info l4;
441 u32 l4_paylen;
442 int ret;
443
444 if (!skb_is_gso(skb))
445 return 0;
446
447 ret = skb_cow_head(skb, 0);
448 if (ret)
449 return ret;
450
451 l3.hdr = skb_network_header(skb);
452 l4.hdr = skb_transport_header(skb);
453
454 /* Software should clear the IPv4's checksum field when tso is
455 * needed.
456 */
457 if (l3.v4->version == 4)
458 l3.v4->check = 0;
459
460 /* tunnel packet.*/
461 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
462 SKB_GSO_GRE_CSUM |
463 SKB_GSO_UDP_TUNNEL |
464 SKB_GSO_UDP_TUNNEL_CSUM)) {
465 if ((!(skb_shinfo(skb)->gso_type &
466 SKB_GSO_PARTIAL)) &&
467 (skb_shinfo(skb)->gso_type &
468 SKB_GSO_UDP_TUNNEL_CSUM)) {
469 /* Software should clear the udp's checksum
470 * field when tso is needed.
471 */
472 l4.udp->check = 0;
473 }
474 /* reset l3&l4 pointers from outer to inner headers */
475 l3.hdr = skb_inner_network_header(skb);
476 l4.hdr = skb_inner_transport_header(skb);
477
478 /* Software should clear the IPv4's checksum field when
479 * tso is needed.
480 */
481 if (l3.v4->version == 4)
482 l3.v4->check = 0;
483 }
484
485 /* normal or tunnel packet*/
486 l4_offset = l4.hdr - skb->data;
487 hdr_len = (l4.tcp->doff * 4) + l4_offset;
488
489 /* remove payload length from inner pseudo checksum when tso*/
490 l4_paylen = skb->len - l4_offset;
491 csum_replace_by_diff(&l4.tcp->check,
492 (__force __wsum)htonl(l4_paylen));
493
494 /* find the txbd field values */
495 *paylen = skb->len - hdr_len;
496 hnae_set_bit(*type_cs_vlan_tso,
497 HNS3_TXD_TSO_B, 1);
498
499 /* get MSS for TSO */
500 *mss = skb_shinfo(skb)->gso_size;
501
502 return 0;
503}
504
1898d4e4
S		 505static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
506 u8 *il4_proto)
76ad4f0e
S		 507{
508 union {
509 struct iphdr *v4;
510 struct ipv6hdr *v6;
511 unsigned char *hdr;
512 } l3;
513 unsigned char *l4_hdr;
514 unsigned char *exthdr;
515 u8 l4_proto_tmp;
516 __be16 frag_off;
517
518 /* find outer header point */
519 l3.hdr = skb_network_header(skb);
35f58fd7 520 l4_hdr = skb_transport_header(skb);
76ad4f0e
S		 521
522 if (skb->protocol == htons(ETH_P_IPV6)) {
523 exthdr = l3.hdr + sizeof(*l3.v6);
524 l4_proto_tmp = l3.v6->nexthdr;
525 if (l4_hdr != exthdr)
526 ipv6_skip_exthdr(skb, exthdr - skb->data,
527 &l4_proto_tmp, &frag_off);
528 } else if (skb->protocol == htons(ETH_P_IP)) {
529 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 530 } else {
531 return -EINVAL;
76ad4f0e
S		 532 }
533
534 *ol4_proto = l4_proto_tmp;
535
536 /* tunnel packet */
537 if (!skb->encapsulation) {
538 *il4_proto = 0;
1898d4e4 539 return 0;
76ad4f0e
S		 540 }
541
542 /* find inner header point */
543 l3.hdr = skb_inner_network_header(skb);
544 l4_hdr = skb_inner_transport_header(skb);
545
546 if (l3.v6->version == 6) {
547 exthdr = l3.hdr + sizeof(*l3.v6);
548 l4_proto_tmp = l3.v6->nexthdr;
549 if (l4_hdr != exthdr)
550 ipv6_skip_exthdr(skb, exthdr - skb->data,
551 &l4_proto_tmp, &frag_off);
552 } else if (l3.v4->version == 4) {
553 l4_proto_tmp = l3.v4->protocol;
554 }
555
556 *il4_proto = l4_proto_tmp;
1898d4e4
S		 557
558 return 0;
76ad4f0e
S		 559}
560
561static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
562 u8 il4_proto, u32 *type_cs_vlan_tso,
563 u32 *ol_type_vlan_len_msec)
564{
565 union {
566 struct iphdr *v4;
567 struct ipv6hdr *v6;
568 unsigned char *hdr;
569 } l3;
570 union {
571 struct tcphdr *tcp;
572 struct udphdr *udp;
573 struct gre_base_hdr *gre;
574 unsigned char *hdr;
575 } l4;
576 unsigned char *l2_hdr;
577 u8 l4_proto = ol4_proto;
578 u32 ol2_len;
579 u32 ol3_len;
580 u32 ol4_len;
581 u32 l2_len;
582 u32 l3_len;
583
584 l3.hdr = skb_network_header(skb);
585 l4.hdr = skb_transport_header(skb);
586
587 /* compute L2 header size for normal packet, defined in 2 Bytes */
588 l2_len = l3.hdr - skb->data;
589 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
590 HNS3_TXD_L2LEN_S, l2_len >> 1);
591
592 /* tunnel packet*/
593 if (skb->encapsulation) {
594 /* compute OL2 header size, defined in 2 Bytes */
595 ol2_len = l2_len;
596 hnae_set_field(*ol_type_vlan_len_msec,
597 HNS3_TXD_L2LEN_M,
598 HNS3_TXD_L2LEN_S, ol2_len >> 1);
599
600 /* compute OL3 header size, defined in 4 Bytes */
601 ol3_len = l4.hdr - l3.hdr;
602 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
603 HNS3_TXD_L3LEN_S, ol3_len >> 2);
604
605 /* MAC in UDP, MAC in GRE (0x6558)*/
606 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
607 /* switch MAC header ptr from outer to inner header.*/
608 l2_hdr = skb_inner_mac_header(skb);
609
610 /* compute OL4 header size, defined in 4 Bytes. */
611 ol4_len = l2_hdr - l4.hdr;
612 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
613 HNS3_TXD_L4LEN_S, ol4_len >> 2);
614
615 /* switch IP header ptr from outer to inner header */
616 l3.hdr = skb_inner_network_header(skb);
617
618 /* compute inner l2 header size, defined in 2 Bytes. */
619 l2_len = l3.hdr - l2_hdr;
620 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
621 HNS3_TXD_L2LEN_S, l2_len >> 1);
622 } else {
623 /* skb packet types not supported by hardware,
624 * txbd len fild doesn't be filled.
625 */
626 return;
627 }
628
629 /* switch L4 header pointer from outer to inner */
630 l4.hdr = skb_inner_transport_header(skb);
631
632 l4_proto = il4_proto;
633 }
634
635 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
636 l3_len = l4.hdr - l3.hdr;
637 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
638 HNS3_TXD_L3LEN_S, l3_len >> 2);
639
640 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
641 switch (l4_proto) {
642 case IPPROTO_TCP:
643 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
644 HNS3_TXD_L4LEN_S, l4.tcp->doff);
645 break;
646 case IPPROTO_SCTP:
647 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
648 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
649 break;
650 case IPPROTO_UDP:
651 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
652 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
653 break;
654 default:
655 /* skb packet types not supported by hardware,
656 * txbd len fild doesn't be filled.
657 */
658 return;
659 }
660}
661
3db084d2
YL		 662/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
663 * and it is udp packet, which has a dest port as the IANA assigned.
664 * the hardware is expected to do the checksum offload, but the
665 * hardware will not do the checksum offload when udp dest port is
666 * 4789.
667 */
668static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
669{
670#define IANA_VXLAN_PORT 4789
671 union {
672 struct tcphdr *tcp;
673 struct udphdr *udp;
674 struct gre_base_hdr *gre;
675 unsigned char *hdr;
676 } l4;
677
678 l4.hdr = skb_transport_header(skb);
679
680 if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
681 return false;
682
683 skb_checksum_help(skb);
684
685 return true;
686}
687
76ad4f0e
S		 688static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
689 u8 il4_proto, u32 *type_cs_vlan_tso,
690 u32 *ol_type_vlan_len_msec)
691{
692 union {
693 struct iphdr *v4;
694 struct ipv6hdr *v6;
695 unsigned char *hdr;
696 } l3;
697 u32 l4_proto = ol4_proto;
698
699 l3.hdr = skb_network_header(skb);
700
701 /* define OL3 type and tunnel type(OL4).*/
702 if (skb->encapsulation) {
703 /* define outer network header type.*/
704 if (skb->protocol == htons(ETH_P_IP)) {
705 if (skb_is_gso(skb))
706 hnae_set_field(*ol_type_vlan_len_msec,
707 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
708 HNS3_OL3T_IPV4_CSUM);
709 else
710 hnae_set_field(*ol_type_vlan_len_msec,
711 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
712 HNS3_OL3T_IPV4_NO_CSUM);
713
714 } else if (skb->protocol == htons(ETH_P_IPV6)) {
715 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
716 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
717 }
718
719 /* define tunnel type(OL4).*/
720 switch (l4_proto) {
721 case IPPROTO_UDP:
722 hnae_set_field(*ol_type_vlan_len_msec,
723 HNS3_TXD_TUNTYPE_M,
724 HNS3_TXD_TUNTYPE_S,
725 HNS3_TUN_MAC_IN_UDP);
726 break;
727 case IPPROTO_GRE:
728 hnae_set_field(*ol_type_vlan_len_msec,
729 HNS3_TXD_TUNTYPE_M,
730 HNS3_TXD_TUNTYPE_S,
731 HNS3_TUN_NVGRE);
732 break;
733 default:
734 /* drop the skb tunnel packet if hardware don't support,
735 * because hardware can't calculate csum when TSO.
736 */
737 if (skb_is_gso(skb))
738 return -EDOM;
739
740 /* the stack computes the IP header already,
741 * driver calculate l4 checksum when not TSO.
742 */
743 skb_checksum_help(skb);
744 return 0;
745 }
746
747 l3.hdr = skb_inner_network_header(skb);
748 l4_proto = il4_proto;
749 }
750
751 if (l3.v4->version == 4) {
752 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
753 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
754
755 /* the stack computes the IP header already, the only time we
756 * need the hardware to recompute it is in the case of TSO.
757 */
758 if (skb_is_gso(skb))
759 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
760
761 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
762 } else if (l3.v6->version == 6) {
763 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
764 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
765 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
766 }
767
768 switch (l4_proto) {
769 case IPPROTO_TCP:
770 hnae_set_field(*type_cs_vlan_tso,
771 HNS3_TXD_L4T_M,
772 HNS3_TXD_L4T_S,
773 HNS3_L4T_TCP);
774 break;
775 case IPPROTO_UDP:
3db084d2
YL		 776 if (hns3_tunnel_csum_bug(skb))
777 break;
778
76ad4f0e
S		 779 hnae_set_field(*type_cs_vlan_tso,
780 HNS3_TXD_L4T_M,
781 HNS3_TXD_L4T_S,
782 HNS3_L4T_UDP);
783 break;
784 case IPPROTO_SCTP:
785 hnae_set_field(*type_cs_vlan_tso,
786 HNS3_TXD_L4T_M,
787 HNS3_TXD_L4T_S,
788 HNS3_L4T_SCTP);
789 break;
790 default:
791 /* drop the skb tunnel packet if hardware don't support,
792 * because hardware can't calculate csum when TSO.
793 */
794 if (skb_is_gso(skb))
795 return -EDOM;
796
797 /* the stack computes the IP header already,
798 * driver calculate l4 checksum when not TSO.
799 */
800 skb_checksum_help(skb);
801 return 0;
802 }
803
804 return 0;
805}
806
807static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
808{
809 /* Config bd buffer end */
810 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
3c8f5c03 811 HNS3_TXD_BDTYPE_S, 0);
76ad4f0e
S		 812 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
813 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
7036d26f 814 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 815}
816
9699cffe
PL		 817static int hns3_fill_desc_vtags(struct sk_buff *skb,
818 struct hns3_enet_ring *tx_ring,
819 u32 *inner_vlan_flag,
820 u32 *out_vlan_flag,
821 u16 *inner_vtag,
822 u16 *out_vtag)
823{
824#define HNS3_TX_VLAN_PRIO_SHIFT 13
825
826 if (skb->protocol == htons(ETH_P_8021Q) &&
827 !(tx_ring->tqp->handle->kinfo.netdev->features &
828 NETIF_F_HW_VLAN_CTAG_TX)) {
829 /* When HW VLAN acceleration is turned off, and the stack
830 * sets the protocol to 802.1q, the driver just need to
831 * set the protocol to the encapsulated ethertype.
832 */
833 skb->protocol = vlan_get_protocol(skb);
834 return 0;
835 }
836
837 if (skb_vlan_tag_present(skb)) {
838 u16 vlan_tag;
839
840 vlan_tag = skb_vlan_tag_get(skb);
841 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
842
843 /* Based on hw strategy, use out_vtag in two layer tag case,
844 * and use inner_vtag in one tag case.
845 */
846 if (skb->protocol == htons(ETH_P_8021Q)) {
847 hnae_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
848 *out_vtag = vlan_tag;
849 } else {
850 hnae_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
851 *inner_vtag = vlan_tag;
852 }
853 } else if (skb->protocol == htons(ETH_P_8021Q)) {
854 struct vlan_ethhdr *vhdr;
855 int rc;
856
857 rc = skb_cow_head(skb, 0);
858 if (rc < 0)
859 return rc;
860 vhdr = (struct vlan_ethhdr *)skb->data;
861 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
862 << HNS3_TX_VLAN_PRIO_SHIFT);
863 }
864
865 skb->protocol = vlan_get_protocol(skb);
866 return 0;
867}
868
76ad4f0e
S		 869static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
870 int size, dma_addr_t dma, int frag_end,
871 enum hns_desc_type type)
872{
873 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
874 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
875 u32 ol_type_vlan_len_msec = 0;
876 u16 bdtp_fe_sc_vld_ra_ri = 0;
877 u32 type_cs_vlan_tso = 0;
878 struct sk_buff *skb;
9699cffe
PL		 879 u16 inner_vtag = 0;
880 u16 out_vtag = 0;
76ad4f0e
S		 881 u32 paylen = 0;
882 u16 mss = 0;
883 __be16 protocol;
884 u8 ol4_proto;
885 u8 il4_proto;
886 int ret;
887
888 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
889 desc_cb->priv = priv;
890 desc_cb->length = size;
891 desc_cb->dma = dma;
892 desc_cb->type = type;
893
894 /* now, fill the descriptor */
895 desc->addr = cpu_to_le64(dma);
896 desc->tx.send_size = cpu_to_le16((u16)size);
897 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
898 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
899
900 if (type == DESC_TYPE_SKB) {
901 skb = (struct sk_buff *)priv;
a90bb9a5 902 paylen = skb->len;
76ad4f0e 903
9699cffe
PL		 904 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
905 &ol_type_vlan_len_msec,
906 &inner_vtag, &out_vtag);
907 if (unlikely(ret))
908 return ret;
909
76ad4f0e
S		 910 if (skb->ip_summed == CHECKSUM_PARTIAL) {
911 skb_reset_mac_len(skb);
912 protocol = skb->protocol;
913
1898d4e4
S		 914 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
915 if (ret)
916 return ret;
76ad4f0e
S		 917 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
918 &type_cs_vlan_tso,
919 &ol_type_vlan_len_msec);
920 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
921 &type_cs_vlan_tso,
922 &ol_type_vlan_len_msec);
923 if (ret)
924 return ret;
925
926 ret = hns3_set_tso(skb, &paylen, &mss,
927 &type_cs_vlan_tso);
928 if (ret)
929 return ret;
930 }
931
932 /* Set txbd */
933 desc->tx.ol_type_vlan_len_msec =
934 cpu_to_le32(ol_type_vlan_len_msec);
935 desc->tx.type_cs_vlan_tso_len =
936 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 937 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 938 desc->tx.mss = cpu_to_le16(mss);
9699cffe
PL		 939 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
940 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
76ad4f0e
S		 941 }
942
943 /* move ring pointer to next.*/
944 ring_ptr_move_fw(ring, next_to_use);
945
946 return 0;
947}
948
949static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
950 int size, dma_addr_t dma, int frag_end,
951 enum hns_desc_type type)
952{
953 unsigned int frag_buf_num;
954 unsigned int k;
955 int sizeoflast;
956 int ret;
957
958 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
959 sizeoflast = size % HNS3_MAX_BD_SIZE;
960 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
961
962 /* When the frag size is bigger than hardware, split this frag */
963 for (k = 0; k < frag_buf_num; k++) {
964 ret = hns3_fill_desc(ring, priv,
965 (k == frag_buf_num - 1) ?
966 sizeoflast : HNS3_MAX_BD_SIZE,
967 dma + HNS3_MAX_BD_SIZE * k,
968 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
969 (type == DESC_TYPE_SKB && !k) ?
970 DESC_TYPE_SKB : DESC_TYPE_PAGE);
971 if (ret)
972 return ret;
973 }
974
975 return 0;
976}
977
978static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
979 struct hns3_enet_ring *ring)
980{
981 struct sk_buff *skb = *out_skb;
982 struct skb_frag_struct *frag;
983 int bdnum_for_frag;
984 int frag_num;
985 int buf_num;
986 int size;
987 int i;
988
989 size = skb_headlen(skb);
990 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
991
992 frag_num = skb_shinfo(skb)->nr_frags;
993 for (i = 0; i < frag_num; i++) {
994 frag = &skb_shinfo(skb)->frags[i];
995 size = skb_frag_size(frag);
996 bdnum_for_frag =
997 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
998 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
999 return -ENOMEM;
1000
1001 buf_num += bdnum_for_frag;
1002 }
1003
1004 if (buf_num > ring_space(ring))
1005 return -EBUSY;
1006
1007 *bnum = buf_num;
1008 return 0;
1009}
1010
1011static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1012 struct hns3_enet_ring *ring)
1013{
1014 struct sk_buff *skb = *out_skb;
1015 int buf_num;
1016
1017 /* No. of segments (plus a header) */
1018 buf_num = skb_shinfo(skb)->nr_frags + 1;
1019
1020 if (buf_num > ring_space(ring))
1021 return -EBUSY;
1022
1023 *bnum = buf_num;
1024
1025 return 0;
1026}
1027
1028static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
1029{
1030 struct device *dev = ring_to_dev(ring);
1031 unsigned int i;
1032
1033 for (i = 0; i < ring->desc_num; i++) {
1034 /* check if this is where we started */
1035 if (ring->next_to_use == next_to_use_orig)
1036 break;
1037
1038 /* unmap the descriptor dma address */
1039 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1040 dma_unmap_single(dev,
1041 ring->desc_cb[ring->next_to_use].dma,
1042 ring->desc_cb[ring->next_to_use].length,
1043 DMA_TO_DEVICE);
1044 else
1045 dma_unmap_page(dev,
1046 ring->desc_cb[ring->next_to_use].dma,
1047 ring->desc_cb[ring->next_to_use].length,
1048 DMA_TO_DEVICE);
1049
1050 /* rollback one */
1051 ring_ptr_move_bw(ring, next_to_use);
1052 }
1053}
1054
d43e5aca 1055netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 1056{
1057 struct hns3_nic_priv *priv = netdev_priv(netdev);
1058 struct hns3_nic_ring_data *ring_data =
1059 &tx_ring_data(priv, skb->queue_mapping);
1060 struct hns3_enet_ring *ring = ring_data->ring;
1061 struct device *dev = priv->dev;
1062 struct netdev_queue *dev_queue;
1063 struct skb_frag_struct *frag;
1064 int next_to_use_head;
1065 int next_to_use_frag;
1066 dma_addr_t dma;
1067 int buf_num;
1068 int seg_num;
1069 int size;
1070 int ret;
1071 int i;
1072
1073 /* Prefetch the data used later */
1074 prefetch(skb->data);
1075
1076 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1077 case -EBUSY:
1078 u64_stats_update_begin(&ring->syncp);
1079 ring->stats.tx_busy++;
1080 u64_stats_update_end(&ring->syncp);
1081
1082 goto out_net_tx_busy;
1083 case -ENOMEM:
1084 u64_stats_update_begin(&ring->syncp);
1085 ring->stats.sw_err_cnt++;
1086 u64_stats_update_end(&ring->syncp);
1087 netdev_err(netdev, "no memory to xmit!\n");
1088
1089 goto out_err_tx_ok;
1090 default:
1091 break;
1092 }
1093
1094 /* No. of segments (plus a header) */
1095 seg_num = skb_shinfo(skb)->nr_frags + 1;
1096 /* Fill the first part */
1097 size = skb_headlen(skb);
1098
1099 next_to_use_head = ring->next_to_use;
1100
1101 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1102 if (dma_mapping_error(dev, dma)) {
1103 netdev_err(netdev, "TX head DMA map failed\n");
1104 ring->stats.sw_err_cnt++;
1105 goto out_err_tx_ok;
1106 }
1107
1108 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
1109 DESC_TYPE_SKB);
1110 if (ret)
1111 goto head_dma_map_err;
1112
1113 next_to_use_frag = ring->next_to_use;
1114 /* Fill the fragments */
1115 for (i = 1; i < seg_num; i++) {
1116 frag = &skb_shinfo(skb)->frags[i - 1];
1117 size = skb_frag_size(frag);
1118 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1119 if (dma_mapping_error(dev, dma)) {
1120 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
1121 ring->stats.sw_err_cnt++;
1122 goto frag_dma_map_err;
1123 }
1124 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
1125 seg_num - 1 == i ? 1 : 0,
1126 DESC_TYPE_PAGE);
1127
1128 if (ret)
1129 goto frag_dma_map_err;
1130 }
1131
1132 /* Complete translate all packets */
1133 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1134 netdev_tx_sent_queue(dev_queue, skb->len);
1135
1136 wmb(); /* Commit all data before submit */
1137
1138 hnae_queue_xmit(ring->tqp, buf_num);
1139
1140 return NETDEV_TX_OK;
1141
1142frag_dma_map_err:
1143 hns_nic_dma_unmap(ring, next_to_use_frag);
1144
1145head_dma_map_err:
1146 hns_nic_dma_unmap(ring, next_to_use_head);
1147
1148out_err_tx_ok:
1149 dev_kfree_skb_any(skb);
1150 return NETDEV_TX_OK;
1151
1152out_net_tx_busy:
1153 netif_stop_subqueue(netdev, ring_data->queue_index);
1154 smp_mb(); /* Commit all data before submit */
1155
1156 return NETDEV_TX_BUSY;
1157}
1158
1159static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1160{
9780cb97 1161 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1162 struct sockaddr *mac_addr = p;
1163 int ret;
1164
1165 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1166 return -EADDRNOTAVAIL;
1167
5ec2a51e
JS		 1168 if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1169 netdev_info(netdev, "already using mac address %pM\n",
1170 mac_addr->sa_data);
1171 return 0;
1172 }
1173
59098055 1174 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
76ad4f0e
S		 1175 if (ret) {
1176 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1177 return ret;
1178 }
1179
1180 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1181
1182 return 0;
1183}
1184
1185static int hns3_nic_set_features(struct net_device *netdev,
1186 netdev_features_t features)
1187{
181d454b 1188 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 1189 struct hns3_nic_priv *priv = netdev_priv(netdev);
052ece6d 1190 struct hnae3_handle *h = priv->ae_handle;
052ece6d 1191 int ret;
76ad4f0e 1192
181d454b
JS		 1193 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1194 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1195 priv->ops.fill_desc = hns3_fill_desc_tso;
1196 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1197 } else {
1198 priv->ops.fill_desc = hns3_fill_desc;
1199 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1200 }
76ad4f0e
S		 1201 }
1202
bd368416
JS		 1203 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1204 h->ae_algo->ops->enable_vlan_filter) {
181d454b
JS		 1205 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1206 h->ae_algo->ops->enable_vlan_filter(h, true);
1207 else
1208 h->ae_algo->ops->enable_vlan_filter(h, false);
1209 }
391b5e93 1210
bd368416
JS		 1211 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1212 h->ae_algo->ops->enable_hw_strip_rxvtag) {
052ece6d
PL		 1213 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1214 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1215 else
1216 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1217
1218 if (ret)
1219 return ret;
1220 }
1221
76ad4f0e
S		 1222 netdev->features = features;
1223 return 0;
1224}
1225
6c88d9d7
PL		 1226static void hns3_nic_get_stats64(struct net_device *netdev,
1227 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1228{
1229 struct hns3_nic_priv *priv = netdev_priv(netdev);
1230 int queue_num = priv->ae_handle->kinfo.num_tqps;
c5f65480 1231 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1232 struct hns3_enet_ring *ring;
1233 unsigned int start;
1234 unsigned int idx;
1235 u64 tx_bytes = 0;
1236 u64 rx_bytes = 0;
1237 u64 tx_pkts = 0;
1238 u64 rx_pkts = 0;
d2a5dca8
JS		 1239 u64 tx_drop = 0;
1240 u64 rx_drop = 0;
76ad4f0e 1241
b875cc37
JS		 1242 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1243 return;
1244
c5f65480
JS		 1245 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1246
76ad4f0e
S		 1247 for (idx = 0; idx < queue_num; idx++) {
1248 /* fetch the tx stats */
1249 ring = priv->ring_data[idx].ring;
1250 do {
d36d36ce 1251 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1252 tx_bytes += ring->stats.tx_bytes;
1253 tx_pkts += ring->stats.tx_pkts;
d2a5dca8
JS		 1254 tx_drop += ring->stats.tx_busy;
1255 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1256 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1257
1258 /* fetch the rx stats */
1259 ring = priv->ring_data[idx + queue_num].ring;
1260 do {
d36d36ce 1261 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1262 rx_bytes += ring->stats.rx_bytes;
1263 rx_pkts += ring->stats.rx_pkts;
d2a5dca8
JS		 1264 rx_drop += ring->stats.non_vld_descs;
1265 rx_drop += ring->stats.err_pkt_len;
1266 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1267 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1268 }
1269
1270 stats->tx_bytes = tx_bytes;
1271 stats->tx_packets = tx_pkts;
1272 stats->rx_bytes = rx_bytes;
1273 stats->rx_packets = rx_pkts;
1274
1275 stats->rx_errors = netdev->stats.rx_errors;
1276 stats->multicast = netdev->stats.multicast;
1277 stats->rx_length_errors = netdev->stats.rx_length_errors;
1278 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1279 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1280
1281 stats->tx_errors = netdev->stats.tx_errors;
d2a5dca8
JS		 1282 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1283 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1284 stats->collisions = netdev->stats.collisions;
1285 stats->rx_over_errors = netdev->stats.rx_over_errors;
1286 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1287 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1288 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1289 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1290 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1291 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1292 stats->tx_window_errors = netdev->stats.tx_window_errors;
1293 stats->rx_compressed = netdev->stats.rx_compressed;
1294 stats->tx_compressed = netdev->stats.tx_compressed;
1295}
1296
30d240df 1297static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1298{
30d240df 1299 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1300 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1301 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1302 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1303 u8 tc = mqprio_qopt->qopt.num_tc;
1304 u16 mode = mqprio_qopt->mode;
1305 u8 hw = mqprio_qopt->qopt.hw;
1306 bool if_running;
76ad4f0e
S		 1307 unsigned int i;
1308 int ret;
1309
30d240df
YL		 1310 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1311 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1312 return -EOPNOTSUPP;
1313
76ad4f0e
S		 1314 if (tc > HNAE3_MAX_TC)
1315 return -EINVAL;
1316
76ad4f0e
S		 1317 if (!netdev)
1318 return -EINVAL;
1319
30d240df
YL		 1320 if_running = netif_running(netdev);
1321 if (if_running) {
1322 hns3_nic_net_stop(netdev);
1323 msleep(100);
76ad4f0e
S		 1324 }
1325
30d240df
YL		 1326 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1327 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1328 if (ret)
30d240df
YL		 1329 goto out;
1330
1331 if (tc <= 1) {
1332 netdev_reset_tc(netdev);
1333 } else {
1334 ret = netdev_set_num_tc(netdev, tc);
1335 if (ret)
1336 goto out;
1337
1338 for (i = 0; i < HNAE3_MAX_TC; i++) {
1339 if (!kinfo->tc_info[i].enable)
1340 continue;
76ad4f0e 1341
76ad4f0e
S		 1342 netdev_set_tc_queue(netdev,
1343 kinfo->tc_info[i].tc,
1344 kinfo->tc_info[i].tqp_count,
1345 kinfo->tc_info[i].tqp_offset);
30d240df 1346 }
76ad4f0e
S		 1347 }
1348
30d240df
YL		 1349 ret = hns3_nic_set_real_num_queue(netdev);
1350
1351out:
1352 if (if_running)
1353 hns3_nic_net_open(netdev);
1354
1355 return ret;
76ad4f0e
S		 1356}
1357
2572ac53 1358static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1359 void *type_data)
76ad4f0e 1360{
575ed7d3 1361 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1362 return -EOPNOTSUPP;
76ad4f0e 1363
30d240df 1364 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1365}
1366
1367static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1368 __be16 proto, u16 vid)
1369{
9780cb97 1370 struct hnae3_handle *h = hns3_get_handle(netdev);
681ec399 1371 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1372 int ret = -EIO;
1373
1374 if (h->ae_algo->ops->set_vlan_filter)
1375 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1376
681ec399
YL		 1377 if (!ret)
1378 set_bit(vid, priv->active_vlans);
1379
76ad4f0e
S		 1380 return ret;
1381}
1382
1383static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1384 __be16 proto, u16 vid)
1385{
9780cb97 1386 struct hnae3_handle *h = hns3_get_handle(netdev);
681ec399 1387 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1388 int ret = -EIO;
1389
1390 if (h->ae_algo->ops->set_vlan_filter)
1391 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1392
681ec399
YL		 1393 if (!ret)
1394 clear_bit(vid, priv->active_vlans);
1395
76ad4f0e
S		 1396 return ret;
1397}
1398
681ec399
YL		 1399static void hns3_restore_vlan(struct net_device *netdev)
1400{
1401 struct hns3_nic_priv *priv = netdev_priv(netdev);
1402 u16 vid;
1403 int ret;
1404
1405 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1406 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1407 if (ret)
1408 netdev_warn(netdev, "Restore vlan: %d filter, ret:%d\n",
1409 vid, ret);
1410 }
1411}
1412
76ad4f0e
S		 1413static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1414 u8 qos, __be16 vlan_proto)
1415{
9780cb97 1416 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1417 int ret = -EIO;
1418
1419 if (h->ae_algo->ops->set_vf_vlan_filter)
1420 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1421 qos, vlan_proto);
1422
1423 return ret;
1424}
1425
a8e8b7ff
S		 1426static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1427{
9780cb97 1428 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1429 bool if_running = netif_running(netdev);
1430 int ret;
1431
1432 if (!h->ae_algo->ops->set_mtu)
1433 return -EOPNOTSUPP;
1434
1435 /* if this was called with netdev up then bring netdevice down */
1436 if (if_running) {
1437 (void)hns3_nic_net_stop(netdev);
1438 msleep(100);
1439 }
1440
1441 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1442 if (ret) {
1443 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1444 ret);
1445 return ret;
1446 }
1447
5bad95a1
FL		 1448 netdev->mtu = new_mtu;
1449
a8e8b7ff
S		 1450 /* if the netdev was running earlier, bring it up again */
1451 if (if_running && hns3_nic_net_open(netdev))
1452 ret = -EINVAL;
1453
1454 return ret;
1455}
1456
f8fa222c
L		 1457static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1458{
1459 struct hns3_nic_priv *priv = netdev_priv(ndev);
1460 struct hns3_enet_ring *tx_ring = NULL;
1461 int timeout_queue = 0;
1462 int hw_head, hw_tail;
1463 int i;
1464
1465 /* Find the stopped queue the same way the stack does */
1466 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1467 struct netdev_queue *q;
1468 unsigned long trans_start;
1469
1470 q = netdev_get_tx_queue(ndev, i);
1471 trans_start = q->trans_start;
1472 if (netif_xmit_stopped(q) &&
1473 time_after(jiffies,
1474 (trans_start + ndev->watchdog_timeo))) {
1475 timeout_queue = i;
1476 break;
1477 }
1478 }
1479
1480 if (i == ndev->num_tx_queues) {
1481 netdev_info(ndev,
1482 "no netdev TX timeout queue found, timeout count: %llu\n",
1483 priv->tx_timeout_count);
1484 return false;
1485 }
1486
1487 tx_ring = priv->ring_data[timeout_queue].ring;
1488
1489 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1490 HNS3_RING_TX_RING_HEAD_REG);
1491 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1492 HNS3_RING_TX_RING_TAIL_REG);
1493 netdev_info(ndev,
1494 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1495 priv->tx_timeout_count,
1496 timeout_queue,
1497 tx_ring->next_to_use,
1498 tx_ring->next_to_clean,
1499 hw_head,
1500 hw_tail,
1501 readl(tx_ring->tqp_vector->mask_addr));
1502
1503 return true;
1504}
1505
1506static void hns3_nic_net_timeout(struct net_device *ndev)
1507{
1508 struct hns3_nic_priv *priv = netdev_priv(ndev);
f8fa222c
L		 1509 struct hnae3_handle *h = priv->ae_handle;
1510
1511 if (!hns3_get_tx_timeo_queue_info(ndev))
1512 return;
1513
1514 priv->tx_timeout_count++;
1515
6d4c3981 1516 if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo)))
f8fa222c
L		 1517 return;
1518
6d4c3981 1519 /* request the reset */
f8fa222c 1520 if (h->ae_algo->ops->reset_event)
6d4c3981 1521 h->ae_algo->ops->reset_event(h);
f8fa222c
L		 1522}
1523
76ad4f0e
S		 1524static const struct net_device_ops hns3_nic_netdev_ops = {
1525 .ndo_open = hns3_nic_net_open,
1526 .ndo_stop = hns3_nic_net_stop,
1527 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1528 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1529 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a8e8b7ff 1530 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1531 .ndo_set_features = hns3_nic_set_features,
1532 .ndo_get_stats64 = hns3_nic_get_stats64,
1533 .ndo_setup_tc = hns3_nic_setup_tc,
1534 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
76ad4f0e
S		 1535 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1536 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1537 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1538};
1539
2312e050
FL		 1540static bool hns3_is_phys_func(struct pci_dev *pdev)
1541{
1542 u32 dev_id = pdev->device;
1543
1544 switch (dev_id) {
1545 case HNAE3_DEV_ID_GE:
1546 case HNAE3_DEV_ID_25GE:
1547 case HNAE3_DEV_ID_25GE_RDMA:
1548 case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1549 case HNAE3_DEV_ID_50GE_RDMA:
1550 case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1551 case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1552 return true;
1553 case HNAE3_DEV_ID_100G_VF:
1554 case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1555 return false;
1556 default:
1557 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1558 dev_id);
1559 }
1560
1561 return false;
1562}
1563
2312e050
FL		 1564static void hns3_disable_sriov(struct pci_dev *pdev)
1565{
1566 /* If our VFs are assigned we cannot shut down SR-IOV
1567 * without causing issues, so just leave the hardware
1568 * available but disabled
1569 */
1570 if (pci_vfs_assigned(pdev)) {
1571 dev_warn(&pdev->dev,
1572 "disabling driver while VFs are assigned\n");
1573 return;
1574 }
1575
1576 pci_disable_sriov(pdev);
1577}
1578
76ad4f0e
S		 1579/* hns3_probe - Device initialization routine
1580 * @pdev: PCI device information struct
1581 * @ent: entry in hns3_pci_tbl
1582 *
1583 * hns3_probe initializes a PF identified by a pci_dev structure.
1584 * The OS initialization, configuring of the PF private structure,
1585 * and a hardware reset occur.
1586 *
1587 * Returns 0 on success, negative on failure
1588 */
1589static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1590{
1591 struct hnae3_ae_dev *ae_dev;
1592 int ret;
1593
1594 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1595 GFP_KERNEL);
1596 if (!ae_dev) {
1597 ret = -ENOMEM;
1598 return ret;
1599 }
1600
1601 ae_dev->pdev = pdev;
e92a0843 1602 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 1603 ae_dev->dev_type = HNAE3_DEV_KNIC;
1604 pci_set_drvdata(pdev, ae_dev);
1605
50fbc237 1606 hnae3_register_ae_dev(ae_dev);
2312e050 1607
2312e050 1608 return 0;
76ad4f0e
S		 1609}
1610
1611/* hns3_remove - Device removal routine
1612 * @pdev: PCI device information struct
1613 */
1614static void hns3_remove(struct pci_dev *pdev)
1615{
1616 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1617
2312e050
FL		 1618 if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1619 hns3_disable_sriov(pdev);
1620
76ad4f0e 1621 hnae3_unregister_ae_dev(ae_dev);
76ad4f0e
S		 1622}
1623
fa8d82e8
PL		 1624/**
1625 * hns3_pci_sriov_configure
1626 * @pdev: pointer to a pci_dev structure
1627 * @num_vfs: number of VFs to allocate
1628 *
1629 * Enable or change the number of VFs. Called when the user updates the number
1630 * of VFs in sysfs.
1631 **/
743e1a84 1632static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
fa8d82e8
PL		 1633{
1634 int ret;
1635
1636 if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1637 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1638 return -EINVAL;
1639 }
1640
1641 if (num_vfs) {
1642 ret = pci_enable_sriov(pdev, num_vfs);
1643 if (ret)
1644 dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
743e1a84
SM		 1645 else
1646 return num_vfs;
fa8d82e8
PL		 1647 } else if (!pci_vfs_assigned(pdev)) {
1648 pci_disable_sriov(pdev);
1649 } else {
1650 dev_warn(&pdev->dev,
1651 "Unable to free VFs because some are assigned to VMs.\n");
1652 }
1653
1654 return 0;
1655}
1656
76ad4f0e
S		 1657static struct pci_driver hns3_driver = {
1658 .name = hns3_driver_name,
1659 .id_table = hns3_pci_tbl,
1660 .probe = hns3_probe,
1661 .remove = hns3_remove,
fa8d82e8 1662 .sriov_configure = hns3_pci_sriov_configure,
76ad4f0e
S		 1663};
1664
1665/* set default feature to hns3 */
1666static void hns3_set_default_feature(struct net_device *netdev)
1667{
1668 netdev->priv_flags |= IFF_UNICAST_FLT;
1669
1670 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1671 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1672 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1673 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1674 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1675
1676 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1677
1678 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1679
1680 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1681 NETIF_F_HW_VLAN_CTAG_FILTER |
052ece6d 1682 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1683 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1684 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1685 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1686 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1687
1688 netdev->vlan_features |=
1689 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1690 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1691 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1692 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1693 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1694
1695 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
b2641e2a 1696 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1697 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1698 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1699 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1700 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1701}
1702
1703static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1704 struct hns3_desc_cb *cb)
1705{
1706 unsigned int order = hnae_page_order(ring);
1707 struct page *p;
1708
1709 p = dev_alloc_pages(order);
1710 if (!p)
1711 return -ENOMEM;
1712
1713 cb->priv = p;
1714 cb->page_offset = 0;
1715 cb->reuse_flag = 0;
1716 cb->buf = page_address(p);
1717 cb->length = hnae_page_size(ring);
1718 cb->type = DESC_TYPE_PAGE;
1719
76ad4f0e
S		 1720 return 0;
1721}
1722
1723static void hns3_free_buffer(struct hns3_enet_ring *ring,
1724 struct hns3_desc_cb *cb)
1725{
1726 if (cb->type == DESC_TYPE_SKB)
1727 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1728 else if (!HNAE3_IS_TX_RING(ring))
1729 put_page((struct page *)cb->priv);
1730 memset(cb, 0, sizeof(*cb));
1731}
1732
1733static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1734{
1735 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1736 cb->length, ring_to_dma_dir(ring));
1737
1738 if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1739 return -EIO;
1740
1741 return 0;
1742}
1743
1744static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1745 struct hns3_desc_cb *cb)
1746{
1747 if (cb->type == DESC_TYPE_SKB)
1748 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1749 ring_to_dma_dir(ring));
1750 else
1751 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1752 ring_to_dma_dir(ring));
1753}
1754
1755static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1756{
1757 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1758 ring->desc[i].addr = 0;
1759}
1760
1761static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1762{
1763 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1764
1765 if (!ring->desc_cb[i].dma)
1766 return;
1767
1768 hns3_buffer_detach(ring, i);
1769 hns3_free_buffer(ring, cb);
1770}
1771
1772static void hns3_free_buffers(struct hns3_enet_ring *ring)
1773{
1774 int i;
1775
1776 for (i = 0; i < ring->desc_num; i++)
1777 hns3_free_buffer_detach(ring, i);
1778}
1779
1780/* free desc along with its attached buffer */
1781static void hns3_free_desc(struct hns3_enet_ring *ring)
1782{
1783 hns3_free_buffers(ring);
1784
1785 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1786 ring->desc_num * sizeof(ring->desc[0]),
1787 DMA_BIDIRECTIONAL);
1788 ring->desc_dma_addr = 0;
1789 kfree(ring->desc);
1790 ring->desc = NULL;
1791}
1792
1793static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1794{
1795 int size = ring->desc_num * sizeof(ring->desc[0]);
1796
1797 ring->desc = kzalloc(size, GFP_KERNEL);
1798 if (!ring->desc)
1799 return -ENOMEM;
1800
1801 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1802 size, DMA_BIDIRECTIONAL);
1803 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1804 ring->desc_dma_addr = 0;
1805 kfree(ring->desc);
1806 ring->desc = NULL;
1807 return -ENOMEM;
1808 }
1809
1810 return 0;
1811}
1812
1813static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1814 struct hns3_desc_cb *cb)
1815{
1816 int ret;
1817
1818 ret = hns3_alloc_buffer(ring, cb);
1819 if (ret)
1820 goto out;
1821
1822 ret = hns3_map_buffer(ring, cb);
1823 if (ret)
1824 goto out_with_buf;
1825
1826 return 0;
1827
1828out_with_buf:
564883bb 1829 hns3_free_buffer(ring, cb);
76ad4f0e
S		 1830out:
1831 return ret;
1832}
1833
1834static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1835{
1836 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1837
1838 if (ret)
1839 return ret;
1840
1841 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1842
1843 return 0;
1844}
1845
1846/* Allocate memory for raw pkg, and map with dma */
1847static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1848{
1849 int i, j, ret;
1850
1851 for (i = 0; i < ring->desc_num; i++) {
1852 ret = hns3_alloc_buffer_attach(ring, i);
1853 if (ret)
1854 goto out_buffer_fail;
1855 }
1856
1857 return 0;
1858
1859out_buffer_fail:
1860 for (j = i - 1; j >= 0; j--)
1861 hns3_free_buffer_detach(ring, j);
1862 return ret;
1863}
1864
1865/* detach a in-used buffer and replace with a reserved one */
1866static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1867 struct hns3_desc_cb *res_cb)
1868{
b9077428 1869 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 1870 ring->desc_cb[i] = *res_cb;
1871 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
7d0b130c 1872 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 1873}
1874
1875static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1876{
1877 ring->desc_cb[i].reuse_flag = 0;
1878 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1879 + ring->desc_cb[i].page_offset);
7d0b130c 1880 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 1881}
1882
1883static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1884 int *pkts)
1885{
1886 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1887
1888 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1889 (*bytes) += desc_cb->length;
1890 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1891 hns3_free_buffer_detach(ring, ring->next_to_clean);
1892
1893 ring_ptr_move_fw(ring, next_to_clean);
1894}
1895
1896static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1897{
1898 int u = ring->next_to_use;
1899 int c = ring->next_to_clean;
1900
1901 if (unlikely(h > ring->desc_num))
1902 return 0;
1903
1904 return u > c ? (h > c && h <= u) : (h > c || h <= u);
1905}
1906
24e750c4 1907bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e
S		 1908{
1909 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1910 struct netdev_queue *dev_queue;
1911 int bytes, pkts;
1912 int head;
1913
1914 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1915 rmb(); /* Make sure head is ready before touch any data */
1916
1917 if (is_ring_empty(ring) || head == ring->next_to_clean)
24e750c4 1918 return true; /* no data to poll */
76ad4f0e
S		 1919
1920 if (!is_valid_clean_head(ring, head)) {
1921 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1922 ring->next_to_use, ring->next_to_clean);
1923
1924 u64_stats_update_begin(&ring->syncp);
1925 ring->stats.io_err_cnt++;
1926 u64_stats_update_end(&ring->syncp);
24e750c4 1927 return true;
76ad4f0e
S		 1928 }
1929
1930 bytes = 0;
1931 pkts = 0;
1932 while (head != ring->next_to_clean && budget) {
1933 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1934 /* Issue prefetch for next Tx descriptor */
1935 prefetch(&ring->desc_cb[ring->next_to_clean]);
1936 budget--;
1937 }
1938
1939 ring->tqp_vector->tx_group.total_bytes += bytes;
1940 ring->tqp_vector->tx_group.total_packets += pkts;
1941
1942 u64_stats_update_begin(&ring->syncp);
1943 ring->stats.tx_bytes += bytes;
1944 ring->stats.tx_pkts += pkts;
1945 u64_stats_update_end(&ring->syncp);
1946
1947 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1948 netdev_tx_completed_queue(dev_queue, pkts, bytes);
1949
1950 if (unlikely(pkts && netif_carrier_ok(netdev) &&
1951 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1952 /* Make sure that anybody stopping the queue after this
1953 * sees the new next_to_clean.
1954 */
1955 smp_mb();
1956 if (netif_tx_queue_stopped(dev_queue)) {
1957 netif_tx_wake_queue(dev_queue);
1958 ring->stats.restart_queue++;
1959 }
1960 }
1961
1962 return !!budget;
1963}
1964
1965static int hns3_desc_unused(struct hns3_enet_ring *ring)
1966{
1967 int ntc = ring->next_to_clean;
1968 int ntu = ring->next_to_use;
1969
1970 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1971}
1972
1973static void
1974hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1975{
1976 struct hns3_desc_cb *desc_cb;
1977 struct hns3_desc_cb res_cbs;
1978 int i, ret;
1979
1980 for (i = 0; i < cleand_count; i++) {
1981 desc_cb = &ring->desc_cb[ring->next_to_use];
1982 if (desc_cb->reuse_flag) {
1983 u64_stats_update_begin(&ring->syncp);
1984 ring->stats.reuse_pg_cnt++;
1985 u64_stats_update_end(&ring->syncp);
1986
1987 hns3_reuse_buffer(ring, ring->next_to_use);
1988 } else {
1989 ret = hns3_reserve_buffer_map(ring, &res_cbs);
1990 if (ret) {
1991 u64_stats_update_begin(&ring->syncp);
1992 ring->stats.sw_err_cnt++;
1993 u64_stats_update_end(&ring->syncp);
1994
1995 netdev_err(ring->tqp->handle->kinfo.netdev,
1996 "hnae reserve buffer map failed.\n");
1997 break;
1998 }
1999 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2000 }
2001
2002 ring_ptr_move_fw(ring, next_to_use);
2003 }
2004
2005 wmb(); /* Make all data has been write before submit */
2006 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2007}
2008
76ad4f0e
S		 2009static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2010 struct hns3_enet_ring *ring, int pull_len,
2011 struct hns3_desc_cb *desc_cb)
2012{
2013 struct hns3_desc *desc;
2014 int truesize, size;
2015 int last_offset;
2016 bool twobufs;
2017
2018 twobufs = ((PAGE_SIZE < 8192) &&
2019 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2020
2021 desc = &ring->desc[ring->next_to_clean];
2022 size = le16_to_cpu(desc->rx.size);
2023
f8d291f0
PL		 2024 truesize = hnae_buf_size(ring);
2025
2026 if (!twobufs)
76ad4f0e 2027 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
76ad4f0e
S		 2028
2029 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
f8d291f0 2030 size - pull_len, truesize);
76ad4f0e
S		 2031
2032 /* Avoid re-using remote pages,flag default unreuse */
2033 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2034 return;
2035
2036 if (twobufs) {
2037 /* If we are only owner of page we can reuse it */
2038 if (likely(page_count(desc_cb->priv) == 1)) {
2039 /* Flip page offset to other buffer */
2040 desc_cb->page_offset ^= truesize;
2041
2042 desc_cb->reuse_flag = 1;
2043 /* bump ref count on page before it is given*/
2044 get_page(desc_cb->priv);
2045 }
2046 return;
2047 }
2048
2049 /* Move offset up to the next cache line */
2050 desc_cb->page_offset += truesize;
2051
2052 if (desc_cb->page_offset <= last_offset) {
2053 desc_cb->reuse_flag = 1;
2054 /* Bump ref count on page before it is given*/
2055 get_page(desc_cb->priv);
2056 }
2057}
2058
2059static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2060 struct hns3_desc *desc)
2061{
2062 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2063 int l3_type, l4_type;
2064 u32 bd_base_info;
2065 int ol4_type;
2066 u32 l234info;
2067
2068 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2069 l234info = le32_to_cpu(desc->rx.l234_info);
2070
2071 skb->ip_summed = CHECKSUM_NONE;
2072
2073 skb_checksum_none_assert(skb);
2074
2075 if (!(netdev->features & NETIF_F_RXCSUM))
2076 return;
2077
2078 /* check if hardware has done checksum */
2079 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2080 return;
2081
2082 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
2083 hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
2084 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2085 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2086 netdev_err(netdev, "L3/L4 error pkt\n");
2087 u64_stats_update_begin(&ring->syncp);
2088 ring->stats.l3l4_csum_err++;
2089 u64_stats_update_end(&ring->syncp);
2090
2091 return;
2092 }
2093
2094 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
2095 HNS3_RXD_L3ID_S);
2096 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
2097 HNS3_RXD_L4ID_S);
2098
2099 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
2100 switch (ol4_type) {
2101 case HNS3_OL4_TYPE_MAC_IN_UDP:
2102 case HNS3_OL4_TYPE_NVGRE:
2103 skb->csum_level = 1;
2104 case HNS3_OL4_TYPE_NO_TUN:
2105 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2106 if (l3_type == HNS3_L3_TYPE_IPV4 ||
2107 (l3_type == HNS3_L3_TYPE_IPV6 &&
2108 (l4_type == HNS3_L4_TYPE_UDP ||
2109 l4_type == HNS3_L4_TYPE_TCP ||
2110 l4_type == HNS3_L4_TYPE_SCTP)))
2111 skb->ip_summed = CHECKSUM_UNNECESSARY;
2112 break;
2113 }
2114}
2115
d43e5aca
YL		 2116static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2117{
2118 napi_gro_receive(&ring->tqp_vector->napi, skb);
2119}
2120
5b5455a9
PL		 2121static u16 hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2122 struct hns3_desc *desc, u32 l234info)
2123{
2124 struct pci_dev *pdev = ring->tqp->handle->pdev;
2125 u16 vlan_tag;
2126
2127 if (pdev->revision == 0x20) {
2128 vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2129 if (!(vlan_tag & VLAN_VID_MASK))
2130 vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2131
2132 return vlan_tag;
2133 }
2134
2135#define HNS3_STRP_OUTER_VLAN 0x1
2136#define HNS3_STRP_INNER_VLAN 0x2
2137
2138 switch (hnae_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2139 HNS3_RXD_STRP_TAGP_S)) {
2140 case HNS3_STRP_OUTER_VLAN:
2141 vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2142 break;
2143 case HNS3_STRP_INNER_VLAN:
2144 vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2145 break;
2146 default:
2147 vlan_tag = 0;
2148 break;
2149 }
2150
2151 return vlan_tag;
2152}
2153
76ad4f0e
S		 2154static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2155 struct sk_buff **out_skb, int *out_bnum)
2156{
2157 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2158 struct hns3_desc_cb *desc_cb;
2159 struct hns3_desc *desc;
2160 struct sk_buff *skb;
2161 unsigned char *va;
2162 u32 bd_base_info;
2163 int pull_len;
2164 u32 l234info;
2165 int length;
2166 int bnum;
2167
2168 desc = &ring->desc[ring->next_to_clean];
2169 desc_cb = &ring->desc_cb[ring->next_to_clean];
2170
2171 prefetch(desc);
2172
846fcc83 2173 length = le16_to_cpu(desc->rx.size);
76ad4f0e 2174 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
76ad4f0e
S		 2175
2176 /* Check valid BD */
2177 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2178 return -EFAULT;
2179
2180 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2181
2182 /* Prefetch first cache line of first page
2183 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2184 * line size is 64B so need to prefetch twice to make it 128B. But in
2185 * actual we can have greater size of caches with 128B Level 1 cache
2186 * lines. In such a case, single fetch would suffice to cache in the
2187 * relevant part of the header.
2188 */
2189 prefetch(va);
2190#if L1_CACHE_BYTES < 128
2191 prefetch(va + L1_CACHE_BYTES);
2192#endif
2193
2194 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2195 HNS3_RX_HEAD_SIZE);
2196 if (unlikely(!skb)) {
2197 netdev_err(netdev, "alloc rx skb fail\n");
2198
2199 u64_stats_update_begin(&ring->syncp);
2200 ring->stats.sw_err_cnt++;
2201 u64_stats_update_end(&ring->syncp);
2202
2203 return -ENOMEM;
2204 }
2205
2206 prefetchw(skb->data);
2207
2208 bnum = 1;
2209 if (length <= HNS3_RX_HEAD_SIZE) {
2210 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2211
2212 /* We can reuse buffer as-is, just make sure it is local */
2213 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2214 desc_cb->reuse_flag = 1;
2215 else /* This page cannot be reused so discard it */
2216 put_page(desc_cb->priv);
2217
2218 ring_ptr_move_fw(ring, next_to_clean);
2219 } else {
2220 u64_stats_update_begin(&ring->syncp);
2221 ring->stats.seg_pkt_cnt++;
2222 u64_stats_update_end(&ring->syncp);
2223
e63cd65f
PL		 2224 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2225
76ad4f0e
S		 2226 memcpy(__skb_put(skb, pull_len), va,
2227 ALIGN(pull_len, sizeof(long)));
2228
2229 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2230 ring_ptr_move_fw(ring, next_to_clean);
2231
2232 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2233 desc = &ring->desc[ring->next_to_clean];
2234 desc_cb = &ring->desc_cb[ring->next_to_clean];
2235 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2236 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2237 ring_ptr_move_fw(ring, next_to_clean);
2238 bnum++;
2239 }
2240 }
2241
2242 *out_bnum = bnum;
5b5455a9
PL		 2243
2244 l234info = le32_to_cpu(desc->rx.l234_info);
2245
846fcc83
PL		 2246 /* Based on hw strategy, the tag offloaded will be stored at
2247 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2248 * in one layer tag case.
2249 */
2250 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2251 u16 vlan_tag;
2252
5b5455a9 2253 vlan_tag = hns3_parse_vlan_tag(ring, desc, l234info);
846fcc83
PL		 2254 if (vlan_tag & VLAN_VID_MASK)
2255 __vlan_hwaccel_put_tag(skb,
2256 htons(ETH_P_8021Q),
2257 vlan_tag);
2258 }
2259
76ad4f0e
S		 2260 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2261 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2262 ((u64 *)desc)[0], ((u64 *)desc)[1]);
2263 u64_stats_update_begin(&ring->syncp);
2264 ring->stats.non_vld_descs++;
2265 u64_stats_update_end(&ring->syncp);
2266
2267 dev_kfree_skb_any(skb);
2268 return -EINVAL;
2269 }
2270
2271 if (unlikely((!desc->rx.pkt_len) ||
2272 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2273 netdev_err(netdev, "truncated pkt\n");
2274 u64_stats_update_begin(&ring->syncp);
2275 ring->stats.err_pkt_len++;
2276 u64_stats_update_end(&ring->syncp);
2277
2278 dev_kfree_skb_any(skb);
2279 return -EFAULT;
2280 }
2281
2282 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2283 netdev_err(netdev, "L2 error pkt\n");
2284 u64_stats_update_begin(&ring->syncp);
2285 ring->stats.l2_err++;
2286 u64_stats_update_end(&ring->syncp);
2287
2288 dev_kfree_skb_any(skb);
2289 return -EFAULT;
2290 }
2291
2292 u64_stats_update_begin(&ring->syncp);
2293 ring->stats.rx_pkts++;
2294 ring->stats.rx_bytes += skb->len;
2295 u64_stats_update_end(&ring->syncp);
2296
2297 ring->tqp_vector->rx_group.total_bytes += skb->len;
2298
2299 hns3_rx_checksum(ring, skb, desc);
2300 return 0;
2301}
2302
d43e5aca
YL		 2303int hns3_clean_rx_ring(
2304 struct hns3_enet_ring *ring, int budget,
2305 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2306{
2307#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2308 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2309 int recv_pkts, recv_bds, clean_count, err;
2310 int unused_count = hns3_desc_unused(ring);
2311 struct sk_buff *skb = NULL;
2312 int num, bnum = 0;
2313
2314 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2315 rmb(); /* Make sure num taken effect before the other data is touched */
2316
2317 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2318 num -= unused_count;
2319
2320 while (recv_pkts < budget && recv_bds < num) {
2321 /* Reuse or realloc buffers */
2322 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2323 hns3_nic_alloc_rx_buffers(ring,
2324 clean_count + unused_count);
2325 clean_count = 0;
2326 unused_count = hns3_desc_unused(ring);
2327 }
2328
2329 /* Poll one pkt */
2330 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2331 if (unlikely(!skb)) /* This fault cannot be repaired */
2332 goto out;
2333
2334 recv_bds += bnum;
2335 clean_count += bnum;
2336 if (unlikely(err)) { /* Do jump the err */
2337 recv_pkts++;
2338 continue;
2339 }
2340
2341 /* Do update ip stack process */
2342 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2343 rx_fn(ring, skb);
76ad4f0e
S		 2344
2345 recv_pkts++;
2346 }
2347
2348out:
2349 /* Make all data has been write before submit */
2350 if (clean_count + unused_count > 0)
2351 hns3_nic_alloc_rx_buffers(ring,
2352 clean_count + unused_count);
2353
2354 return recv_pkts;
2355}
2356
2357static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2358{
a95e1f86
FL		 2359 struct hns3_enet_tqp_vector *tqp_vector =
2360 ring_group->ring->tqp_vector;
76ad4f0e 2361 enum hns3_flow_level_range new_flow_level;
a95e1f86
FL		 2362 int packets_per_msecs;
2363 int bytes_per_msecs;
2364 u32 time_passed_ms;
76ad4f0e 2365 u16 new_int_gl;
76ad4f0e 2366
a95e1f86 2367 if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
76ad4f0e
S		 2368 return false;
2369
2370 if (ring_group->total_packets == 0) {
9bc727a9
YL		 2371 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2372 ring_group->coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 2373 return true;
2374 }
2375
2376 /* Simple throttlerate management
2377 * 0-10MB/s lower (50000 ints/s)
2378 * 10-20MB/s middle (20000 ints/s)
2379 * 20-1249MB/s high (18000 ints/s)
2380 * > 40000pps ultra (8000 ints/s)
2381 */
9bc727a9
YL		 2382 new_flow_level = ring_group->coal.flow_level;
2383 new_int_gl = ring_group->coal.int_gl;
a95e1f86
FL		 2384 time_passed_ms =
2385 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2386
2387 if (!time_passed_ms)
2388 return false;
2389
2390 do_div(ring_group->total_packets, time_passed_ms);
2391 packets_per_msecs = ring_group->total_packets;
2392
2393 do_div(ring_group->total_bytes, time_passed_ms);
2394 bytes_per_msecs = ring_group->total_bytes;
2395
2396#define HNS3_RX_LOW_BYTE_RATE 10000
2397#define HNS3_RX_MID_BYTE_RATE 20000
76ad4f0e
S		 2398
2399 switch (new_flow_level) {
2400 case HNS3_FLOW_LOW:
a95e1f86 2401 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2402 new_flow_level = HNS3_FLOW_MID;
2403 break;
2404 case HNS3_FLOW_MID:
a95e1f86 2405 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
76ad4f0e 2406 new_flow_level = HNS3_FLOW_HIGH;
a95e1f86 2407 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2408 new_flow_level = HNS3_FLOW_LOW;
2409 break;
2410 case HNS3_FLOW_HIGH:
2411 case HNS3_FLOW_ULTRA:
2412 default:
a95e1f86 2413 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
76ad4f0e
S		 2414 new_flow_level = HNS3_FLOW_MID;
2415 break;
2416 }
2417
a95e1f86
FL		 2418#define HNS3_RX_ULTRA_PACKET_RATE 40
2419
2420 if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2421 &tqp_vector->rx_group == ring_group)
76ad4f0e
S		 2422 new_flow_level = HNS3_FLOW_ULTRA;
2423
2424 switch (new_flow_level) {
2425 case HNS3_FLOW_LOW:
2426 new_int_gl = HNS3_INT_GL_50K;
2427 break;
2428 case HNS3_FLOW_MID:
2429 new_int_gl = HNS3_INT_GL_20K;
2430 break;
2431 case HNS3_FLOW_HIGH:
2432 new_int_gl = HNS3_INT_GL_18K;
2433 break;
2434 case HNS3_FLOW_ULTRA:
2435 new_int_gl = HNS3_INT_GL_8K;
2436 break;
2437 default:
2438 break;
2439 }
2440
2441 ring_group->total_bytes = 0;
2442 ring_group->total_packets = 0;
9bc727a9
YL		 2443 ring_group->coal.flow_level = new_flow_level;
2444 if (new_int_gl != ring_group->coal.int_gl) {
2445 ring_group->coal.int_gl = new_int_gl;
76ad4f0e
S		 2446 return true;
2447 }
2448 return false;
2449}
2450
2451static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2452{
8b1ff1ea
FL		 2453 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2454 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2455 bool rx_update, tx_update;
2456
cd9d187b
FL		 2457 if (tqp_vector->int_adapt_down > 0) {
2458 tqp_vector->int_adapt_down--;
2459 return;
2460 }
2461
9bc727a9 2462 if (rx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2463 rx_update = hns3_get_new_int_gl(rx_group);
2464 if (rx_update)
2465 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 2466 rx_group->coal.int_gl);
8b1ff1ea
FL		 2467 }
2468
9bc727a9 2469 if (tx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2470 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2471 if (tx_update)
2472 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 2473 tx_group->coal.int_gl);
76ad4f0e 2474 }
cd9d187b 2475
a95e1f86 2476 tqp_vector->last_jiffies = jiffies;
cd9d187b 2477 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
76ad4f0e
S		 2478}
2479
2480static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2481{
2482 struct hns3_enet_ring *ring;
2483 int rx_pkt_total = 0;
2484
2485 struct hns3_enet_tqp_vector *tqp_vector =
2486 container_of(napi, struct hns3_enet_tqp_vector, napi);
2487 bool clean_complete = true;
2488 int rx_budget;
2489
2490 /* Since the actual Tx work is minimal, we can give the Tx a larger
2491 * budget and be more aggressive about cleaning up the Tx descriptors.
2492 */
2493 hns3_for_each_ring(ring, tqp_vector->tx_group) {
2494 if (!hns3_clean_tx_ring(ring, budget))
2495 clean_complete = false;
2496 }
2497
2498 /* make sure rx ring budget not smaller than 1 */
2499 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2500
2501 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2502 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2503 hns3_rx_skb);
76ad4f0e
S		 2504
2505 if (rx_cleaned >= rx_budget)
2506 clean_complete = false;
2507
2508 rx_pkt_total += rx_cleaned;
2509 }
2510
2511 tqp_vector->rx_group.total_packets += rx_pkt_total;
2512
2513 if (!clean_complete)
2514 return budget;
2515
2516 napi_complete(napi);
2517 hns3_update_new_int_gl(tqp_vector);
2518 hns3_mask_vector_irq(tqp_vector, 1);
2519
2520 return rx_pkt_total;
2521}
2522
2523static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2524 struct hnae3_ring_chain_node *head)
2525{
2526 struct pci_dev *pdev = tqp_vector->handle->pdev;
2527 struct hnae3_ring_chain_node *cur_chain = head;
2528 struct hnae3_ring_chain_node *chain;
2529 struct hns3_enet_ring *tx_ring;
2530 struct hns3_enet_ring *rx_ring;
2531
2532 tx_ring = tqp_vector->tx_group.ring;
2533 if (tx_ring) {
2534 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2535 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2536 HNAE3_RING_TYPE_TX);
11af96a4
FL		 2537 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2538 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 2539
2540 cur_chain->next = NULL;
2541
2542 while (tx_ring->next) {
2543 tx_ring = tx_ring->next;
2544
2545 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2546 GFP_KERNEL);
2547 if (!chain)
2548 return -ENOMEM;
2549
2550 cur_chain->next = chain;
2551 chain->tqp_index = tx_ring->tqp->tqp_index;
2552 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2553 HNAE3_RING_TYPE_TX);
11af96a4
FL		 2554 hnae_set_field(chain->int_gl_idx,
2555 HNAE3_RING_GL_IDX_M,
2556 HNAE3_RING_GL_IDX_S,
2557 HNAE3_RING_GL_TX);
76ad4f0e
S		 2558
2559 cur_chain = chain;
2560 }
2561 }
2562
2563 rx_ring = tqp_vector->rx_group.ring;
2564 if (!tx_ring && rx_ring) {
2565 cur_chain->next = NULL;
2566 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2567 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2568 HNAE3_RING_TYPE_RX);
11af96a4
FL		 2569 hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2570 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 2571
2572 rx_ring = rx_ring->next;
2573 }
2574
2575 while (rx_ring) {
2576 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2577 if (!chain)
2578 return -ENOMEM;
2579
2580 cur_chain->next = chain;
2581 chain->tqp_index = rx_ring->tqp->tqp_index;
2582 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2583 HNAE3_RING_TYPE_RX);
11af96a4
FL		 2584 hnae_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2585 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2586
76ad4f0e
S		 2587 cur_chain = chain;
2588
2589 rx_ring = rx_ring->next;
2590 }
2591
2592 return 0;
2593}
2594
2595static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2596 struct hnae3_ring_chain_node *head)
2597{
2598 struct pci_dev *pdev = tqp_vector->handle->pdev;
2599 struct hnae3_ring_chain_node *chain_tmp, *chain;
2600
2601 chain = head->next;
2602
2603 while (chain) {
2604 chain_tmp = chain->next;
2605 devm_kfree(&pdev->dev, chain);
2606 chain = chain_tmp;
2607 }
2608}
2609
2610static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2611 struct hns3_enet_ring *ring)
2612{
2613 ring->next = group->ring;
2614 group->ring = ring;
2615
2616 group->count++;
2617}
2618
2619static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2620{
2621 struct hnae3_ring_chain_node vector_ring_chain;
2622 struct hnae3_handle *h = priv->ae_handle;
2623 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 2624 int ret = 0;
2625 u16 i;
2626
dd38c726
YL		 2627 for (i = 0; i < priv->vector_num; i++) {
2628 tqp_vector = &priv->tqp_vector[i];
2629 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2630 tqp_vector->num_tqps = 0;
2631 }
76ad4f0e 2632
dd38c726
YL		 2633 for (i = 0; i < h->kinfo.num_tqps; i++) {
2634 u16 vector_i = i % priv->vector_num;
2635 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 2636
2637 tqp_vector = &priv->tqp_vector[vector_i];
2638
2639 hns3_add_ring_to_group(&tqp_vector->tx_group,
2640 priv->ring_data[i].ring);
2641
2642 hns3_add_ring_to_group(&tqp_vector->rx_group,
2643 priv->ring_data[i + tqp_num].ring);
2644
76ad4f0e
S		 2645 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2646 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
dd38c726 2647 tqp_vector->num_tqps++;
76ad4f0e
S		 2648 }
2649
dd38c726 2650 for (i = 0; i < priv->vector_num; i++) {
76ad4f0e
S		 2651 tqp_vector = &priv->tqp_vector[i];
2652
2653 tqp_vector->rx_group.total_bytes = 0;
2654 tqp_vector->rx_group.total_packets = 0;
2655 tqp_vector->tx_group.total_bytes = 0;
2656 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 2657 tqp_vector->handle = h;
2658
2659 ret = hns3_get_vector_ring_chain(tqp_vector,
2660 &vector_ring_chain);
2661 if (ret)
dd38c726 2662 return ret;
76ad4f0e
S		 2663
2664 ret = h->ae_algo->ops->map_ring_to_vector(h,
2665 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 2666
2667 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2668
dd38c726
YL		 2669 if (ret)
2670 return ret;
2671
76ad4f0e
S		 2672 netif_napi_add(priv->netdev, &tqp_vector->napi,
2673 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2674 }
2675
dd38c726
YL		 2676 return 0;
2677}
2678
2679static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2680{
2681 struct hnae3_handle *h = priv->ae_handle;
2682 struct hns3_enet_tqp_vector *tqp_vector;
2683 struct hnae3_vector_info *vector;
2684 struct pci_dev *pdev = h->pdev;
2685 u16 tqp_num = h->kinfo.num_tqps;
2686 u16 vector_num;
2687 int ret = 0;
2688 u16 i;
2689
2690 /* RSS size, cpu online and vector_num should be the same */
2691 /* Should consider 2p/4p later */
2692 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2693 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2694 GFP_KERNEL);
2695 if (!vector)
2696 return -ENOMEM;
2697
2698 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2699
2700 priv->vector_num = vector_num;
2701 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2702 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2703 GFP_KERNEL);
2704 if (!priv->tqp_vector) {
2705 ret = -ENOMEM;
2706 goto out;
2707 }
2708
2709 for (i = 0; i < priv->vector_num; i++) {
2710 tqp_vector = &priv->tqp_vector[i];
2711 tqp_vector->idx = i;
2712 tqp_vector->mask_addr = vector[i].io_addr;
2713 tqp_vector->vector_irq = vector[i].vector;
2714 hns3_vector_gl_rl_init(tqp_vector, priv);
2715 }
2716
76ad4f0e
S		 2717out:
2718 devm_kfree(&pdev->dev, vector);
2719 return ret;
2720}
2721
dd38c726
YL		 2722static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2723{
2724 group->ring = NULL;
2725 group->count = 0;
2726}
2727
76ad4f0e
S		 2728static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2729{
2730 struct hnae3_ring_chain_node vector_ring_chain;
2731 struct hnae3_handle *h = priv->ae_handle;
2732 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 2733 int i, ret;
2734
2735 for (i = 0; i < priv->vector_num; i++) {
2736 tqp_vector = &priv->tqp_vector[i];
2737
2738 ret = hns3_get_vector_ring_chain(tqp_vector,
2739 &vector_ring_chain);
2740 if (ret)
2741 return ret;
2742
2743 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2744 tqp_vector->vector_irq, &vector_ring_chain);
2745 if (ret)
2746 return ret;
2747
0d3e6631
YL		 2748 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2749 if (ret)
2750 return ret;
2751
76ad4f0e
S		 2752 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2753
2754 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2755 (void)irq_set_affinity_hint(
2756 priv->tqp_vector[i].vector_irq,
2757 NULL);
ae064e61 2758 free_irq(priv->tqp_vector[i].vector_irq,
2759 &priv->tqp_vector[i]);
76ad4f0e
S		 2760 }
2761
2762 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
dd38c726
YL		 2763 hns3_clear_ring_group(&tqp_vector->rx_group);
2764 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 2765 netif_napi_del(&priv->tqp_vector[i].napi);
2766 }
2767
dd38c726
YL		 2768 return 0;
2769}
2770
2771static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2772{
2773 struct hnae3_handle *h = priv->ae_handle;
2774 struct pci_dev *pdev = h->pdev;
2775 int i, ret;
2776
2777 for (i = 0; i < priv->vector_num; i++) {
2778 struct hns3_enet_tqp_vector *tqp_vector;
2779
2780 tqp_vector = &priv->tqp_vector[i];
2781 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2782 if (ret)
2783 return ret;
2784 }
76ad4f0e 2785
dd38c726 2786 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 2787 return 0;
2788}
2789
2790static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2791 int ring_type)
2792{
2793 struct hns3_nic_ring_data *ring_data = priv->ring_data;
2794 int queue_num = priv->ae_handle->kinfo.num_tqps;
2795 struct pci_dev *pdev = priv->ae_handle->pdev;
2796 struct hns3_enet_ring *ring;
2797
2798 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2799 if (!ring)
2800 return -ENOMEM;
2801
2802 if (ring_type == HNAE3_RING_TYPE_TX) {
2803 ring_data[q->tqp_index].ring = ring;
66b44730 2804 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 2805 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2806 } else {
2807 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 2808 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 2809 ring->io_base = q->io_base;
2810 }
2811
2812 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2813
76ad4f0e
S		 2814 ring->tqp = q;
2815 ring->desc = NULL;
2816 ring->desc_cb = NULL;
2817 ring->dev = priv->dev;
2818 ring->desc_dma_addr = 0;
2819 ring->buf_size = q->buf_size;
2820 ring->desc_num = q->desc_num;
2821 ring->next_to_use = 0;
2822 ring->next_to_clean = 0;
2823
2824 return 0;
2825}
2826
2827static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2828 struct hns3_nic_priv *priv)
2829{
2830 int ret;
2831
2832 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2833 if (ret)
2834 return ret;
2835
2836 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2837 if (ret)
2838 return ret;
2839
2840 return 0;
2841}
2842
2843static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2844{
2845 struct hnae3_handle *h = priv->ae_handle;
2846 struct pci_dev *pdev = h->pdev;
2847 int i, ret;
2848
2849 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2850 sizeof(*priv->ring_data) * 2,
2851 GFP_KERNEL);
2852 if (!priv->ring_data)
2853 return -ENOMEM;
2854
2855 for (i = 0; i < h->kinfo.num_tqps; i++) {
2856 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2857 if (ret)
2858 goto err;
2859 }
2860
2861 return 0;
2862err:
2863 devm_kfree(&pdev->dev, priv->ring_data);
2864 return ret;
2865}
2866
09f2af64
PL		 2867static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2868{
2869 struct hnae3_handle *h = priv->ae_handle;
2870 int i;
2871
2872 for (i = 0; i < h->kinfo.num_tqps; i++) {
2873 devm_kfree(priv->dev, priv->ring_data[i].ring);
2874 devm_kfree(priv->dev,
2875 priv->ring_data[i + h->kinfo.num_tqps].ring);
2876 }
2877 devm_kfree(priv->dev, priv->ring_data);
2878}
2879
76ad4f0e
S		 2880static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2881{
2882 int ret;
2883
2884 if (ring->desc_num <= 0 || ring->buf_size <= 0)
2885 return -EINVAL;
2886
2887 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2888 GFP_KERNEL);
2889 if (!ring->desc_cb) {
2890 ret = -ENOMEM;
2891 goto out;
2892 }
2893
2894 ret = hns3_alloc_desc(ring);
2895 if (ret)
2896 goto out_with_desc_cb;
2897
2898 if (!HNAE3_IS_TX_RING(ring)) {
2899 ret = hns3_alloc_ring_buffers(ring);
2900 if (ret)
2901 goto out_with_desc;
2902 }
2903
2904 return 0;
2905
2906out_with_desc:
2907 hns3_free_desc(ring);
2908out_with_desc_cb:
2909 kfree(ring->desc_cb);
2910 ring->desc_cb = NULL;
2911out:
2912 return ret;
2913}
2914
2915static void hns3_fini_ring(struct hns3_enet_ring *ring)
2916{
2917 hns3_free_desc(ring);
2918 kfree(ring->desc_cb);
2919 ring->desc_cb = NULL;
2920 ring->next_to_clean = 0;
2921 ring->next_to_use = 0;
2922}
2923
1db9b1bf 2924static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 2925{
2926 int bd_size_type;
2927
2928 switch (buf_size) {
2929 case 512:
2930 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2931 break;
2932 case 1024:
2933 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2934 break;
2935 case 2048:
2936 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2937 break;
2938 case 4096:
2939 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2940 break;
2941 default:
2942 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2943 }
2944
2945 return bd_size_type;
2946}
2947
2948static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2949{
2950 dma_addr_t dma = ring->desc_dma_addr;
2951 struct hnae3_queue *q = ring->tqp;
2952
2953 if (!HNAE3_IS_TX_RING(ring)) {
2954 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2955 (u32)dma);
2956 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2957 (u32)((dma >> 31) >> 1));
2958
2959 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2960 hns3_buf_size2type(ring->buf_size));
2961 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2962 ring->desc_num / 8 - 1);
2963
2964 } else {
2965 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2966 (u32)dma);
2967 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2968 (u32)((dma >> 31) >> 1));
2969
2970 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2971 hns3_buf_size2type(ring->buf_size));
2972 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2973 ring->desc_num / 8 - 1);
2974 }
2975}
2976
5668abda 2977int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2978{
2979 struct hnae3_handle *h = priv->ae_handle;
2980 int ring_num = h->kinfo.num_tqps * 2;
2981 int i, j;
2982 int ret;
2983
2984 for (i = 0; i < ring_num; i++) {
2985 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2986 if (ret) {
2987 dev_err(priv->dev,
2988 "Alloc ring memory fail! ret=%d\n", ret);
2989 goto out_when_alloc_ring_memory;
2990 }
2991
76ad4f0e
S		 2992 u64_stats_init(&priv->ring_data[i].ring->syncp);
2993 }
2994
2995 return 0;
2996
2997out_when_alloc_ring_memory:
2998 for (j = i - 1; j >= 0; j--)
ee83f776 2999 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 3000
3001 return -ENOMEM;
3002}
3003
5668abda 3004int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3005{
3006 struct hnae3_handle *h = priv->ae_handle;
3007 int i;
3008
3009 for (i = 0; i < h->kinfo.num_tqps; i++) {
3010 if (h->ae_algo->ops->reset_queue)
3011 h->ae_algo->ops->reset_queue(h, i);
3012
3013 hns3_fini_ring(priv->ring_data[i].ring);
3014 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3015 }
76ad4f0e
S		 3016 return 0;
3017}
3018
3019/* Set mac addr if it is configured. or leave it to the AE driver */
f09555ff 3020static void hns3_init_mac_addr(struct net_device *netdev, bool init)
76ad4f0e
S		 3021{
3022 struct hns3_nic_priv *priv = netdev_priv(netdev);
3023 struct hnae3_handle *h = priv->ae_handle;
3024 u8 mac_addr_temp[ETH_ALEN];
3025
f09555ff 3026 if (h->ae_algo->ops->get_mac_addr && init) {
76ad4f0e
S		 3027 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3028 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3029 }
3030
3031 /* Check if the MAC address is valid, if not get a random one */
3032 if (!is_valid_ether_addr(netdev->dev_addr)) {
3033 eth_hw_addr_random(netdev);
3034 dev_warn(priv->dev, "using random MAC address %pM\n",
3035 netdev->dev_addr);
76ad4f0e 3036 }
139e8792
L		 3037
3038 if (h->ae_algo->ops->set_mac_addr)
59098055 3039 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
139e8792 3040
76ad4f0e
S		 3041}
3042
c7fc8fb6
JS		 3043static void hns3_uninit_mac_addr(struct net_device *netdev)
3044{
3045 struct hns3_nic_priv *priv = netdev_priv(netdev);
3046 struct hnae3_handle *h = priv->ae_handle;
3047
3048 if (h->ae_algo->ops->rm_uc_addr)
3049 h->ae_algo->ops->rm_uc_addr(h, netdev->dev_addr);
3050}
3051
76ad4f0e
S		 3052static void hns3_nic_set_priv_ops(struct net_device *netdev)
3053{
3054 struct hns3_nic_priv *priv = netdev_priv(netdev);
3055
3056 if ((netdev->features & NETIF_F_TSO) ||
3057 (netdev->features & NETIF_F_TSO6)) {
3058 priv->ops.fill_desc = hns3_fill_desc_tso;
3059 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3060 } else {
3061 priv->ops.fill_desc = hns3_fill_desc;
3062 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3063 }
3064}
3065
3066static int hns3_client_init(struct hnae3_handle *handle)
3067{
3068 struct pci_dev *pdev = handle->pdev;
3069 struct hns3_nic_priv *priv;
3070 struct net_device *netdev;
3071 int ret;
3072
3073 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
678335a1 3074 hns3_get_max_available_channels(handle));
76ad4f0e
S		 3075 if (!netdev)
3076 return -ENOMEM;
3077
3078 priv = netdev_priv(netdev);
3079 priv->dev = &pdev->dev;
3080 priv->netdev = netdev;
3081 priv->ae_handle = handle;
6d4c3981
SM		 3082 priv->ae_handle->reset_level = HNAE3_NONE_RESET;
3083 priv->ae_handle->last_reset_time = jiffies;
f8fa222c 3084 priv->tx_timeout_count = 0;
76ad4f0e
S		 3085
3086 handle->kinfo.netdev = netdev;
3087 handle->priv = (void *)priv;
3088
f09555ff 3089 hns3_init_mac_addr(netdev, true);
76ad4f0e
S		 3090
3091 hns3_set_default_feature(netdev);
3092
3093 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3094 netdev->priv_flags |= IFF_UNICAST_FLT;
3095 netdev->netdev_ops = &hns3_nic_netdev_ops;
3096 SET_NETDEV_DEV(netdev, &pdev->dev);
3097 hns3_ethtool_set_ops(netdev);
3098 hns3_nic_set_priv_ops(netdev);
3099
3100 /* Carrier off reporting is important to ethtool even BEFORE open */
3101 netif_carrier_off(netdev);
3102
3103 ret = hns3_get_ring_config(priv);
3104 if (ret) {
3105 ret = -ENOMEM;
3106 goto out_get_ring_cfg;
3107 }
3108
dd38c726
YL		 3109 ret = hns3_nic_alloc_vector_data(priv);
3110 if (ret) {
3111 ret = -ENOMEM;
3112 goto out_alloc_vector_data;
3113 }
3114
76ad4f0e
S		 3115 ret = hns3_nic_init_vector_data(priv);
3116 if (ret) {
3117 ret = -ENOMEM;
3118 goto out_init_vector_data;
3119 }
3120
3121 ret = hns3_init_all_ring(priv);
3122 if (ret) {
3123 ret = -ENOMEM;
3124 goto out_init_ring_data;
3125 }
3126
3127 ret = register_netdev(netdev);
3128 if (ret) {
3129 dev_err(priv->dev, "probe register netdev fail!\n");
3130 goto out_reg_netdev_fail;
3131 }
3132
986743db
YL		 3133 hns3_dcbnl_setup(handle);
3134
a8e8b7ff
S		 3135 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3136 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3137
76ad4f0e
S		 3138 return ret;
3139
3140out_reg_netdev_fail:
3141out_init_ring_data:
3142 (void)hns3_nic_uninit_vector_data(priv);
76ad4f0e 3143out_init_vector_data:
dd38c726
YL		 3144 hns3_nic_dealloc_vector_data(priv);
3145out_alloc_vector_data:
3146 priv->ring_data = NULL;
76ad4f0e
S		 3147out_get_ring_cfg:
3148 priv->ae_handle = NULL;
3149 free_netdev(netdev);
3150 return ret;
3151}
3152
3153static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3154{
3155 struct net_device *netdev = handle->kinfo.netdev;
3156 struct hns3_nic_priv *priv = netdev_priv(netdev);
3157 int ret;
3158
3159 if (netdev->reg_state != NETREG_UNINITIALIZED)
3160 unregister_netdev(netdev);
3161
7b763f3f
FL		 3162 hns3_force_clear_all_rx_ring(handle);
3163
76ad4f0e
S		 3164 ret = hns3_nic_uninit_vector_data(priv);
3165 if (ret)
3166 netdev_err(netdev, "uninit vector error\n");
3167
dd38c726
YL		 3168 ret = hns3_nic_dealloc_vector_data(priv);
3169 if (ret)
3170 netdev_err(netdev, "dealloc vector error\n");
3171
76ad4f0e
S		 3172 ret = hns3_uninit_all_ring(priv);
3173 if (ret)
3174 netdev_err(netdev, "uninit ring error\n");
3175
ec777890
YL		 3176 hns3_put_ring_config(priv);
3177
76ad4f0e
S		 3178 priv->ring_data = NULL;
3179
c7fc8fb6
JS		 3180 hns3_uninit_mac_addr(netdev);
3181
76ad4f0e
S		 3182 free_netdev(netdev);
3183}
3184
3185static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3186{
3187 struct net_device *netdev = handle->kinfo.netdev;
3188
3189 if (!netdev)
3190 return;
3191
3192 if (linkup) {
3193 netif_carrier_on(netdev);
3194 netif_tx_wake_all_queues(netdev);
3195 netdev_info(netdev, "link up\n");
3196 } else {
3197 netif_carrier_off(netdev);
3198 netif_tx_stop_all_queues(netdev);
3199 netdev_info(netdev, "link down\n");
3200 }
3201}
3202
9df8f79a
YL		 3203static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3204{
3205 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3206 struct net_device *ndev = kinfo->netdev;
075cfdd6 3207 bool if_running;
9df8f79a
YL		 3208 int ret;
3209 u8 i;
3210
3211 if (tc > HNAE3_MAX_TC)
3212 return -EINVAL;
3213
3214 if (!ndev)
3215 return -ENODEV;
3216
075cfdd6
CIK		 3217 if_running = netif_running(ndev);
3218
9df8f79a
YL		 3219 ret = netdev_set_num_tc(ndev, tc);
3220 if (ret)
3221 return ret;
3222
3223 if (if_running) {
3224 (void)hns3_nic_net_stop(ndev);
3225 msleep(100);
3226 }
3227
3228 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3229 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3230 if (ret)
3231 goto err_out;
3232
3233 if (tc <= 1) {
3234 netdev_reset_tc(ndev);
3235 goto out;
3236 }
3237
3238 for (i = 0; i < HNAE3_MAX_TC; i++) {
3239 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3240
3241 if (tc_info->enable)
3242 netdev_set_tc_queue(ndev,
3243 tc_info->tc,
3244 tc_info->tqp_count,
3245 tc_info->tqp_offset);
3246 }
3247
3248 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
3249 netdev_set_prio_tc_map(ndev, i,
3250 kinfo->prio_tc[i]);
3251 }
3252
3253out:
3254 ret = hns3_nic_set_real_num_queue(ndev);
3255
3256err_out:
3257 if (if_running)
3258 (void)hns3_nic_net_open(ndev);
3259
3260 return ret;
3261}
3262
bb6b94a8
L		 3263static void hns3_recover_hw_addr(struct net_device *ndev)
3264{
3265 struct netdev_hw_addr_list *list;
3266 struct netdev_hw_addr *ha, *tmp;
3267
3268 /* go through and sync uc_addr entries to the device */
3269 list = &ndev->uc;
3270 list_for_each_entry_safe(ha, tmp, &list->list, list)
3271 hns3_nic_uc_sync(ndev, ha->addr);
3272
3273 /* go through and sync mc_addr entries to the device */
3274 list = &ndev->mc;
3275 list_for_each_entry_safe(ha, tmp, &list->list, list)
3276 hns3_nic_mc_sync(ndev, ha->addr);
3277}
3278
beebca3a 3279static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
bb6b94a8 3280{
beebca3a 3281 while (ring->next_to_clean != ring->next_to_use) {
7b763f3f 3282 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
beebca3a
YL		 3283 hns3_free_buffer_detach(ring, ring->next_to_clean);
3284 ring_ptr_move_fw(ring, next_to_clean);
3285 }
3286}
3287
7b763f3f
FL		 3288static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3289{
3290 struct hns3_desc_cb res_cbs;
3291 int ret;
3292
3293 while (ring->next_to_use != ring->next_to_clean) {
3294 /* When a buffer is not reused, it's memory has been
3295 * freed in hns3_handle_rx_bd or will be freed by
3296 * stack, so we need to replace the buffer here.
3297 */
3298 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3299 ret = hns3_reserve_buffer_map(ring, &res_cbs);
3300 if (ret) {
3301 u64_stats_update_begin(&ring->syncp);
3302 ring->stats.sw_err_cnt++;
3303 u64_stats_update_end(&ring->syncp);
3304 /* if alloc new buffer fail, exit directly
3305 * and reclear in up flow.
3306 */
3307 netdev_warn(ring->tqp->handle->kinfo.netdev,
3308 "reserve buffer map failed, ret = %d\n",
3309 ret);
3310 return ret;
3311 }
3312 hns3_replace_buffer(ring, ring->next_to_use,
3313 &res_cbs);
3314 }
3315 ring_ptr_move_fw(ring, next_to_use);
3316 }
3317
3318 return 0;
3319}
3320
3321static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
beebca3a 3322{
beebca3a
YL		 3323 while (ring->next_to_use != ring->next_to_clean) {
3324 /* When a buffer is not reused, it's memory has been
3325 * freed in hns3_handle_rx_bd or will be freed by
3326 * stack, so only need to unmap the buffer here.
3327 */
3328 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3329 hns3_unmap_buffer(ring,
3330 &ring->desc_cb[ring->next_to_use]);
3331 ring->desc_cb[ring->next_to_use].dma = 0;
3332 }
3333
3334 ring_ptr_move_fw(ring, next_to_use);
3335 }
bb6b94a8
L		 3336}
3337
7b763f3f
FL		 3338static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3339{
3340 struct net_device *ndev = h->kinfo.netdev;
3341 struct hns3_nic_priv *priv = netdev_priv(ndev);
3342 struct hns3_enet_ring *ring;
3343 u32 i;
3344
3345 for (i = 0; i < h->kinfo.num_tqps; i++) {
3346 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3347 hns3_force_clear_rx_ring(ring);
3348 }
3349}
3350
bb6b94a8
L		 3351static void hns3_clear_all_ring(struct hnae3_handle *h)
3352{
3353 struct net_device *ndev = h->kinfo.netdev;
3354 struct hns3_nic_priv *priv = netdev_priv(ndev);
3355 u32 i;
3356
3357 for (i = 0; i < h->kinfo.num_tqps; i++) {
3358 struct netdev_queue *dev_queue;
3359 struct hns3_enet_ring *ring;
3360
3361 ring = priv->ring_data[i].ring;
beebca3a 3362 hns3_clear_tx_ring(ring);
bb6b94a8
L		 3363 dev_queue = netdev_get_tx_queue(ndev,
3364 priv->ring_data[i].queue_index);
3365 netdev_tx_reset_queue(dev_queue);
3366
3367 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
7b763f3f
FL		 3368 /* Continue to clear other rings even if clearing some
3369 * rings failed.
3370 */
beebca3a 3371 hns3_clear_rx_ring(ring);
bb6b94a8
L		 3372 }
3373}
3374
7b763f3f
FL		 3375int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3376{
3377 struct net_device *ndev = h->kinfo.netdev;
3378 struct hns3_nic_priv *priv = netdev_priv(ndev);
3379 struct hns3_enet_ring *rx_ring;
3380 int i, j;
3381 int ret;
3382
3383 for (i = 0; i < h->kinfo.num_tqps; i++) {
3384 h->ae_algo->ops->reset_queue(h, i);
3385 hns3_init_ring_hw(priv->ring_data[i].ring);
3386
3387 /* We need to clear tx ring here because self test will
3388 * use the ring and will not run down before up
3389 */
3390 hns3_clear_tx_ring(priv->ring_data[i].ring);
3391 priv->ring_data[i].ring->next_to_clean = 0;
3392 priv->ring_data[i].ring->next_to_use = 0;
3393
3394 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3395 hns3_init_ring_hw(rx_ring);
3396 ret = hns3_clear_rx_ring(rx_ring);
3397 if (ret)
3398 return ret;
3399
3400 /* We can not know the hardware head and tail when this
3401 * function is called in reset flow, so we reuse all desc.
3402 */
3403 for (j = 0; j < rx_ring->desc_num; j++)
3404 hns3_reuse_buffer(rx_ring, j);
3405
3406 rx_ring->next_to_clean = 0;
3407 rx_ring->next_to_use = 0;
3408 }
3409
3410 return 0;
3411}
3412
bb6b94a8
L		 3413static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3414{
3415 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3416 struct net_device *ndev = kinfo->netdev;
3417
3418 if (!netif_running(ndev))
3419 return -EIO;
3420
3421 return hns3_nic_net_stop(ndev);
3422}
3423
3424static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3425{
3426 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
bb6b94a8
L		 3427 int ret = 0;
3428
3429 if (netif_running(kinfo->netdev)) {
3430 ret = hns3_nic_net_up(kinfo->netdev);
3431 if (ret) {
3432 netdev_err(kinfo->netdev,
3433 "hns net up fail, ret=%d!\n", ret);
3434 return ret;
3435 }
6d4c3981 3436 handle->last_reset_time = jiffies;
bb6b94a8
L		 3437 }
3438
3439 return ret;
3440}
3441
3442static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3443{
3444 struct net_device *netdev = handle->kinfo.netdev;
3445 struct hns3_nic_priv *priv = netdev_priv(netdev);
3446 int ret;
3447
f09555ff 3448 hns3_init_mac_addr(netdev, false);
bb6b94a8
L		 3449 hns3_nic_set_rx_mode(netdev);
3450 hns3_recover_hw_addr(netdev);
3451
681ec399
YL		 3452 /* Hardware table is only clear when pf resets */
3453 if (!(handle->flags & HNAE3_SUPPORT_VF))
3454 hns3_restore_vlan(netdev);
3455
bb6b94a8
L		 3456 /* Carrier off reporting is important to ethtool even BEFORE open */
3457 netif_carrier_off(netdev);
3458
3459 ret = hns3_get_ring_config(priv);
3460 if (ret)
3461 return ret;
3462
3463 ret = hns3_nic_init_vector_data(priv);
3464 if (ret)
3465 return ret;
3466
3467 ret = hns3_init_all_ring(priv);
3468 if (ret) {
3469 hns3_nic_uninit_vector_data(priv);
3470 priv->ring_data = NULL;
3471 }
3472
3473 return ret;
3474}
3475
3476static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3477{
3478 struct net_device *netdev = handle->kinfo.netdev;
3479 struct hns3_nic_priv *priv = netdev_priv(netdev);
3480 int ret;
3481
7b763f3f 3482 hns3_force_clear_all_rx_ring(handle);
bb6b94a8
L		 3483
3484 ret = hns3_nic_uninit_vector_data(priv);
3485 if (ret) {
3486 netdev_err(netdev, "uninit vector error\n");
3487 return ret;
3488 }
3489
3490 ret = hns3_uninit_all_ring(priv);
3491 if (ret)
3492 netdev_err(netdev, "uninit ring error\n");
3493
ec777890
YL		 3494 hns3_put_ring_config(priv);
3495
bb6b94a8
L		 3496 priv->ring_data = NULL;
3497
c7fc8fb6
JS		 3498 hns3_uninit_mac_addr(netdev);
3499
bb6b94a8
L		 3500 return ret;
3501}
3502
3503static int hns3_reset_notify(struct hnae3_handle *handle,
3504 enum hnae3_reset_notify_type type)
3505{
3506 int ret = 0;
3507
3508 switch (type) {
3509 case HNAE3_UP_CLIENT:
e1586241
SM		 3510 ret = hns3_reset_notify_up_enet(handle);
3511 break;
bb6b94a8
L		 3512 case HNAE3_DOWN_CLIENT:
3513 ret = hns3_reset_notify_down_enet(handle);
3514 break;
3515 case HNAE3_INIT_CLIENT:
3516 ret = hns3_reset_notify_init_enet(handle);
3517 break;
3518 case HNAE3_UNINIT_CLIENT:
3519 ret = hns3_reset_notify_uninit_enet(handle);
3520 break;
3521 default:
3522 break;
3523 }
3524
3525 return ret;
3526}
3527
7a242b23
YL		 3528static void hns3_restore_coal(struct hns3_nic_priv *priv,
3529 struct hns3_enet_coalesce *tx,
3530 struct hns3_enet_coalesce *rx)
3531{
3532 u16 vector_num = priv->vector_num;
3533 int i;
3534
3535 for (i = 0; i < vector_num; i++) {
3536 memcpy(&priv->tqp_vector[i].tx_group.coal, tx,
3537 sizeof(struct hns3_enet_coalesce));
3538 memcpy(&priv->tqp_vector[i].rx_group.coal, rx,
3539 sizeof(struct hns3_enet_coalesce));
3540 }
3541}
3542
3543static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num,
3544 struct hns3_enet_coalesce *tx,
3545 struct hns3_enet_coalesce *rx)
09f2af64
PL		 3546{
3547 struct hns3_nic_priv *priv = netdev_priv(netdev);
3548 struct hnae3_handle *h = hns3_get_handle(netdev);
3549 int ret;
3550
3551 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3552 if (ret)
3553 return ret;
3554
3555 ret = hns3_get_ring_config(priv);
3556 if (ret)
3557 return ret;
3558
dd38c726
YL		 3559 ret = hns3_nic_alloc_vector_data(priv);
3560 if (ret)
3561 goto err_alloc_vector;
3562
7a242b23
YL		 3563 hns3_restore_coal(priv, tx, rx);
3564
09f2af64
PL		 3565 ret = hns3_nic_init_vector_data(priv);
3566 if (ret)
3567 goto err_uninit_vector;
3568
3569 ret = hns3_init_all_ring(priv);
3570 if (ret)
3571 goto err_put_ring;
3572
3573 return 0;
3574
3575err_put_ring:
3576 hns3_put_ring_config(priv);
3577err_uninit_vector:
3578 hns3_nic_uninit_vector_data(priv);
dd38c726
YL		 3579err_alloc_vector:
3580 hns3_nic_dealloc_vector_data(priv);
09f2af64
PL		 3581 return ret;
3582}
3583
3584static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3585{
3586 return (new_tqp_num / num_tc) * num_tc;
3587}
3588
3589int hns3_set_channels(struct net_device *netdev,
3590 struct ethtool_channels *ch)
3591{
3592 struct hns3_nic_priv *priv = netdev_priv(netdev);
3593 struct hnae3_handle *h = hns3_get_handle(netdev);
3594 struct hnae3_knic_private_info *kinfo = &h->kinfo;
7a242b23 3595 struct hns3_enet_coalesce tx_coal, rx_coal;
09f2af64
PL		 3596 bool if_running = netif_running(netdev);
3597 u32 new_tqp_num = ch->combined_count;
3598 u16 org_tqp_num;
3599 int ret;
3600
3601 if (ch->rx_count || ch->tx_count)
3602 return -EINVAL;
3603
678335a1 3604 if (new_tqp_num > hns3_get_max_available_channels(h) ||
09f2af64
PL		 3605 new_tqp_num < kinfo->num_tc) {
3606 dev_err(&netdev->dev,
3607 "Change tqps fail, the tqp range is from %d to %d",
3608 kinfo->num_tc,
678335a1 3609 hns3_get_max_available_channels(h));
09f2af64
PL		 3610 return -EINVAL;
3611 }
3612
3613 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3614 if (kinfo->num_tqps == new_tqp_num)
3615 return 0;
3616
3617 if (if_running)
20e4bf98 3618 hns3_nic_net_stop(netdev);
09f2af64 3619
09f2af64
PL		 3620 ret = hns3_nic_uninit_vector_data(priv);
3621 if (ret) {
3622 dev_err(&netdev->dev,
3623 "Unbind vector with tqp fail, nothing is changed");
3624 goto open_netdev;
3625 }
3626
7a242b23
YL		 3627 /* Changing the tqp num may also change the vector num,
3628 * ethtool only support setting and querying one coal
3629 * configuation for now, so save the vector 0' coal
3630 * configuation here in order to restore it.
3631 */
3632 memcpy(&tx_coal, &priv->tqp_vector[0].tx_group.coal,
3633 sizeof(struct hns3_enet_coalesce));
3634 memcpy(&rx_coal, &priv->tqp_vector[0].rx_group.coal,
3635 sizeof(struct hns3_enet_coalesce));
3636
dd38c726
YL		 3637 hns3_nic_dealloc_vector_data(priv);
3638
09f2af64 3639 hns3_uninit_all_ring(priv);
ec777890 3640 hns3_put_ring_config(priv);
09f2af64
PL		 3641
3642 org_tqp_num = h->kinfo.num_tqps;
7a242b23 3643 ret = hns3_modify_tqp_num(netdev, new_tqp_num, &tx_coal, &rx_coal);
09f2af64 3644 if (ret) {
7a242b23
YL		 3645 ret = hns3_modify_tqp_num(netdev, org_tqp_num,
3646 &tx_coal, &rx_coal);
09f2af64
PL		 3647 if (ret) {
3648 /* If revert to old tqp failed, fatal error occurred */
3649 dev_err(&netdev->dev,
3650 "Revert to old tqp num fail, ret=%d", ret);
3651 return ret;
3652 }
3653 dev_info(&netdev->dev,
3654 "Change tqp num fail, Revert to old tqp num");
3655 }
3656
3657open_netdev:
3658 if (if_running)
20e4bf98 3659 hns3_nic_net_open(netdev);
09f2af64
PL		 3660
3661 return ret;
3662}
3663
1db9b1bf 3664static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 3665 .init_instance = hns3_client_init,
3666 .uninit_instance = hns3_client_uninit,
3667 .link_status_change = hns3_link_status_change,
9df8f79a 3668 .setup_tc = hns3_client_setup_tc,
bb6b94a8 3669 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 3670};
3671
3672/* hns3_init_module - Driver registration routine
3673 * hns3_init_module is the first routine called when the driver is
3674 * loaded. All it does is register with the PCI subsystem.
3675 */
3676static int __init hns3_init_module(void)
3677{
3678 int ret;
3679
3680 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3681 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3682
3683 client.type = HNAE3_CLIENT_KNIC;
3684 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3685 hns3_driver_name);
3686
3687 client.ops = &client_ops;
3688
13562d1f
XW		 3689 INIT_LIST_HEAD(&client.node);
3690
76ad4f0e
S		 3691 ret = hnae3_register_client(&client);
3692 if (ret)
3693 return ret;
3694
3695 ret = pci_register_driver(&hns3_driver);
3696 if (ret)
3697 hnae3_unregister_client(&client);
3698
3699 return ret;
3700}
3701module_init(hns3_init_module);
3702
3703/* hns3_exit_module - Driver exit cleanup routine
3704 * hns3_exit_module is called just before the driver is removed
3705 * from memory.
3706 */
3707static void __exit hns3_exit_module(void)
3708{
3709 pci_unregister_driver(&hns3_driver);
3710 hnae3_unregister_client(&client);
3711}
3712module_exit(hns3_exit_module);
3713
3714MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3715MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3716MODULE_LICENSE("GPL");
3717MODULE_ALIAS("pci:hns-nic");
3c7624d8 3718MODULE_VERSION(HNS3_MOD_VERSION);
Ubuntu Focal Linux kernel mirror