]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
projects / mirror_ubuntu-bionic-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: rename process_hw_error function
[mirror_ubuntu-bionic-kernel.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
ef57c40f
JS		 1// SPDX-License-Identifier: GPL-2.0+
2// Copyright (c) 2016-2017 Hisilicon Limited.
76ad4f0e
S		 3
4#include <linux/dma-mapping.h>
5#include <linux/etherdevice.h>
6#include <linux/interrupt.h>
7#include <linux/if_vlan.h>
8#include <linux/ip.h>
9#include <linux/ipv6.h>
10#include <linux/module.h>
11#include <linux/pci.h>
538d8ba0 12#include <linux/aer.h>
76ad4f0e
S		 13#include <linux/skbuff.h>
14#include <linux/sctp.h>
15#include <linux/vermagic.h>
16#include <net/gre.h>
30d240df 17#include <net/pkt_cls.h>
5337b725 18#include <net/tcp.h>
76ad4f0e
S		 19#include <net/vxlan.h>
20
21#include "hnae3.h"
22#include "hns3_enet.h"
23
e32a805a
FL		 24static void hns3_clear_all_ring(struct hnae3_handle *h);
25static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
2c794374 26static void hns3_remove_hw_addr(struct net_device *netdev);
e32a805a 27
1db9b1bf 28static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 29const char hns3_driver_version[] = VERMAGIC_STRING;
30static const char hns3_driver_string[] =
31 "Hisilicon Ethernet Network Driver for Hip08 Family";
32static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33static struct hnae3_client client;
34
35/* hns3_pci_tbl - PCI Device ID Table
36 *
37 * Last entry must be all 0s
38 *
39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40 * Class, Class Mask, private data (not used) }
41 */
42static const struct pci_device_id hns3_pci_tbl[] = {
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
a9c89a3f 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
f4e4e86c
JS		 56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
57 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
76ad4f0e
S		 58 /* required last entry */
59 {0, }
60};
61MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
62
5a8b1a40 63static irqreturn_t hns3_irq_handle(int irq, void *vector)
76ad4f0e 64{
5a8b1a40 65 struct hns3_enet_tqp_vector *tqp_vector = vector;
76ad4f0e
S		 66
67 napi_schedule(&tqp_vector->napi);
68
69 return IRQ_HANDLED;
70}
71
15040788
PL		 72/* This callback function is used to set affinity changes to the irq affinity
73 * masks when the irq_set_affinity_notifier function is used.
74 */
75static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
76 const cpumask_t *mask)
77{
78 struct hns3_enet_tqp_vector *tqp_vectors =
79 container_of(notify, struct hns3_enet_tqp_vector,
80 affinity_notify);
81
82 tqp_vectors->affinity_mask = *mask;
83}
84
85static void hns3_nic_irq_affinity_release(struct kref *ref)
86{
87}
88
76ad4f0e
S		 89static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
90{
91 struct hns3_enet_tqp_vector *tqp_vectors;
92 unsigned int i;
93
94 for (i = 0; i < priv->vector_num; i++) {
95 tqp_vectors = &priv->tqp_vector[i];
96
97 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
98 continue;
99
15040788
PL		 100 /* clear the affinity notifier and affinity mask */
101 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
102 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
103
76ad4f0e
S		 104 /* release the irq resource */
105 free_irq(tqp_vectors->vector_irq, tqp_vectors);
106 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
107 }
108}
109
110static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
111{
112 struct hns3_enet_tqp_vector *tqp_vectors;
113 int txrx_int_idx = 0;
114 int rx_int_idx = 0;
115 int tx_int_idx = 0;
116 unsigned int i;
117 int ret;
118
119 for (i = 0; i < priv->vector_num; i++) {
120 tqp_vectors = &priv->tqp_vector[i];
121
122 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
123 continue;
124
125 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
126 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
127 "%s-%s-%d", priv->netdev->name, "TxRx",
128 txrx_int_idx++);
129 txrx_int_idx++;
130 } else if (tqp_vectors->rx_group.ring) {
131 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
132 "%s-%s-%d", priv->netdev->name, "Rx",
133 rx_int_idx++);
134 } else if (tqp_vectors->tx_group.ring) {
135 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
136 "%s-%s-%d", priv->netdev->name, "Tx",
137 tx_int_idx++);
138 } else {
139 /* Skip this unused q_vector */
140 continue;
141 }
142
143 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
144
145 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
146 tqp_vectors->name,
147 tqp_vectors);
148 if (ret) {
149 netdev_err(priv->netdev, "request irq(%d) fail\n",
150 tqp_vectors->vector_irq);
151 return ret;
152 }
153
15040788
PL		 154 tqp_vectors->affinity_notify.notify =
155 hns3_nic_irq_affinity_notify;
156 tqp_vectors->affinity_notify.release =
157 hns3_nic_irq_affinity_release;
158 irq_set_affinity_notifier(tqp_vectors->vector_irq,
159 &tqp_vectors->affinity_notify);
160 irq_set_affinity_hint(tqp_vectors->vector_irq,
161 &tqp_vectors->affinity_mask);
162
76ad4f0e
S		 163 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
164 }
165
166 return 0;
167}
168
169static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
170 u32 mask_en)
171{
172 writel(mask_en, tqp_vector->mask_addr);
173}
174
175static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
176{
177 napi_enable(&tqp_vector->napi);
178
179 /* enable vector */
180 hns3_mask_vector_irq(tqp_vector, 1);
181}
182
183static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
184{
185 /* disable vector */
186 hns3_mask_vector_irq(tqp_vector, 0);
187
188 disable_irq(tqp_vector->vector_irq);
189 napi_disable(&tqp_vector->napi);
190}
191
5acd0356
FL		 192void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
193 u32 rl_value)
76ad4f0e 194{
5acd0356
FL		 195 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
196
76ad4f0e
S		 197 /* this defines the configuration for RL (Interrupt Rate Limiter).
198 * Rl defines rate of interrupts i.e. number of interrupts-per-second
199 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
200 */
5acd0356 201
d420d2de
YL		 202 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
203 !tqp_vector->rx_group.coal.gl_adapt_enable)
5acd0356
FL		 204 /* According to the hardware, the range of rl_reg is
205 * 0-59 and the unit is 4.
206 */
207 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
208
209 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
210}
211
212void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
213 u32 gl_value)
214{
215 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
216
217 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
218}
219
220void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
221 u32 gl_value)
222{
223 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
224
225 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 226}
227
2b27decc
FL		 228static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
229 struct hns3_nic_priv *priv)
76ad4f0e
S		 230{
231 /* initialize the configuration for interrupt coalescing.
232 * 1. GL (Interrupt Gap Limiter)
233 * 2. RL (Interrupt Rate Limiter)
234 */
235
2b27decc 236 /* Default: enable interrupt coalescing self-adaptive and GL */
d420d2de
YL		 237 tqp_vector->tx_group.coal.gl_adapt_enable = 1;
238 tqp_vector->rx_group.coal.gl_adapt_enable = 1;
2b27decc 239
d420d2de
YL		 240 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
241 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
2b27decc 242
3f97bd23 243 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
d420d2de
YL		 244 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
245 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 246}
247
6cbd6d33
YL		 248static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
249 struct hns3_nic_priv *priv)
250{
251 struct hnae3_handle *h = priv->ae_handle;
252
253 hns3_set_vector_coalesce_tx_gl(tqp_vector,
d420d2de 254 tqp_vector->tx_group.coal.int_gl);
6cbd6d33 255 hns3_set_vector_coalesce_rx_gl(tqp_vector,
d420d2de 256 tqp_vector->rx_group.coal.int_gl);
6cbd6d33
YL		 257 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
258}
259
9df8f79a
YL		 260static int hns3_nic_set_real_num_queue(struct net_device *netdev)
261{
9780cb97 262 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 263 struct hnae3_knic_private_info *kinfo = &h->kinfo;
264 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
7e114e7d
YL		 265 int i, ret;
266
267 if (kinfo->num_tc <= 1) {
268 netdev_reset_tc(netdev);
269 } else {
270 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
271 if (ret) {
272 netdev_err(netdev,
273 "netdev_set_num_tc fail, ret=%d!\n", ret);
274 return ret;
275 }
276
277 for (i = 0; i < HNAE3_MAX_TC; i++) {
278 if (!kinfo->tc_info[i].enable)
279 continue;
280
281 netdev_set_tc_queue(netdev,
282 kinfo->tc_info[i].tc,
283 kinfo->tc_info[i].tqp_count,
284 kinfo->tc_info[i].tqp_offset);
285 }
286 }
9df8f79a
YL		 287
288 ret = netif_set_real_num_tx_queues(netdev, queue_size);
289 if (ret) {
290 netdev_err(netdev,
291 "netif_set_real_num_tx_queues fail, ret=%d!\n",
292 ret);
293 return ret;
294 }
295
296 ret = netif_set_real_num_rx_queues(netdev, queue_size);
297 if (ret) {
298 netdev_err(netdev,
299 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
300 return ret;
301 }
302
303 return 0;
304}
305
2d7187ce
PL		 306static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
307{
08ca3d58 308 u16 alloc_tqps, max_rss_size, rss_size;
2d7187ce 309
08ca3d58
HT		 310 h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
311 rss_size = alloc_tqps / h->kinfo.num_tc;
2d7187ce 312
08ca3d58 313 return min_t(u16, rss_size, max_rss_size);
2d7187ce
PL		 314}
315
2b5251b7
HT		 316static void hns3_tqp_enable(struct hnae3_queue *tqp)
317{
318 u32 rcb_reg;
319
320 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
321 rcb_reg |= BIT(HNS3_RING_EN_B);
322 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
323}
324
325static void hns3_tqp_disable(struct hnae3_queue *tqp)
326{
327 u32 rcb_reg;
328
329 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
330 rcb_reg &= ~BIT(HNS3_RING_EN_B);
331 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
332}
333
76ad4f0e
S		 334static int hns3_nic_net_up(struct net_device *netdev)
335{
336 struct hns3_nic_priv *priv = netdev_priv(netdev);
337 struct hnae3_handle *h = priv->ae_handle;
338 int i, j;
339 int ret;
340
e32a805a
FL		 341 ret = hns3_nic_reset_all_ring(h);
342 if (ret)
343 return ret;
344
76ad4f0e
S		 345 /* get irq resource for all vectors */
346 ret = hns3_nic_init_irq(priv);
347 if (ret) {
348 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
349 return ret;
350 }
351
352 /* enable the vectors */
353 for (i = 0; i < priv->vector_num; i++)
354 hns3_vector_enable(&priv->tqp_vector[i]);
355
2b5251b7
HT		 356 /* enable rcb */
357 for (j = 0; j < h->kinfo.num_tqps; j++)
358 hns3_tqp_enable(h->kinfo.tqp[j]);
359
76ad4f0e
S		 360 /* start the ae_dev */
361 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
362 if (ret)
363 goto out_start_err;
364
a3083abb
JS		 365 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
366
76ad4f0e
S		 367 return 0;
368
369out_start_err:
2b5251b7
HT		 370 while (j--)
371 hns3_tqp_disable(h->kinfo.tqp[j]);
372
76ad4f0e
S		 373 for (j = i - 1; j >= 0; j--)
374 hns3_vector_disable(&priv->tqp_vector[j]);
375
376 hns3_nic_uninit_irq(priv);
377
378 return ret;
379}
380
381static int hns3_nic_net_open(struct net_device *netdev)
382{
7e114e7d
YL		 383 struct hnae3_handle *h = hns3_get_handle(netdev);
384 struct hnae3_knic_private_info *kinfo;
385 int i, ret;
76ad4f0e 386
3a11f446
HT		 387 if (hns3_nic_resetting(netdev))
388 return -EBUSY;
389
76ad4f0e
S		 390 netif_carrier_off(netdev);
391
9df8f79a
YL		 392 ret = hns3_nic_set_real_num_queue(netdev);
393 if (ret)
76ad4f0e 394 return ret;
76ad4f0e
S		 395
396 ret = hns3_nic_net_up(netdev);
397 if (ret) {
398 netdev_err(netdev,
399 "hns net up fail, ret=%d!\n", ret);
400 return ret;
401 }
402
7e114e7d
YL		 403 kinfo = &h->kinfo;
404 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
405 netdev_set_prio_tc_map(netdev, i,
406 kinfo->prio_tc[i]);
407 }
408
76ad4f0e
S		 409 return 0;
410}
411
412static void hns3_nic_net_down(struct net_device *netdev)
413{
414 struct hns3_nic_priv *priv = netdev_priv(netdev);
2b5251b7 415 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 416 const struct hnae3_ae_ops *ops;
417 int i;
418
e32a805a
FL		 419 /* disable vectors */
420 for (i = 0; i < priv->vector_num; i++)
421 hns3_vector_disable(&priv->tqp_vector[i]);
422
2b5251b7
HT		 423 /* disable rcb */
424 for (i = 0; i < h->kinfo.num_tqps; i++)
425 hns3_tqp_disable(h->kinfo.tqp[i]);
426
76ad4f0e
S		 427 /* stop ae_dev */
428 ops = priv->ae_handle->ae_algo->ops;
429 if (ops->stop)
430 ops->stop(priv->ae_handle);
431
76ad4f0e
S		 432 /* free irq resources */
433 hns3_nic_uninit_irq(priv);
e32a805a
FL		 434
435 hns3_clear_all_ring(priv->ae_handle);
76ad4f0e
S		 436}
437
438static int hns3_nic_net_stop(struct net_device *netdev)
439{
f22463fd
HT		 440 struct hns3_nic_priv *priv = netdev_priv(netdev);
441
442 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
443 return 0;
444
76ad4f0e
S		 445 netif_tx_stop_all_queues(netdev);
446 netif_carrier_off(netdev);
447
448 hns3_nic_net_down(netdev);
449
450 return 0;
451}
452
76ad4f0e
S		 453static int hns3_nic_uc_sync(struct net_device *netdev,
454 const unsigned char *addr)
455{
9780cb97 456 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 457
458 if (h->ae_algo->ops->add_uc_addr)
459 return h->ae_algo->ops->add_uc_addr(h, addr);
460
461 return 0;
462}
463
464static int hns3_nic_uc_unsync(struct net_device *netdev,
465 const unsigned char *addr)
466{
9780cb97 467 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 468
469 if (h->ae_algo->ops->rm_uc_addr)
470 return h->ae_algo->ops->rm_uc_addr(h, addr);
471
472 return 0;
473}
474
475static int hns3_nic_mc_sync(struct net_device *netdev,
476 const unsigned char *addr)
477{
9780cb97 478 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 479
720a8478 480 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 481 return h->ae_algo->ops->add_mc_addr(h, addr);
482
483 return 0;
484}
485
486static int hns3_nic_mc_unsync(struct net_device *netdev,
487 const unsigned char *addr)
488{
9780cb97 489 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 490
720a8478 491 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 492 return h->ae_algo->ops->rm_mc_addr(h, addr);
493
494 return 0;
495}
496
1e3653db
JS		 497static u8 hns3_get_netdev_flags(struct net_device *netdev)
498{
499 u8 flags = 0;
500
501 if (netdev->flags & IFF_PROMISC) {
502 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
503 } else {
504 flags |= HNAE3_VLAN_FLTR;
505 if (netdev->flags & IFF_ALLMULTI)
506 flags |= HNAE3_USER_MPE;
507 }
508
509 return flags;
510}
511
1db9b1bf 512static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 513{
9780cb97 514 struct hnae3_handle *h = hns3_get_handle(netdev);
1e3653db
JS		 515 u8 new_flags;
516 int ret;
76ad4f0e 517
1e3653db
JS		 518 new_flags = hns3_get_netdev_flags(netdev);
519
520 ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
521 if (ret) {
76ad4f0e 522 netdev_err(netdev, "sync uc address fail\n");
1e3653db
JS		 523 if (ret == -ENOSPC)
524 new_flags |= HNAE3_OVERFLOW_UPE;
525 }
526
a832d8b5 527 if (netdev->flags & IFF_MULTICAST) {
1e3653db
JS		 528 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
529 hns3_nic_mc_unsync);
530 if (ret) {
76ad4f0e 531 netdev_err(netdev, "sync mc address fail\n");
1e3653db
JS		 532 if (ret == -ENOSPC)
533 new_flags |= HNAE3_OVERFLOW_MPE;
534 }
535 }
536
537 hns3_update_promisc_mode(netdev, new_flags);
538 /* User mode Promisc mode enable and vlan filtering is disabled to
539 * let all packets in. MAC-VLAN Table overflow Promisc enabled and
540 * vlan fitering is enabled
541 */
542 hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
543 h->netdev_flags = new_flags;
544}
545
abe62a63 546int hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
1e3653db
JS		 547{
548 struct hns3_nic_priv *priv = netdev_priv(netdev);
549 struct hnae3_handle *h = priv->ae_handle;
550
551 if (h->ae_algo->ops->set_promisc_mode) {
abe62a63
HT		 552 return h->ae_algo->ops->set_promisc_mode(h,
553 promisc_flags & HNAE3_UPE,
554 promisc_flags & HNAE3_MPE);
1e3653db 555 }
abe62a63
HT		 556
557 return 0;
1e3653db
JS		 558}
559
560void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
561{
562 struct hns3_nic_priv *priv = netdev_priv(netdev);
563 struct hnae3_handle *h = priv->ae_handle;
564 bool last_state;
565
566 if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
567 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
568 if (enable != last_state) {
569 netdev_info(netdev,
570 "%s vlan filter\n",
571 enable ? "enable" : "disable");
572 h->ae_algo->ops->enable_vlan_filter(h, enable);
573 }
a832d8b5 574 }
76ad4f0e
S		 575}
576
577static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
578 u16 *mss, u32 *type_cs_vlan_tso)
579{
580 u32 l4_offset, hdr_len;
581 union l3_hdr_info l3;
582 union l4_hdr_info l4;
583 u32 l4_paylen;
584 int ret;
585
586 if (!skb_is_gso(skb))
587 return 0;
588
589 ret = skb_cow_head(skb, 0);
590 if (ret)
591 return ret;
592
593 l3.hdr = skb_network_header(skb);
594 l4.hdr = skb_transport_header(skb);
595
596 /* Software should clear the IPv4's checksum field when tso is
597 * needed.
598 */
599 if (l3.v4->version == 4)
600 l3.v4->check = 0;
601
602 /* tunnel packet.*/
603 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
604 SKB_GSO_GRE_CSUM |
605 SKB_GSO_UDP_TUNNEL |
606 SKB_GSO_UDP_TUNNEL_CSUM)) {
607 if ((!(skb_shinfo(skb)->gso_type &
608 SKB_GSO_PARTIAL)) &&
609 (skb_shinfo(skb)->gso_type &
610 SKB_GSO_UDP_TUNNEL_CSUM)) {
611 /* Software should clear the udp's checksum
612 * field when tso is needed.
613 */
614 l4.udp->check = 0;
615 }
616 /* reset l3&l4 pointers from outer to inner headers */
617 l3.hdr = skb_inner_network_header(skb);
618 l4.hdr = skb_inner_transport_header(skb);
619
620 /* Software should clear the IPv4's checksum field when
621 * tso is needed.
622 */
623 if (l3.v4->version == 4)
624 l3.v4->check = 0;
625 }
626
627 /* normal or tunnel packet*/
628 l4_offset = l4.hdr - skb->data;
629 hdr_len = (l4.tcp->doff * 4) + l4_offset;
630
631 /* remove payload length from inner pseudo checksum when tso*/
632 l4_paylen = skb->len - l4_offset;
633 csum_replace_by_diff(&l4.tcp->check,
634 (__force __wsum)htonl(l4_paylen));
635
636 /* find the txbd field values */
637 *paylen = skb->len - hdr_len;
ccc23ef3
PL		 638 hnae3_set_bit(*type_cs_vlan_tso,
639 HNS3_TXD_TSO_B, 1);
76ad4f0e
S		 640
641 /* get MSS for TSO */
642 *mss = skb_shinfo(skb)->gso_size;
643
644 return 0;
645}
646
1898d4e4
S		 647static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
648 u8 *il4_proto)
76ad4f0e
S		 649{
650 union {
651 struct iphdr *v4;
652 struct ipv6hdr *v6;
653 unsigned char *hdr;
654 } l3;
655 unsigned char *l4_hdr;
656 unsigned char *exthdr;
657 u8 l4_proto_tmp;
658 __be16 frag_off;
659
660 /* find outer header point */
661 l3.hdr = skb_network_header(skb);
c82a5497 662 l4_hdr = skb_transport_header(skb);
76ad4f0e
S		 663
664 if (skb->protocol == htons(ETH_P_IPV6)) {
665 exthdr = l3.hdr + sizeof(*l3.v6);
666 l4_proto_tmp = l3.v6->nexthdr;
667 if (l4_hdr != exthdr)
668 ipv6_skip_exthdr(skb, exthdr - skb->data,
669 &l4_proto_tmp, &frag_off);
670 } else if (skb->protocol == htons(ETH_P_IP)) {
671 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 672 } else {
673 return -EINVAL;
76ad4f0e
S		 674 }
675
676 *ol4_proto = l4_proto_tmp;
677
678 /* tunnel packet */
679 if (!skb->encapsulation) {
680 *il4_proto = 0;
1898d4e4 681 return 0;
76ad4f0e
S		 682 }
683
684 /* find inner header point */
685 l3.hdr = skb_inner_network_header(skb);
686 l4_hdr = skb_inner_transport_header(skb);
687
688 if (l3.v6->version == 6) {
689 exthdr = l3.hdr + sizeof(*l3.v6);
690 l4_proto_tmp = l3.v6->nexthdr;
691 if (l4_hdr != exthdr)
692 ipv6_skip_exthdr(skb, exthdr - skb->data,
693 &l4_proto_tmp, &frag_off);
694 } else if (l3.v4->version == 4) {
695 l4_proto_tmp = l3.v4->protocol;
696 }
697
698 *il4_proto = l4_proto_tmp;
1898d4e4
S		 699
700 return 0;
76ad4f0e
S		 701}
702
703static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
704 u8 il4_proto, u32 *type_cs_vlan_tso,
705 u32 *ol_type_vlan_len_msec)
706{
707 union {
708 struct iphdr *v4;
709 struct ipv6hdr *v6;
710 unsigned char *hdr;
711 } l3;
712 union {
713 struct tcphdr *tcp;
714 struct udphdr *udp;
715 struct gre_base_hdr *gre;
716 unsigned char *hdr;
717 } l4;
718 unsigned char *l2_hdr;
719 u8 l4_proto = ol4_proto;
720 u32 ol2_len;
721 u32 ol3_len;
722 u32 ol4_len;
723 u32 l2_len;
724 u32 l3_len;
725
726 l3.hdr = skb_network_header(skb);
727 l4.hdr = skb_transport_header(skb);
728
729 /* compute L2 header size for normal packet, defined in 2 Bytes */
730 l2_len = l3.hdr - skb->data;
ccc23ef3
PL		 731 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
732 HNS3_TXD_L2LEN_S, l2_len >> 1);
76ad4f0e
S		 733
734 /* tunnel packet*/
735 if (skb->encapsulation) {
736 /* compute OL2 header size, defined in 2 Bytes */
737 ol2_len = l2_len;
ccc23ef3
PL		 738 hnae3_set_field(*ol_type_vlan_len_msec,
739 HNS3_TXD_L2LEN_M,
740 HNS3_TXD_L2LEN_S, ol2_len >> 1);
76ad4f0e
S		 741
742 /* compute OL3 header size, defined in 4 Bytes */
743 ol3_len = l4.hdr - l3.hdr;
ccc23ef3
PL		 744 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
745 HNS3_TXD_L3LEN_S, ol3_len >> 2);
76ad4f0e
S		 746
747 /* MAC in UDP, MAC in GRE (0x6558)*/
748 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
749 /* switch MAC header ptr from outer to inner header.*/
750 l2_hdr = skb_inner_mac_header(skb);
751
752 /* compute OL4 header size, defined in 4 Bytes. */
753 ol4_len = l2_hdr - l4.hdr;
ccc23ef3
PL		 754 hnae3_set_field(*ol_type_vlan_len_msec,
755 HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
756 ol4_len >> 2);
76ad4f0e
S		 757
758 /* switch IP header ptr from outer to inner header */
759 l3.hdr = skb_inner_network_header(skb);
760
761 /* compute inner l2 header size, defined in 2 Bytes. */
762 l2_len = l3.hdr - l2_hdr;
ccc23ef3
PL		 763 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
764 HNS3_TXD_L2LEN_S, l2_len >> 1);
76ad4f0e
S		 765 } else {
766 /* skb packet types not supported by hardware,
767 * txbd len fild doesn't be filled.
768 */
769 return;
770 }
771
772 /* switch L4 header pointer from outer to inner */
773 l4.hdr = skb_inner_transport_header(skb);
774
775 l4_proto = il4_proto;
776 }
777
778 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
779 l3_len = l4.hdr - l3.hdr;
ccc23ef3
PL		 780 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
781 HNS3_TXD_L3LEN_S, l3_len >> 2);
76ad4f0e
S		 782
783 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
784 switch (l4_proto) {
785 case IPPROTO_TCP:
ccc23ef3
PL		 786 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
787 HNS3_TXD_L4LEN_S, l4.tcp->doff);
76ad4f0e
S		 788 break;
789 case IPPROTO_SCTP:
ccc23ef3
PL		 790 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
791 HNS3_TXD_L4LEN_S,
792 (sizeof(struct sctphdr) >> 2));
76ad4f0e
S		 793 break;
794 case IPPROTO_UDP:
ccc23ef3
PL		 795 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
796 HNS3_TXD_L4LEN_S,
797 (sizeof(struct udphdr) >> 2));
76ad4f0e
S		 798 break;
799 default:
800 /* skb packet types not supported by hardware,
801 * txbd len fild doesn't be filled.
802 */
803 return;
804 }
805}
806
cfdd6e86
YL		 807/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
808 * and it is udp packet, which has a dest port as the IANA assigned.
809 * the hardware is expected to do the checksum offload, but the
810 * hardware will not do the checksum offload when udp dest port is
811 * 4789.
812 */
813static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
814{
815#define IANA_VXLAN_PORT 4789
816 union {
817 struct tcphdr *tcp;
818 struct udphdr *udp;
819 struct gre_base_hdr *gre;
820 unsigned char *hdr;
821 } l4;
822
823 l4.hdr = skb_transport_header(skb);
824
825 if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
826 return false;
827
828 skb_checksum_help(skb);
829
830 return true;
831}
832
76ad4f0e
S		 833static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
834 u8 il4_proto, u32 *type_cs_vlan_tso,
835 u32 *ol_type_vlan_len_msec)
836{
837 union {
838 struct iphdr *v4;
839 struct ipv6hdr *v6;
840 unsigned char *hdr;
841 } l3;
842 u32 l4_proto = ol4_proto;
843
844 l3.hdr = skb_network_header(skb);
845
846 /* define OL3 type and tunnel type(OL4).*/
847 if (skb->encapsulation) {
848 /* define outer network header type.*/
849 if (skb->protocol == htons(ETH_P_IP)) {
850 if (skb_is_gso(skb))
ccc23ef3
PL		 851 hnae3_set_field(*ol_type_vlan_len_msec,
852 HNS3_TXD_OL3T_M,
853 HNS3_TXD_OL3T_S,
854 HNS3_OL3T_IPV4_CSUM);
76ad4f0e 855 else
ccc23ef3
PL		 856 hnae3_set_field(*ol_type_vlan_len_msec,
857 HNS3_TXD_OL3T_M,
858 HNS3_TXD_OL3T_S,
859 HNS3_OL3T_IPV4_NO_CSUM);
76ad4f0e
S		 860
861 } else if (skb->protocol == htons(ETH_P_IPV6)) {
ccc23ef3
PL		 862 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
863 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
76ad4f0e
S		 864 }
865
866 /* define tunnel type(OL4).*/
867 switch (l4_proto) {
868 case IPPROTO_UDP:
ccc23ef3
PL		 869 hnae3_set_field(*ol_type_vlan_len_msec,
870 HNS3_TXD_TUNTYPE_M,
871 HNS3_TXD_TUNTYPE_S,
872 HNS3_TUN_MAC_IN_UDP);
76ad4f0e
S		 873 break;
874 case IPPROTO_GRE:
ccc23ef3
PL		 875 hnae3_set_field(*ol_type_vlan_len_msec,
876 HNS3_TXD_TUNTYPE_M,
877 HNS3_TXD_TUNTYPE_S,
878 HNS3_TUN_NVGRE);
76ad4f0e
S		 879 break;
880 default:
881 /* drop the skb tunnel packet if hardware don't support,
882 * because hardware can't calculate csum when TSO.
883 */
884 if (skb_is_gso(skb))
885 return -EDOM;
886
887 /* the stack computes the IP header already,
888 * driver calculate l4 checksum when not TSO.
889 */
890 skb_checksum_help(skb);
891 return 0;
892 }
893
894 l3.hdr = skb_inner_network_header(skb);
895 l4_proto = il4_proto;
896 }
897
898 if (l3.v4->version == 4) {
ccc23ef3
PL		 899 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
900 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
76ad4f0e
S		 901
902 /* the stack computes the IP header already, the only time we
903 * need the hardware to recompute it is in the case of TSO.
904 */
905 if (skb_is_gso(skb))
ccc23ef3 906 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
76ad4f0e 907 } else if (l3.v6->version == 6) {
ccc23ef3
PL		 908 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
909 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
76ad4f0e
S		 910 }
911
912 switch (l4_proto) {
913 case IPPROTO_TCP:
79fa1b6a 914 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
ccc23ef3
PL		 915 hnae3_set_field(*type_cs_vlan_tso,
916 HNS3_TXD_L4T_M,
917 HNS3_TXD_L4T_S,
918 HNS3_L4T_TCP);
76ad4f0e
S		 919 break;
920 case IPPROTO_UDP:
cfdd6e86
YL		 921 if (hns3_tunnel_csum_bug(skb))
922 break;
923
79fa1b6a 924 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
ccc23ef3
PL		 925 hnae3_set_field(*type_cs_vlan_tso,
926 HNS3_TXD_L4T_M,
927 HNS3_TXD_L4T_S,
928 HNS3_L4T_UDP);
76ad4f0e
S		 929 break;
930 case IPPROTO_SCTP:
79fa1b6a 931 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
ccc23ef3
PL		 932 hnae3_set_field(*type_cs_vlan_tso,
933 HNS3_TXD_L4T_M,
934 HNS3_TXD_L4T_S,
935 HNS3_L4T_SCTP);
76ad4f0e
S		 936 break;
937 default:
938 /* drop the skb tunnel packet if hardware don't support,
939 * because hardware can't calculate csum when TSO.
940 */
941 if (skb_is_gso(skb))
942 return -EDOM;
943
944 /* the stack computes the IP header already,
945 * driver calculate l4 checksum when not TSO.
946 */
947 skb_checksum_help(skb);
948 return 0;
949 }
950
951 return 0;
952}
953
954static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
955{
956 /* Config bd buffer end */
ccc23ef3
PL		 957 hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
958 HNS3_TXD_BDTYPE_S, 0);
959 hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
960 hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
961 hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 962}
963
1fdd8dc5
PL		 964static int hns3_fill_desc_vtags(struct sk_buff *skb,
965 struct hns3_enet_ring *tx_ring,
966 u32 *inner_vlan_flag,
967 u32 *out_vlan_flag,
968 u16 *inner_vtag,
969 u16 *out_vtag)
970{
971#define HNS3_TX_VLAN_PRIO_SHIFT 13
972
973 if (skb->protocol == htons(ETH_P_8021Q) &&
974 !(tx_ring->tqp->handle->kinfo.netdev->features &
975 NETIF_F_HW_VLAN_CTAG_TX)) {
976 /* When HW VLAN acceleration is turned off, and the stack
977 * sets the protocol to 802.1q, the driver just need to
978 * set the protocol to the encapsulated ethertype.
979 */
980 skb->protocol = vlan_get_protocol(skb);
981 return 0;
982 }
983
984 if (skb_vlan_tag_present(skb)) {
985 u16 vlan_tag;
986
987 vlan_tag = skb_vlan_tag_get(skb);
988 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
989
990 /* Based on hw strategy, use out_vtag in two layer tag case,
991 * and use inner_vtag in one tag case.
992 */
993 if (skb->protocol == htons(ETH_P_8021Q)) {
ccc23ef3 994 hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
1fdd8dc5
PL		 995 *out_vtag = vlan_tag;
996 } else {
ccc23ef3 997 hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
1fdd8dc5
PL		 998 *inner_vtag = vlan_tag;
999 }
1000 } else if (skb->protocol == htons(ETH_P_8021Q)) {
1001 struct vlan_ethhdr *vhdr;
1002 int rc;
1003
1004 rc = skb_cow_head(skb, 0);
1005 if (rc < 0)
1006 return rc;
1007 vhdr = (struct vlan_ethhdr *)skb->data;
1008 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
1009 << HNS3_TX_VLAN_PRIO_SHIFT);
1010 }
1011
1012 skb->protocol = vlan_get_protocol(skb);
1013 return 0;
1014}
1015
76ad4f0e 1016static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
91040825 1017 int size, int frag_end, enum hns_desc_type type)
76ad4f0e
S		 1018{
1019 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1020 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
91040825 1021 struct device *dev = ring_to_dev(ring);
76ad4f0e
S		 1022 u32 ol_type_vlan_len_msec = 0;
1023 u16 bdtp_fe_sc_vld_ra_ri = 0;
91040825 1024 struct skb_frag_struct *frag;
13491411 1025 unsigned int frag_buf_num;
76ad4f0e
S		 1026 u32 type_cs_vlan_tso = 0;
1027 struct sk_buff *skb;
1fdd8dc5
PL		 1028 u16 inner_vtag = 0;
1029 u16 out_vtag = 0;
13491411
FL		 1030 unsigned int k;
1031 int sizeoflast;
76ad4f0e 1032 u32 paylen = 0;
91040825 1033 dma_addr_t dma;
76ad4f0e 1034 u16 mss = 0;
76ad4f0e
S		 1035 u8 ol4_proto;
1036 u8 il4_proto;
1037 int ret;
1038
76ad4f0e
S		 1039 if (type == DESC_TYPE_SKB) {
1040 skb = (struct sk_buff *)priv;
a90bb9a5 1041 paylen = skb->len;
76ad4f0e 1042
1fdd8dc5
PL		 1043 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1044 &ol_type_vlan_len_msec,
1045 &inner_vtag, &out_vtag);
1046 if (unlikely(ret))
1047 return ret;
1048
76ad4f0e
S		 1049 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1050 skb_reset_mac_len(skb);
76ad4f0e 1051
1898d4e4
S		 1052 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1053 if (ret)
1054 return ret;
76ad4f0e
S		 1055 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1056 &type_cs_vlan_tso,
1057 &ol_type_vlan_len_msec);
1058 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1059 &type_cs_vlan_tso,
1060 &ol_type_vlan_len_msec);
1061 if (ret)
1062 return ret;
1063
1064 ret = hns3_set_tso(skb, &paylen, &mss,
1065 &type_cs_vlan_tso);
1066 if (ret)
1067 return ret;
1068 }
1069
1070 /* Set txbd */
1071 desc->tx.ol_type_vlan_len_msec =
1072 cpu_to_le32(ol_type_vlan_len_msec);
1073 desc->tx.type_cs_vlan_tso_len =
1074 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 1075 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 1076 desc->tx.mss = cpu_to_le16(mss);
1fdd8dc5
PL		 1077 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1078 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
91040825
PL		 1079
1080 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1081 } else {
1082 frag = (struct skb_frag_struct *)priv;
1083 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1084 }
1085
1086 if (dma_mapping_error(ring->dev, dma)) {
1087 ring->stats.sw_err_cnt++;
1088 return -ENOMEM;
76ad4f0e
S		 1089 }
1090
6a3bff0d
FL		 1091 desc_cb->length = size;
1092
13491411
FL		 1093 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1094 sizeoflast = size % HNS3_MAX_BD_SIZE;
1095 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1096
1097 /* When frag size is bigger than hardware limit, split this frag */
1098 for (k = 0; k < frag_buf_num; k++) {
1099 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1100 desc_cb->priv = priv;
13491411
FL		 1101 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1102 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1103 DESC_TYPE_SKB : DESC_TYPE_PAGE;
1104
1105 /* now, fill the descriptor */
1106 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
6a3bff0d
FL		 1107 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1108 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
13491411
FL		 1109 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1110 frag_end && (k == frag_buf_num - 1) ?
1111 1 : 0);
1112 desc->tx.bdtp_fe_sc_vld_ra_ri =
1113 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1114
1115 /* move ring pointer to next.*/
1116 ring_ptr_move_fw(ring, next_to_use);
1117
1118 desc_cb = &ring->desc_cb[ring->next_to_use];
1119 desc = &ring->desc[ring->next_to_use];
1120 }
76ad4f0e
S		 1121
1122 return 0;
1123}
1124
76ad4f0e
S		 1125static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1126 struct hns3_enet_ring *ring)
1127{
1128 struct sk_buff *skb = *out_skb;
1129 struct skb_frag_struct *frag;
1130 int bdnum_for_frag;
1131 int frag_num;
1132 int buf_num;
1133 int size;
1134 int i;
1135
1136 size = skb_headlen(skb);
1137 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1138
1139 frag_num = skb_shinfo(skb)->nr_frags;
1140 for (i = 0; i < frag_num; i++) {
1141 frag = &skb_shinfo(skb)->frags[i];
1142 size = skb_frag_size(frag);
1143 bdnum_for_frag =
1144 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1145 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1146 return -ENOMEM;
1147
1148 buf_num += bdnum_for_frag;
1149 }
1150
1151 if (buf_num > ring_space(ring))
1152 return -EBUSY;
1153
1154 *bnum = buf_num;
1155 return 0;
1156}
1157
1158static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1159 struct hns3_enet_ring *ring)
1160{
1161 struct sk_buff *skb = *out_skb;
1162 int buf_num;
1163
1164 /* No. of segments (plus a header) */
1165 buf_num = skb_shinfo(skb)->nr_frags + 1;
1166
91a2f02f 1167 if (unlikely(ring_space(ring) < buf_num))
76ad4f0e
S		 1168 return -EBUSY;
1169
1170 *bnum = buf_num;
1171
1172 return 0;
1173}
1174
fc09d03d 1175static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
76ad4f0e
S		 1176{
1177 struct device *dev = ring_to_dev(ring);
1178 unsigned int i;
1179
1180 for (i = 0; i < ring->desc_num; i++) {
1181 /* check if this is where we started */
1182 if (ring->next_to_use == next_to_use_orig)
1183 break;
1184
1185 /* unmap the descriptor dma address */
1186 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1187 dma_unmap_single(dev,
1188 ring->desc_cb[ring->next_to_use].dma,
1189 ring->desc_cb[ring->next_to_use].length,
1190 DMA_TO_DEVICE);
6a3bff0d 1191 else if (ring->desc_cb[ring->next_to_use].length)
76ad4f0e
S		 1192 dma_unmap_page(dev,
1193 ring->desc_cb[ring->next_to_use].dma,
1194 ring->desc_cb[ring->next_to_use].length,
1195 DMA_TO_DEVICE);
1196
6a3bff0d
FL		 1197 ring->desc_cb[ring->next_to_use].length = 0;
1198
76ad4f0e
S		 1199 /* rollback one */
1200 ring_ptr_move_bw(ring, next_to_use);
1201 }
1202}
1203
d43e5aca 1204netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 1205{
1206 struct hns3_nic_priv *priv = netdev_priv(netdev);
1207 struct hns3_nic_ring_data *ring_data =
1208 &tx_ring_data(priv, skb->queue_mapping);
1209 struct hns3_enet_ring *ring = ring_data->ring;
76ad4f0e
S		 1210 struct netdev_queue *dev_queue;
1211 struct skb_frag_struct *frag;
1212 int next_to_use_head;
1213 int next_to_use_frag;
76ad4f0e
S		 1214 int buf_num;
1215 int seg_num;
1216 int size;
1217 int ret;
1218 int i;
1219
1220 /* Prefetch the data used later */
1221 prefetch(skb->data);
1222
1223 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1224 case -EBUSY:
1225 u64_stats_update_begin(&ring->syncp);
1226 ring->stats.tx_busy++;
1227 u64_stats_update_end(&ring->syncp);
1228
1229 goto out_net_tx_busy;
1230 case -ENOMEM:
1231 u64_stats_update_begin(&ring->syncp);
1232 ring->stats.sw_err_cnt++;
1233 u64_stats_update_end(&ring->syncp);
1234 netdev_err(netdev, "no memory to xmit!\n");
1235
1236 goto out_err_tx_ok;
1237 default:
1238 break;
1239 }
1240
1241 /* No. of segments (plus a header) */
1242 seg_num = skb_shinfo(skb)->nr_frags + 1;
1243 /* Fill the first part */
1244 size = skb_headlen(skb);
1245
1246 next_to_use_head = ring->next_to_use;
1247
91040825
PL		 1248 ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1249 DESC_TYPE_SKB);
76ad4f0e 1250 if (ret)
fc09d03d 1251 goto head_fill_err;
76ad4f0e
S		 1252
1253 next_to_use_frag = ring->next_to_use;
1254 /* Fill the fragments */
1255 for (i = 1; i < seg_num; i++) {
1256 frag = &skb_shinfo(skb)->frags[i - 1];
1257 size = skb_frag_size(frag);
91040825
PL		 1258
1259 ret = priv->ops.fill_desc(ring, frag, size,
1260 seg_num - 1 == i ? 1 : 0,
1261 DESC_TYPE_PAGE);
76ad4f0e
S		 1262
1263 if (ret)
fc09d03d 1264 goto frag_fill_err;
76ad4f0e
S		 1265 }
1266
1267 /* Complete translate all packets */
1268 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1269 netdev_tx_sent_queue(dev_queue, skb->len);
1270
1271 wmb(); /* Commit all data before submit */
1272
ccc23ef3 1273 hnae3_queue_xmit(ring->tqp, buf_num);
76ad4f0e
S		 1274
1275 return NETDEV_TX_OK;
1276
fc09d03d
FL		 1277frag_fill_err:
1278 hns3_clear_desc(ring, next_to_use_frag);
76ad4f0e 1279
fc09d03d
FL		 1280head_fill_err:
1281 hns3_clear_desc(ring, next_to_use_head);
76ad4f0e
S		 1282
1283out_err_tx_ok:
1284 dev_kfree_skb_any(skb);
1285 return NETDEV_TX_OK;
1286
1287out_net_tx_busy:
1288 netif_stop_subqueue(netdev, ring_data->queue_index);
1289 smp_mb(); /* Commit all data before submit */
1290
1291 return NETDEV_TX_BUSY;
1292}
1293
1294static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1295{
9780cb97 1296 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1297 struct sockaddr *mac_addr = p;
1298 int ret;
1299
1300 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1301 return -EADDRNOTAVAIL;
1302
f37fc980
JS		 1303 if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1304 netdev_info(netdev, "already using mac address %pM\n",
1305 mac_addr->sa_data);
1306 return 0;
1307 }
1308
3cbf5e2d 1309 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
76ad4f0e
S		 1310 if (ret) {
1311 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1312 return ret;
1313 }
1314
1315 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1316
1317 return 0;
1318}
1319
a185d723
XW		 1320static int hns3_nic_do_ioctl(struct net_device *netdev,
1321 struct ifreq *ifr, int cmd)
1322{
1323 struct hnae3_handle *h = hns3_get_handle(netdev);
1324
1325 if (!netif_running(netdev))
1326 return -EINVAL;
1327
1328 if (!h->ae_algo->ops->do_ioctl)
1329 return -EOPNOTSUPP;
1330
1331 return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1332}
1333
76ad4f0e
S		 1334static int hns3_nic_set_features(struct net_device *netdev,
1335 netdev_features_t features)
1336{
21b6fd34 1337 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 1338 struct hns3_nic_priv *priv = netdev_priv(netdev);
5f9a7732 1339 struct hnae3_handle *h = priv->ae_handle;
5f9a7732 1340 int ret;
76ad4f0e 1341
21b6fd34 1342 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
e4c38d6e 1343 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
21b6fd34 1344 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
e4c38d6e 1345 else
21b6fd34 1346 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
76ad4f0e
S		 1347 }
1348
88576b4b
JS		 1349 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1350 h->ae_algo->ops->enable_vlan_filter) {
21b6fd34
JS		 1351 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1352 h->ae_algo->ops->enable_vlan_filter(h, true);
1353 else
1354 h->ae_algo->ops->enable_vlan_filter(h, false);
1355 }
d818396d 1356
88576b4b
JS		 1357 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1358 h->ae_algo->ops->enable_hw_strip_rxvtag) {
5f9a7732
PL		 1359 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1360 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1361 else
1362 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1363
1364 if (ret)
1365 return ret;
1366 }
1367
d1f04a80
JS		 1368 if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1369 if (features & NETIF_F_NTUPLE)
1370 h->ae_algo->ops->enable_fd(h, true);
1371 else
1372 h->ae_algo->ops->enable_fd(h, false);
1373 }
1374
76ad4f0e
S		 1375 netdev->features = features;
1376 return 0;
1377}
1378
9596f6f0
PL		 1379static void hns3_nic_get_stats64(struct net_device *netdev,
1380 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1381{
1382 struct hns3_nic_priv *priv = netdev_priv(netdev);
1383 int queue_num = priv->ae_handle->kinfo.num_tqps;
7a5d2a39 1384 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1385 struct hns3_enet_ring *ring;
1386 unsigned int start;
1387 unsigned int idx;
1388 u64 tx_bytes = 0;
1389 u64 rx_bytes = 0;
1390 u64 tx_pkts = 0;
1391 u64 rx_pkts = 0;
0a83231f
JS		 1392 u64 tx_drop = 0;
1393 u64 rx_drop = 0;
76ad4f0e 1394
a3083abb
JS		 1395 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1396 return;
1397
7a5d2a39
JS		 1398 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1399
76ad4f0e
S		 1400 for (idx = 0; idx < queue_num; idx++) {
1401 /* fetch the tx stats */
1402 ring = priv->ring_data[idx].ring;
1403 do {
d36d36ce 1404 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1405 tx_bytes += ring->stats.tx_bytes;
1406 tx_pkts += ring->stats.tx_pkts;
0a83231f
JS		 1407 tx_drop += ring->stats.tx_busy;
1408 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1409 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1410
1411 /* fetch the rx stats */
1412 ring = priv->ring_data[idx + queue_num].ring;
1413 do {
d36d36ce 1414 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1415 rx_bytes += ring->stats.rx_bytes;
1416 rx_pkts += ring->stats.rx_pkts;
0a83231f
JS		 1417 rx_drop += ring->stats.non_vld_descs;
1418 rx_drop += ring->stats.err_pkt_len;
1419 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1420 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1421 }
1422
1423 stats->tx_bytes = tx_bytes;
1424 stats->tx_packets = tx_pkts;
1425 stats->rx_bytes = rx_bytes;
1426 stats->rx_packets = rx_pkts;
1427
1428 stats->rx_errors = netdev->stats.rx_errors;
1429 stats->multicast = netdev->stats.multicast;
1430 stats->rx_length_errors = netdev->stats.rx_length_errors;
1431 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1432 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1433
1434 stats->tx_errors = netdev->stats.tx_errors;
0a83231f
JS		 1435 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1436 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1437 stats->collisions = netdev->stats.collisions;
1438 stats->rx_over_errors = netdev->stats.rx_over_errors;
1439 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1440 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1441 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1442 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1443 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1444 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1445 stats->tx_window_errors = netdev->stats.tx_window_errors;
1446 stats->rx_compressed = netdev->stats.rx_compressed;
1447 stats->tx_compressed = netdev->stats.tx_compressed;
1448}
1449
30d240df 1450static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1451{
30d240df 1452 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1453 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1454 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1455 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1456 u8 tc = mqprio_qopt->qopt.num_tc;
1457 u16 mode = mqprio_qopt->mode;
1458 u8 hw = mqprio_qopt->qopt.hw;
1459 bool if_running;
76ad4f0e
S		 1460 int ret;
1461
30d240df
YL		 1462 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1463 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1464 return -EOPNOTSUPP;
1465
76ad4f0e
S		 1466 if (tc > HNAE3_MAX_TC)
1467 return -EINVAL;
1468
76ad4f0e
S		 1469 if (!netdev)
1470 return -EINVAL;
1471
30d240df
YL		 1472 if_running = netif_running(netdev);
1473 if (if_running) {
1474 hns3_nic_net_stop(netdev);
1475 msleep(100);
76ad4f0e
S		 1476 }
1477
30d240df
YL		 1478 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1479 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1480 if (ret)
30d240df
YL		 1481 goto out;
1482
30d240df
YL		 1483 ret = hns3_nic_set_real_num_queue(netdev);
1484
1485out:
1486 if (if_running)
1487 hns3_nic_net_open(netdev);
1488
1489 return ret;
76ad4f0e
S		 1490}
1491
2572ac53 1492static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1493 void *type_data)
76ad4f0e 1494{
575ed7d3 1495 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1496 return -EOPNOTSUPP;
76ad4f0e 1497
30d240df 1498 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1499}
1500
1501static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1502 __be16 proto, u16 vid)
1503{
9780cb97 1504 struct hnae3_handle *h = hns3_get_handle(netdev);
103ce052 1505 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1506 int ret = -EIO;
1507
1508 if (h->ae_algo->ops->set_vlan_filter)
1509 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1510
103ce052
YL		 1511 if (!ret)
1512 set_bit(vid, priv->active_vlans);
1513
76ad4f0e
S		 1514 return ret;
1515}
1516
1517static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1518 __be16 proto, u16 vid)
1519{
9780cb97 1520 struct hnae3_handle *h = hns3_get_handle(netdev);
103ce052 1521 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1522 int ret = -EIO;
1523
1524 if (h->ae_algo->ops->set_vlan_filter)
1525 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1526
103ce052
YL		 1527 if (!ret)
1528 clear_bit(vid, priv->active_vlans);
1529
76ad4f0e
S		 1530 return ret;
1531}
1532
abe62a63 1533static int hns3_restore_vlan(struct net_device *netdev)
103ce052
YL		 1534{
1535 struct hns3_nic_priv *priv = netdev_priv(netdev);
abe62a63 1536 int ret = 0;
103ce052 1537 u16 vid;
103ce052
YL		 1538
1539 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1540 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
abe62a63
HT		 1541 if (ret) {
1542 netdev_err(netdev, "Restore vlan: %d filter, ret:%d\n",
1543 vid, ret);
1544 return ret;
1545 }
103ce052 1546 }
abe62a63
HT		 1547
1548 return ret;
103ce052
YL		 1549}
1550
76ad4f0e
S		 1551static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1552 u8 qos, __be16 vlan_proto)
1553{
9780cb97 1554 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1555 int ret = -EIO;
1556
1557 if (h->ae_algo->ops->set_vf_vlan_filter)
1558 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1559 qos, vlan_proto);
1560
1561 return ret;
1562}
1563
a8e8b7ff
S		 1564static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1565{
9780cb97 1566 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1567 int ret;
1568
1569 if (!h->ae_algo->ops->set_mtu)
1570 return -EOPNOTSUPP;
1571
a8e8b7ff 1572 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
6a2953dd 1573 if (ret)
a8e8b7ff
S		 1574 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1575 ret);
6a2953dd
YL		 1576 else
1577 netdev->mtu = new_mtu;
fe6362f9 1578
a8e8b7ff
S		 1579 return ret;
1580}
1581
f8fa222c
L		 1582static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1583{
1584 struct hns3_nic_priv *priv = netdev_priv(ndev);
1585 struct hns3_enet_ring *tx_ring = NULL;
1586 int timeout_queue = 0;
1587 int hw_head, hw_tail;
1588 int i;
1589
1590 /* Find the stopped queue the same way the stack does */
1591 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1592 struct netdev_queue *q;
1593 unsigned long trans_start;
1594
1595 q = netdev_get_tx_queue(ndev, i);
1596 trans_start = q->trans_start;
1597 if (netif_xmit_stopped(q) &&
1598 time_after(jiffies,
1599 (trans_start + ndev->watchdog_timeo))) {
1600 timeout_queue = i;
1601 break;
1602 }
1603 }
1604
1605 if (i == ndev->num_tx_queues) {
1606 netdev_info(ndev,
1607 "no netdev TX timeout queue found, timeout count: %llu\n",
1608 priv->tx_timeout_count);
1609 return false;
1610 }
1611
1612 tx_ring = priv->ring_data[timeout_queue].ring;
1613
1614 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1615 HNS3_RING_TX_RING_HEAD_REG);
1616 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1617 HNS3_RING_TX_RING_TAIL_REG);
1618 netdev_info(ndev,
1619 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1620 priv->tx_timeout_count,
1621 timeout_queue,
1622 tx_ring->next_to_use,
1623 tx_ring->next_to_clean,
1624 hw_head,
1625 hw_tail,
1626 readl(tx_ring->tqp_vector->mask_addr));
1627
1628 return true;
1629}
1630
1631static void hns3_nic_net_timeout(struct net_device *ndev)
1632{
1633 struct hns3_nic_priv *priv = netdev_priv(ndev);
f8fa222c
L		 1634 struct hnae3_handle *h = priv->ae_handle;
1635
1636 if (!hns3_get_tx_timeo_queue_info(ndev))
1637 return;
1638
1639 priv->tx_timeout_count++;
1640
1a2f7bf2
HT		 1641 /* request the reset, and let the hclge to determine
1642 * which reset level should be done
1643 */
f8fa222c 1644 if (h->ae_algo->ops->reset_event)
538d8ba0 1645 h->ae_algo->ops->reset_event(h->pdev, h);
f8fa222c
L		 1646}
1647
76ad4f0e
S		 1648static const struct net_device_ops hns3_nic_netdev_ops = {
1649 .ndo_open = hns3_nic_net_open,
1650 .ndo_stop = hns3_nic_net_stop,
1651 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1652 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1653 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a185d723 1654 .ndo_do_ioctl = hns3_nic_do_ioctl,
a8e8b7ff 1655 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1656 .ndo_set_features = hns3_nic_set_features,
1657 .ndo_get_stats64 = hns3_nic_get_stats64,
1658 .ndo_setup_tc = hns3_nic_setup_tc,
1659 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
76ad4f0e
S		 1660 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1661 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1662 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1663};
1664
bc59f827
FL		 1665static bool hns3_is_phys_func(struct pci_dev *pdev)
1666{
1667 u32 dev_id = pdev->device;
1668
1669 switch (dev_id) {
1670 case HNAE3_DEV_ID_GE:
1671 case HNAE3_DEV_ID_25GE:
1672 case HNAE3_DEV_ID_25GE_RDMA:
1673 case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1674 case HNAE3_DEV_ID_50GE_RDMA:
1675 case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1676 case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1677 return true;
1678 case HNAE3_DEV_ID_100G_VF:
1679 case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1680 return false;
1681 default:
1682 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1683 dev_id);
1684 }
1685
1686 return false;
1687}
1688
bc59f827
FL		 1689static void hns3_disable_sriov(struct pci_dev *pdev)
1690{
1691 /* If our VFs are assigned we cannot shut down SR-IOV
1692 * without causing issues, so just leave the hardware
1693 * available but disabled
1694 */
1695 if (pci_vfs_assigned(pdev)) {
1696 dev_warn(&pdev->dev,
1697 "disabling driver while VFs are assigned\n");
1698 return;
1699 }
1700
1701 pci_disable_sriov(pdev);
1702}
1703
10a954bc
JS		 1704static void hns3_get_dev_capability(struct pci_dev *pdev,
1705 struct hnae3_ae_dev *ae_dev)
1706{
73f88b00 1707 if (pdev->revision >= 0x21) {
10a954bc 1708 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
73f88b00
PL		 1709 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_GRO_B, 1);
1710 }
10a954bc
JS		 1711}
1712
76ad4f0e
S		 1713/* hns3_probe - Device initialization routine
1714 * @pdev: PCI device information struct
1715 * @ent: entry in hns3_pci_tbl
1716 *
1717 * hns3_probe initializes a PF identified by a pci_dev structure.
1718 * The OS initialization, configuring of the PF private structure,
1719 * and a hardware reset occur.
1720 *
1721 * Returns 0 on success, negative on failure
1722 */
1723static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1724{
1725 struct hnae3_ae_dev *ae_dev;
1726 int ret;
1727
1728 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1729 GFP_KERNEL);
1730 if (!ae_dev) {
1731 ret = -ENOMEM;
1732 return ret;
1733 }
1734
1735 ae_dev->pdev = pdev;
e92a0843 1736 ae_dev->flag = ent->driver_data;
76ad4f0e 1737 ae_dev->dev_type = HNAE3_DEV_KNIC;
7ce98982 1738 ae_dev->reset_type = HNAE3_NONE_RESET;
10a954bc 1739 hns3_get_dev_capability(pdev, ae_dev);
76ad4f0e
S		 1740 pci_set_drvdata(pdev, ae_dev);
1741
fb919349 1742 hnae3_register_ae_dev(ae_dev);
bc59f827 1743
bc59f827 1744 return 0;
76ad4f0e
S		 1745}
1746
1747/* hns3_remove - Device removal routine
1748 * @pdev: PCI device information struct
1749 */
1750static void hns3_remove(struct pci_dev *pdev)
1751{
1752 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1753
bc59f827
FL		 1754 if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1755 hns3_disable_sriov(pdev);
1756
76ad4f0e 1757 hnae3_unregister_ae_dev(ae_dev);
76ad4f0e
S		 1758}
1759
cfeff578
PL		 1760/**
1761 * hns3_pci_sriov_configure
1762 * @pdev: pointer to a pci_dev structure
1763 * @num_vfs: number of VFs to allocate
1764 *
1765 * Enable or change the number of VFs. Called when the user updates the number
1766 * of VFs in sysfs.
1767 **/
baff3ed7 1768static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
cfeff578
PL		 1769{
1770 int ret;
1771
1772 if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1773 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1774 return -EINVAL;
1775 }
1776
1777 if (num_vfs) {
1778 ret = pci_enable_sriov(pdev, num_vfs);
1779 if (ret)
1780 dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
baff3ed7
SM		 1781 else
1782 return num_vfs;
cfeff578
PL		 1783 } else if (!pci_vfs_assigned(pdev)) {
1784 pci_disable_sriov(pdev);
1785 } else {
1786 dev_warn(&pdev->dev,
1787 "Unable to free VFs because some are assigned to VMs.\n");
1788 }
1789
1790 return 0;
1791}
1792
4f043c3e
YL		 1793static void hns3_shutdown(struct pci_dev *pdev)
1794{
1795 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1796
1797 hnae3_unregister_ae_dev(ae_dev);
1798 devm_kfree(&pdev->dev, ae_dev);
1799 pci_set_drvdata(pdev, NULL);
1800
1801 if (system_state == SYSTEM_POWER_OFF)
1802 pci_set_power_state(pdev, PCI_D3hot);
1803}
1804
00bb612a
SJ		 1805static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1806 pci_channel_state_t state)
1807{
1808 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1809 pci_ers_result_t ret;
1810
1811 dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1812
1813 if (state == pci_channel_io_perm_failure)
1814 return PCI_ERS_RESULT_DISCONNECT;
1815
1816 if (!ae_dev) {
1817 dev_err(&pdev->dev,
1818 "Can't recover - error happened during device init\n");
1819 return PCI_ERS_RESULT_NONE;
1820 }
1821
af72a21f
SJ		 1822 if (ae_dev->ops->handle_hw_ras_error)
1823 ret = ae_dev->ops->handle_hw_ras_error(ae_dev);
00bb612a
SJ		 1824 else
1825 return PCI_ERS_RESULT_NONE;
1826
1827 return ret;
1828}
1829
538d8ba0
SJ		 1830static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1831{
1832 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1833 struct device *dev = &pdev->dev;
1834
1835 dev_info(dev, "requesting reset due to PCI error\n");
1836
1837 /* request the reset */
1838 if (ae_dev->ops->reset_event) {
1839 ae_dev->ops->reset_event(pdev, NULL);
1840 return PCI_ERS_RESULT_RECOVERED;
1841 }
1842
1843 return PCI_ERS_RESULT_DISCONNECT;
1844}
1845
26977990
HT		 1846static void hns3_reset_prepare(struct pci_dev *pdev)
1847{
1848 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1849
1850 dev_info(&pdev->dev, "hns3 flr prepare\n");
1851 if (ae_dev && ae_dev->ops && ae_dev->ops->flr_prepare)
1852 ae_dev->ops->flr_prepare(ae_dev);
1853}
1854
1855static void hns3_reset_done(struct pci_dev *pdev)
1856{
1857 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1858
1859 dev_info(&pdev->dev, "hns3 flr done\n");
1860 if (ae_dev && ae_dev->ops && ae_dev->ops->flr_done)
1861 ae_dev->ops->flr_done(ae_dev);
1862}
1863
00bb612a
SJ		 1864static const struct pci_error_handlers hns3_err_handler = {
1865 .error_detected = hns3_error_detected,
538d8ba0 1866 .slot_reset = hns3_slot_reset,
26977990
HT		 1867 .reset_prepare = hns3_reset_prepare,
1868 .reset_done = hns3_reset_done,
00bb612a
SJ		 1869};
1870
76ad4f0e
S		 1871static struct pci_driver hns3_driver = {
1872 .name = hns3_driver_name,
1873 .id_table = hns3_pci_tbl,
1874 .probe = hns3_probe,
1875 .remove = hns3_remove,
4f043c3e 1876 .shutdown = hns3_shutdown,
cfeff578 1877 .sriov_configure = hns3_pci_sriov_configure,
00bb612a 1878 .err_handler = &hns3_err_handler,
76ad4f0e
S		 1879};
1880
1881/* set default feature to hns3 */
1882static void hns3_set_default_feature(struct net_device *netdev)
1883{
a4c378bb
PL		 1884 struct hnae3_handle *h = hns3_get_handle(netdev);
1885 struct pci_dev *pdev = h->pdev;
1886
76ad4f0e
S		 1887 netdev->priv_flags |= IFF_UNICAST_FLT;
1888
1889 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1890 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1891 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1892 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
0361a8bc 1893 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1894
1895 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1896
1897 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1898
1899 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1900 NETIF_F_HW_VLAN_CTAG_FILTER |
5f9a7732 1901 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1902 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1903 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1904 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
0361a8bc 1905 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1906
1907 netdev->vlan_features |=
1908 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1909 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1910 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1911 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
0361a8bc 1912 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1913
1914 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3849d494 1915 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1916 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1917 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1918 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
0361a8bc 1919 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
a4c378bb 1920
d1f04a80 1921 if (pdev->revision >= 0x21) {
a4c378bb 1922 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
d1f04a80
JS		 1923
1924 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1925 netdev->hw_features |= NETIF_F_NTUPLE;
1926 netdev->features |= NETIF_F_NTUPLE;
1927 }
1928 }
76ad4f0e
S		 1929}
1930
1931static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1932 struct hns3_desc_cb *cb)
1933{
ccc23ef3 1934 unsigned int order = hnae3_page_order(ring);
76ad4f0e
S		 1935 struct page *p;
1936
1937 p = dev_alloc_pages(order);
1938 if (!p)
1939 return -ENOMEM;
1940
1941 cb->priv = p;
1942 cb->page_offset = 0;
1943 cb->reuse_flag = 0;
1944 cb->buf = page_address(p);
ccc23ef3 1945 cb->length = hnae3_page_size(ring);
76ad4f0e
S		 1946 cb->type = DESC_TYPE_PAGE;
1947
76ad4f0e
S		 1948 return 0;
1949}
1950
1951static void hns3_free_buffer(struct hns3_enet_ring *ring,
1952 struct hns3_desc_cb *cb)
1953{
1954 if (cb->type == DESC_TYPE_SKB)
1955 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1956 else if (!HNAE3_IS_TX_RING(ring))
1957 put_page((struct page *)cb->priv);
1958 memset(cb, 0, sizeof(*cb));
1959}
1960
1961static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1962{
1963 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1964 cb->length, ring_to_dma_dir(ring));
1965
39bbc873 1966 if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
76ad4f0e
S		 1967 return -EIO;
1968
1969 return 0;
1970}
1971
1972static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1973 struct hns3_desc_cb *cb)
1974{
1975 if (cb->type == DESC_TYPE_SKB)
1976 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1977 ring_to_dma_dir(ring));
6a3bff0d 1978 else if (cb->length)
76ad4f0e
S		 1979 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1980 ring_to_dma_dir(ring));
1981}
1982
1983static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1984{
1985 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1986 ring->desc[i].addr = 0;
1987}
1988
1989static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1990{
1991 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1992
1993 if (!ring->desc_cb[i].dma)
1994 return;
1995
1996 hns3_buffer_detach(ring, i);
1997 hns3_free_buffer(ring, cb);
1998}
1999
2000static void hns3_free_buffers(struct hns3_enet_ring *ring)
2001{
2002 int i;
2003
2004 for (i = 0; i < ring->desc_num; i++)
2005 hns3_free_buffer_detach(ring, i);
2006}
2007
2008/* free desc along with its attached buffer */
2009static void hns3_free_desc(struct hns3_enet_ring *ring)
2010{
d61c1a80
HT		 2011 int size = ring->desc_num * sizeof(ring->desc[0]);
2012
76ad4f0e
S		 2013 hns3_free_buffers(ring);
2014
d61c1a80
HT		 2015 if (ring->desc) {
2016 dma_free_coherent(ring_to_dev(ring), size,
2017 ring->desc, ring->desc_dma_addr);
2018 ring->desc = NULL;
2019 }
76ad4f0e
S		 2020}
2021
2022static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2023{
2024 int size = ring->desc_num * sizeof(ring->desc[0]);
2025
d61c1a80
HT		 2026 ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
2027 &ring->desc_dma_addr,
2028 GFP_KERNEL);
76ad4f0e
S		 2029 if (!ring->desc)
2030 return -ENOMEM;
2031
76ad4f0e
S		 2032 return 0;
2033}
2034
2035static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
2036 struct hns3_desc_cb *cb)
2037{
2038 int ret;
2039
2040 ret = hns3_alloc_buffer(ring, cb);
2041 if (ret)
2042 goto out;
2043
2044 ret = hns3_map_buffer(ring, cb);
2045 if (ret)
2046 goto out_with_buf;
2047
2048 return 0;
2049
2050out_with_buf:
564883bb 2051 hns3_free_buffer(ring, cb);
76ad4f0e
S		 2052out:
2053 return ret;
2054}
2055
2056static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2057{
2058 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2059
2060 if (ret)
2061 return ret;
2062
2063 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2064
2065 return 0;
2066}
2067
2068/* Allocate memory for raw pkg, and map with dma */
2069static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2070{
2071 int i, j, ret;
2072
2073 for (i = 0; i < ring->desc_num; i++) {
2074 ret = hns3_alloc_buffer_attach(ring, i);
2075 if (ret)
2076 goto out_buffer_fail;
2077 }
2078
2079 return 0;
2080
2081out_buffer_fail:
2082 for (j = i - 1; j >= 0; j--)
2083 hns3_free_buffer_detach(ring, j);
2084 return ret;
2085}
2086
2087/* detach a in-used buffer and replace with a reserved one */
2088static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2089 struct hns3_desc_cb *res_cb)
2090{
b9077428 2091 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 2092 ring->desc_cb[i] = *res_cb;
2093 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
4169a686 2094 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 2095}
2096
2097static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2098{
2099 ring->desc_cb[i].reuse_flag = 0;
2100 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2101 + ring->desc_cb[i].page_offset);
4169a686 2102 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 2103}
2104
2105static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2106 int *pkts)
2107{
2108 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2109
2110 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2111 (*bytes) += desc_cb->length;
ccc23ef3 2112 /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
76ad4f0e
S		 2113 hns3_free_buffer_detach(ring, ring->next_to_clean);
2114
2115 ring_ptr_move_fw(ring, next_to_clean);
2116}
2117
2118static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2119{
2120 int u = ring->next_to_use;
2121 int c = ring->next_to_clean;
2122
2123 if (unlikely(h > ring->desc_num))
2124 return 0;
2125
2126 return u > c ? (h > c && h <= u) : (h > c || h <= u);
2127}
2128
6a38a95f 2129void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
76ad4f0e
S		 2130{
2131 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
c3e4287e 2132 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 2133 struct netdev_queue *dev_queue;
2134 int bytes, pkts;
2135 int head;
2136
2137 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2138 rmb(); /* Make sure head is ready before touch any data */
2139
2140 if (is_ring_empty(ring) || head == ring->next_to_clean)
6a38a95f 2141 return; /* no data to poll */
76ad4f0e 2142
2ea8667b 2143 if (unlikely(!is_valid_clean_head(ring, head))) {
76ad4f0e
S		 2144 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2145 ring->next_to_use, ring->next_to_clean);
2146
2147 u64_stats_update_begin(&ring->syncp);
2148 ring->stats.io_err_cnt++;
2149 u64_stats_update_end(&ring->syncp);
6a38a95f 2150 return;
76ad4f0e
S		 2151 }
2152
2153 bytes = 0;
2154 pkts = 0;
6a38a95f 2155 while (head != ring->next_to_clean) {
76ad4f0e
S		 2156 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2157 /* Issue prefetch for next Tx descriptor */
2158 prefetch(&ring->desc_cb[ring->next_to_clean]);
76ad4f0e
S		 2159 }
2160
2161 ring->tqp_vector->tx_group.total_bytes += bytes;
2162 ring->tqp_vector->tx_group.total_packets += pkts;
2163
2164 u64_stats_update_begin(&ring->syncp);
2165 ring->stats.tx_bytes += bytes;
2166 ring->stats.tx_pkts += pkts;
2167 u64_stats_update_end(&ring->syncp);
2168
2169 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2170 netdev_tx_completed_queue(dev_queue, pkts, bytes);
2171
2172 if (unlikely(pkts && netif_carrier_ok(netdev) &&
2173 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2174 /* Make sure that anybody stopping the queue after this
2175 * sees the new next_to_clean.
2176 */
2177 smp_mb();
c3e4287e
JS		 2178 if (netif_tx_queue_stopped(dev_queue) &&
2179 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
76ad4f0e
S		 2180 netif_tx_wake_queue(dev_queue);
2181 ring->stats.restart_queue++;
2182 }
2183 }
76ad4f0e
S		 2184}
2185
2186static int hns3_desc_unused(struct hns3_enet_ring *ring)
2187{
2188 int ntc = ring->next_to_clean;
2189 int ntu = ring->next_to_use;
2190
2191 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2192}
2193
2194static void
2195hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2196{
2197 struct hns3_desc_cb *desc_cb;
2198 struct hns3_desc_cb res_cbs;
2199 int i, ret;
2200
2201 for (i = 0; i < cleand_count; i++) {
2202 desc_cb = &ring->desc_cb[ring->next_to_use];
2203 if (desc_cb->reuse_flag) {
2204 u64_stats_update_begin(&ring->syncp);
2205 ring->stats.reuse_pg_cnt++;
2206 u64_stats_update_end(&ring->syncp);
2207
2208 hns3_reuse_buffer(ring, ring->next_to_use);
2209 } else {
2210 ret = hns3_reserve_buffer_map(ring, &res_cbs);
2211 if (ret) {
2212 u64_stats_update_begin(&ring->syncp);
2213 ring->stats.sw_err_cnt++;
2214 u64_stats_update_end(&ring->syncp);
2215
2216 netdev_err(ring->tqp->handle->kinfo.netdev,
2217 "hnae reserve buffer map failed.\n");
2218 break;
2219 }
2220 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2221 }
2222
2223 ring_ptr_move_fw(ring, next_to_use);
2224 }
2225
2226 wmb(); /* Make all data has been write before submit */
2227 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2228}
2229
76ad4f0e
S		 2230static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2231 struct hns3_enet_ring *ring, int pull_len,
2232 struct hns3_desc_cb *desc_cb)
2233{
2234 struct hns3_desc *desc;
e56a15b7
HT		 2235 u32 truesize;
2236 int size;
76ad4f0e
S		 2237 int last_offset;
2238 bool twobufs;
2239
2240 twobufs = ((PAGE_SIZE < 8192) &&
ccc23ef3 2241 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
76ad4f0e
S		 2242
2243 desc = &ring->desc[ring->next_to_clean];
2244 size = le16_to_cpu(desc->rx.size);
2245
ccc23ef3 2246 truesize = hnae3_buf_size(ring);
885a882a
PL		 2247
2248 if (!twobufs)
ccc23ef3 2249 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
76ad4f0e
S		 2250
2251 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
885a882a 2252 size - pull_len, truesize);
76ad4f0e
S		 2253
2254 /* Avoid re-using remote pages,flag default unreuse */
2255 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2256 return;
2257
2258 if (twobufs) {
2259 /* If we are only owner of page we can reuse it */
2260 if (likely(page_count(desc_cb->priv) == 1)) {
2261 /* Flip page offset to other buffer */
2262 desc_cb->page_offset ^= truesize;
2263
2264 desc_cb->reuse_flag = 1;
2265 /* bump ref count on page before it is given*/
2266 get_page(desc_cb->priv);
2267 }
2268 return;
2269 }
2270
2271 /* Move offset up to the next cache line */
2272 desc_cb->page_offset += truesize;
2273
2274 if (desc_cb->page_offset <= last_offset) {
2275 desc_cb->reuse_flag = 1;
2276 /* Bump ref count on page before it is given*/
2277 get_page(desc_cb->priv);
2278 }
2279}
2280
2281static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2282 struct hns3_desc *desc)
2283{
2284 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2285 int l3_type, l4_type;
2286 u32 bd_base_info;
2287 int ol4_type;
2288 u32 l234info;
2289
2290 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2291 l234info = le32_to_cpu(desc->rx.l234_info);
2292
2293 skb->ip_summed = CHECKSUM_NONE;
2294
2295 skb_checksum_none_assert(skb);
2296
2297 if (!(netdev->features & NETIF_F_RXCSUM))
2298 return;
2299
5337b725
PL		 2300 /* We MUST enable hardware checksum before enabling hardware GRO */
2301 if (skb_shinfo(skb)->gso_size) {
2302 skb->ip_summed = CHECKSUM_UNNECESSARY;
2303 return;
2304 }
2305
76ad4f0e 2306 /* check if hardware has done checksum */
ccc23ef3 2307 if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
76ad4f0e
S		 2308 return;
2309
ccc23ef3
PL		 2310 if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2311 hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2312 hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2313 hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
76ad4f0e
S		 2314 u64_stats_update_begin(&ring->syncp);
2315 ring->stats.l3l4_csum_err++;
2316 u64_stats_update_end(&ring->syncp);
2317
2318 return;
2319 }
2320
ccc23ef3
PL		 2321 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2322 HNS3_RXD_L3ID_S);
2323 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2324 HNS3_RXD_L4ID_S);
76ad4f0e 2325
ccc23ef3
PL		 2326 ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2327 HNS3_RXD_OL4ID_S);
76ad4f0e
S		 2328 switch (ol4_type) {
2329 case HNS3_OL4_TYPE_MAC_IN_UDP:
2330 case HNS3_OL4_TYPE_NVGRE:
2331 skb->csum_level = 1;
64061deb 2332 /* fall through */
76ad4f0e
S		 2333 case HNS3_OL4_TYPE_NO_TUN:
2334 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
ffa051fa
PL		 2335 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2336 l3_type == HNS3_L3_TYPE_IPV6) &&
2337 (l4_type == HNS3_L4_TYPE_UDP ||
2338 l4_type == HNS3_L4_TYPE_TCP ||
2339 l4_type == HNS3_L4_TYPE_SCTP))
76ad4f0e
S		 2340 skb->ip_summed = CHECKSUM_UNNECESSARY;
2341 break;
085920ba
JS		 2342 default:
2343 break;
76ad4f0e
S		 2344 }
2345}
2346
d43e5aca
YL		 2347static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2348{
4110bcf6
PL		 2349 if (skb_has_frag_list(skb))
2350 napi_gro_flush(&ring->tqp_vector->napi, false);
2351
d43e5aca
YL		 2352 napi_gro_receive(&ring->tqp_vector->napi, skb);
2353}
2354
34456a7c
JS		 2355static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2356 struct hns3_desc *desc, u32 l234info,
2357 u16 *vlan_tag)
1e8f8bd3
PL		 2358{
2359 struct pci_dev *pdev = ring->tqp->handle->pdev;
1e8f8bd3
PL		 2360
2361 if (pdev->revision == 0x20) {
34456a7c
JS		 2362 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2363 if (!(*vlan_tag & VLAN_VID_MASK))
2364 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
1e8f8bd3 2365
34456a7c 2366 return (*vlan_tag != 0);
1e8f8bd3
PL		 2367 }
2368
2369#define HNS3_STRP_OUTER_VLAN 0x1
2370#define HNS3_STRP_INNER_VLAN 0x2
2371
ccc23ef3
PL		 2372 switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2373 HNS3_RXD_STRP_TAGP_S)) {
1e8f8bd3 2374 case HNS3_STRP_OUTER_VLAN:
34456a7c
JS		 2375 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2376 return true;
1e8f8bd3 2377 case HNS3_STRP_INNER_VLAN:
34456a7c
JS		 2378 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2379 return true;
1e8f8bd3 2380 default:
34456a7c 2381 return false;
1e8f8bd3 2382 }
1e8f8bd3
PL		 2383}
2384
72b50a78
PL		 2385static int hns3_alloc_skb(struct hns3_enet_ring *ring, int length,
2386 unsigned char *va)
2387{
2388#define HNS3_NEED_ADD_FRAG 1
2389 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2390 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2391 struct sk_buff *skb;
2392
2393 ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
2394 skb = ring->skb;
2395 if (unlikely(!skb)) {
2396 netdev_err(netdev, "alloc rx skb fail\n");
2397
2398 u64_stats_update_begin(&ring->syncp);
2399 ring->stats.sw_err_cnt++;
2400 u64_stats_update_end(&ring->syncp);
2401
2402 return -ENOMEM;
2403 }
2404
2405 prefetchw(skb->data);
2406
2407 ring->pending_buf = 1;
4110bcf6
PL		 2408 ring->frag_num = 0;
2409 ring->tail_skb = NULL;
72b50a78
PL		 2410 if (length <= HNS3_RX_HEAD_SIZE) {
2411 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2412
2413 /* We can reuse buffer as-is, just make sure it is local */
2414 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2415 desc_cb->reuse_flag = 1;
2416 else /* This page cannot be reused so discard it */
2417 put_page(desc_cb->priv);
2418
2419 ring_ptr_move_fw(ring, next_to_clean);
2420 return 0;
2421 }
2422 u64_stats_update_begin(&ring->syncp);
2423 ring->stats.seg_pkt_cnt++;
2424 u64_stats_update_end(&ring->syncp);
2425
2426 ring->pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2427 __skb_put(skb, ring->pull_len);
4110bcf6 2428 hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,
72b50a78
PL		 2429 desc_cb);
2430 ring_ptr_move_fw(ring, next_to_clean);
2431
2432 return HNS3_NEED_ADD_FRAG;
2433}
2434
2435static int hns3_add_frag(struct hns3_enet_ring *ring, struct hns3_desc *desc,
2436 struct sk_buff **out_skb, bool pending)
2437{
2438 struct sk_buff *skb = *out_skb;
4110bcf6
PL		 2439 struct sk_buff *head_skb = *out_skb;
2440 struct sk_buff *new_skb;
72b50a78
PL		 2441 struct hns3_desc_cb *desc_cb;
2442 struct hns3_desc *pre_desc;
2443 u32 bd_base_info;
2444 int pre_bd;
2445
2446 /* if there is pending bd, the SW param next_to_clean has moved
2447 * to next and the next is NULL
2448 */
2449 if (pending) {
2450 pre_bd = (ring->next_to_clean - 1 + ring->desc_num) %
2451 ring->desc_num;
2452 pre_desc = &ring->desc[pre_bd];
2453 bd_base_info = le32_to_cpu(pre_desc->rx.bd_base_info);
2454 } else {
2455 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2456 }
2457
2458 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2459 desc = &ring->desc[ring->next_to_clean];
2460 desc_cb = &ring->desc_cb[ring->next_to_clean];
2461 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2462 if (!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2463 return -ENXIO;
2464
4110bcf6
PL		 2465 if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {
2466 new_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2467 HNS3_RX_HEAD_SIZE);
2468 if (unlikely(!new_skb)) {
2469 netdev_err(ring->tqp->handle->kinfo.netdev,
2470 "alloc rx skb frag fail\n");
2471 return -ENXIO;
2472 }
2473 ring->frag_num = 0;
2474
2475 if (ring->tail_skb) {
2476 ring->tail_skb->next = new_skb;
2477 ring->tail_skb = new_skb;
2478 } else {
2479 skb_shinfo(skb)->frag_list = new_skb;
2480 ring->tail_skb = new_skb;
2481 }
2482 }
2483
2484 if (ring->tail_skb) {
2485 head_skb->truesize += hnae3_buf_size(ring);
2486 head_skb->data_len += le16_to_cpu(desc->rx.size);
2487 head_skb->len += le16_to_cpu(desc->rx.size);
2488 skb = ring->tail_skb;
2489 }
2490
2491 hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);
72b50a78
PL		 2492 ring_ptr_move_fw(ring, next_to_clean);
2493 ring->pending_buf++;
2494 }
2495
2496 return 0;
2497}
2498
5337b725
PL		 2499static void hns3_set_gro_param(struct sk_buff *skb, u32 l234info,
2500 u32 bd_base_info)
2501{
2502 u16 gro_count;
2503 u32 l3_type;
2504
2505 gro_count = hnae3_get_field(l234info, HNS3_RXD_GRO_COUNT_M,
2506 HNS3_RXD_GRO_COUNT_S);
2507 /* if there is no HW GRO, do not set gro params */
2508 if (!gro_count)
2509 return;
2510
2511 /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
2512 * to skb_shinfo(skb)->gso_segs
2513 */
2514 NAPI_GRO_CB(skb)->count = gro_count;
2515
2516 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2517 HNS3_RXD_L3ID_S);
2518 if (l3_type == HNS3_L3_TYPE_IPV4)
2519 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
2520 else if (l3_type == HNS3_L3_TYPE_IPV6)
2521 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
2522 else
2523 return;
2524
2525 skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,
2526 HNS3_RXD_GRO_SIZE_M,
2527 HNS3_RXD_GRO_SIZE_S);
2528 if (skb_shinfo(skb)->gso_size)
2529 tcp_gro_complete(skb);
2530}
2531
8e4c877d
PL		 2532static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2533 struct sk_buff *skb)
2534{
2535 struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2536 struct hnae3_handle *handle = ring->tqp->handle;
2537 enum pkt_hash_types rss_type;
2538
2539 if (le32_to_cpu(desc->rx.rss_hash))
2540 rss_type = handle->kinfo.rss_type;
2541 else
2542 rss_type = PKT_HASH_TYPE_NONE;
2543
2544 skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2545}
2546
76ad4f0e 2547static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
72b50a78 2548 struct sk_buff **out_skb)
76ad4f0e
S		 2549{
2550 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
72b50a78 2551 struct sk_buff *skb = ring->skb;
76ad4f0e
S		 2552 struct hns3_desc_cb *desc_cb;
2553 struct hns3_desc *desc;
76ad4f0e 2554 u32 bd_base_info;
76ad4f0e
S		 2555 u32 l234info;
2556 int length;
72b50a78 2557 int ret;
76ad4f0e
S		 2558
2559 desc = &ring->desc[ring->next_to_clean];
2560 desc_cb = &ring->desc_cb[ring->next_to_clean];
2561
2562 prefetch(desc);
2563
ca61f05e 2564 length = le16_to_cpu(desc->rx.size);
76ad4f0e 2565 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
76ad4f0e
S		 2566
2567 /* Check valid BD */
ccc23ef3 2568 if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
72b50a78 2569 return -ENXIO;
76ad4f0e 2570
72b50a78
PL		 2571 if (!skb)
2572 ring->va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
76ad4f0e
S		 2573
2574 /* Prefetch first cache line of first page
2575 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2576 * line size is 64B so need to prefetch twice to make it 128B. But in
2577 * actual we can have greater size of caches with 128B Level 1 cache
2578 * lines. In such a case, single fetch would suffice to cache in the
2579 * relevant part of the header.
2580 */
72b50a78 2581 prefetch(ring->va);
76ad4f0e 2582#if L1_CACHE_BYTES < 128
72b50a78 2583 prefetch(ring->va + L1_CACHE_BYTES);
76ad4f0e
S		 2584#endif
2585
72b50a78
PL		 2586 if (!skb) {
2587 ret = hns3_alloc_skb(ring, length, ring->va);
2588 *out_skb = skb = ring->skb;
76ad4f0e 2589
72b50a78
PL		 2590 if (ret < 0) /* alloc buffer fail */
2591 return ret;
2592 if (ret > 0) { /* need add frag */
2593 ret = hns3_add_frag(ring, desc, &skb, false);
2594 if (ret)
2595 return ret;
76ad4f0e 2596
72b50a78
PL		 2597 /* As the head data may be changed when GRO enable, copy
2598 * the head data in after other data rx completed
2599 */
2600 memcpy(skb->data, ring->va,
2601 ALIGN(ring->pull_len, sizeof(long)));
2602 }
76ad4f0e 2603 } else {
72b50a78
PL		 2604 ret = hns3_add_frag(ring, desc, &skb, true);
2605 if (ret)
2606 return ret;
76ad4f0e 2607
72b50a78
PL		 2608 /* As the head data may be changed when GRO enable, copy
2609 * the head data in after other data rx completed
2610 */
2611 memcpy(skb->data, ring->va,
2612 ALIGN(ring->pull_len, sizeof(long)));
76ad4f0e
S		 2613 }
2614
928d369a 2615 l234info = le32_to_cpu(desc->rx.l234_info);
72b50a78 2616 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
928d369a 2617
ca61f05e
PL		 2618 /* Based on hw strategy, the tag offloaded will be stored at
2619 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2620 * in one layer tag case.
2621 */
2622 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2623 u16 vlan_tag;
2624
34456a7c 2625 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
ca61f05e
PL		 2626 __vlan_hwaccel_put_tag(skb,
2627 htons(ETH_P_8021Q),
2628 vlan_tag);
2629 }
2630
ccc23ef3 2631 if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
76ad4f0e
S		 2632 u64_stats_update_begin(&ring->syncp);
2633 ring->stats.non_vld_descs++;
2634 u64_stats_update_end(&ring->syncp);
2635
2636 dev_kfree_skb_any(skb);
2637 return -EINVAL;
2638 }
2639
2640 if (unlikely((!desc->rx.pkt_len) ||
ccc23ef3 2641 hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
76ad4f0e
S		 2642 u64_stats_update_begin(&ring->syncp);
2643 ring->stats.err_pkt_len++;
2644 u64_stats_update_end(&ring->syncp);
2645
2646 dev_kfree_skb_any(skb);
2647 return -EFAULT;
2648 }
2649
ccc23ef3 2650 if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
76ad4f0e
S		 2651 u64_stats_update_begin(&ring->syncp);
2652 ring->stats.l2_err++;
2653 u64_stats_update_end(&ring->syncp);
2654
2655 dev_kfree_skb_any(skb);
2656 return -EFAULT;
2657 }
2658
2659 u64_stats_update_begin(&ring->syncp);
2660 ring->stats.rx_pkts++;
2661 ring->stats.rx_bytes += skb->len;
2662 u64_stats_update_end(&ring->syncp);
2663
2664 ring->tqp_vector->rx_group.total_bytes += skb->len;
2665
5337b725
PL		 2666 /* This is needed in order to enable forwarding support */
2667 hns3_set_gro_param(skb, l234info, bd_base_info);
2668
76ad4f0e 2669 hns3_rx_checksum(ring, skb, desc);
72b50a78 2670 *out_skb = skb;
8e4c877d
PL		 2671 hns3_set_rx_skb_rss_type(ring, skb);
2672
76ad4f0e
S		 2673 return 0;
2674}
2675
d43e5aca
YL		 2676int hns3_clean_rx_ring(
2677 struct hns3_enet_ring *ring, int budget,
2678 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2679{
2680#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2681 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2682 int recv_pkts, recv_bds, clean_count, err;
72b50a78
PL		 2683 int unused_count = hns3_desc_unused(ring) - ring->pending_buf;
2684 struct sk_buff *skb = ring->skb;
2685 int num;
76ad4f0e
S		 2686
2687 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2688 rmb(); /* Make sure num taken effect before the other data is touched */
2689
2690 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2691 num -= unused_count;
2692
2693 while (recv_pkts < budget && recv_bds < num) {
2694 /* Reuse or realloc buffers */
2695 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2696 hns3_nic_alloc_rx_buffers(ring,
2697 clean_count + unused_count);
2698 clean_count = 0;
72b50a78
PL		 2699 unused_count = hns3_desc_unused(ring) -
2700 ring->pending_buf;
76ad4f0e
S		 2701 }
2702
2703 /* Poll one pkt */
72b50a78 2704 err = hns3_handle_rx_bd(ring, &skb);
76ad4f0e
S		 2705 if (unlikely(!skb)) /* This fault cannot be repaired */
2706 goto out;
2707
72b50a78
PL		 2708 if (err == -ENXIO) { /* Do not get FE for the packet */
2709 goto out;
2710 } else if (unlikely(err)) { /* Do jump the err */
2711 recv_bds += ring->pending_buf;
2712 clean_count += ring->pending_buf;
2713 ring->skb = NULL;
2714 ring->pending_buf = 0;
76ad4f0e
S		 2715 continue;
2716 }
2717
2718 /* Do update ip stack process */
2719 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2720 rx_fn(ring, skb);
72b50a78
PL		 2721 recv_bds += ring->pending_buf;
2722 clean_count += ring->pending_buf;
2723 ring->skb = NULL;
2724 ring->pending_buf = 0;
76ad4f0e
S		 2725
2726 recv_pkts++;
2727 }
2728
2729out:
2730 /* Make all data has been write before submit */
2731 if (clean_count + unused_count > 0)
2732 hns3_nic_alloc_rx_buffers(ring,
2733 clean_count + unused_count);
2734
2735 return recv_pkts;
2736}
2737
2738static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2739{
50477f37
FL		 2740 struct hns3_enet_tqp_vector *tqp_vector =
2741 ring_group->ring->tqp_vector;
76ad4f0e 2742 enum hns3_flow_level_range new_flow_level;
50477f37
FL		 2743 int packets_per_msecs;
2744 int bytes_per_msecs;
2745 u32 time_passed_ms;
76ad4f0e 2746 u16 new_int_gl;
76ad4f0e 2747
50477f37 2748 if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
76ad4f0e
S		 2749 return false;
2750
2751 if (ring_group->total_packets == 0) {
d420d2de
YL		 2752 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2753 ring_group->coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 2754 return true;
2755 }
2756
2757 /* Simple throttlerate management
2758 * 0-10MB/s lower (50000 ints/s)
2759 * 10-20MB/s middle (20000 ints/s)
2760 * 20-1249MB/s high (18000 ints/s)
2761 * > 40000pps ultra (8000 ints/s)
2762 */
d420d2de
YL		 2763 new_flow_level = ring_group->coal.flow_level;
2764 new_int_gl = ring_group->coal.int_gl;
50477f37
FL		 2765 time_passed_ms =
2766 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2767
2768 if (!time_passed_ms)
2769 return false;
2770
2771 do_div(ring_group->total_packets, time_passed_ms);
2772 packets_per_msecs = ring_group->total_packets;
2773
2774 do_div(ring_group->total_bytes, time_passed_ms);
2775 bytes_per_msecs = ring_group->total_bytes;
2776
2777#define HNS3_RX_LOW_BYTE_RATE 10000
2778#define HNS3_RX_MID_BYTE_RATE 20000
76ad4f0e
S		 2779
2780 switch (new_flow_level) {
2781 case HNS3_FLOW_LOW:
50477f37 2782 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2783 new_flow_level = HNS3_FLOW_MID;
2784 break;
2785 case HNS3_FLOW_MID:
50477f37 2786 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
76ad4f0e 2787 new_flow_level = HNS3_FLOW_HIGH;
50477f37 2788 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2789 new_flow_level = HNS3_FLOW_LOW;
2790 break;
2791 case HNS3_FLOW_HIGH:
2792 case HNS3_FLOW_ULTRA:
2793 default:
50477f37 2794 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
76ad4f0e
S		 2795 new_flow_level = HNS3_FLOW_MID;
2796 break;
2797 }
2798
50477f37
FL		 2799#define HNS3_RX_ULTRA_PACKET_RATE 40
2800
2801 if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2802 &tqp_vector->rx_group == ring_group)
76ad4f0e
S		 2803 new_flow_level = HNS3_FLOW_ULTRA;
2804
2805 switch (new_flow_level) {
2806 case HNS3_FLOW_LOW:
2807 new_int_gl = HNS3_INT_GL_50K;
2808 break;
2809 case HNS3_FLOW_MID:
2810 new_int_gl = HNS3_INT_GL_20K;
2811 break;
2812 case HNS3_FLOW_HIGH:
2813 new_int_gl = HNS3_INT_GL_18K;
2814 break;
2815 case HNS3_FLOW_ULTRA:
2816 new_int_gl = HNS3_INT_GL_8K;
2817 break;
2818 default:
2819 break;
2820 }
2821
2822 ring_group->total_bytes = 0;
2823 ring_group->total_packets = 0;
d420d2de
YL		 2824 ring_group->coal.flow_level = new_flow_level;
2825 if (new_int_gl != ring_group->coal.int_gl) {
2826 ring_group->coal.int_gl = new_int_gl;
76ad4f0e
S		 2827 return true;
2828 }
2829 return false;
2830}
2831
2832static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2833{
dc114fce
FL		 2834 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2835 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2836 bool rx_update, tx_update;
2837
3f97bd23
FL		 2838 if (tqp_vector->int_adapt_down > 0) {
2839 tqp_vector->int_adapt_down--;
2840 return;
2841 }
2842
d420d2de 2843 if (rx_group->coal.gl_adapt_enable) {
dc114fce
FL		 2844 rx_update = hns3_get_new_int_gl(rx_group);
2845 if (rx_update)
2846 hns3_set_vector_coalesce_rx_gl(tqp_vector,
d420d2de 2847 rx_group->coal.int_gl);
dc114fce
FL		 2848 }
2849
d420d2de 2850 if (tx_group->coal.gl_adapt_enable) {
dc114fce
FL		 2851 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2852 if (tx_update)
2853 hns3_set_vector_coalesce_tx_gl(tqp_vector,
d420d2de 2854 tx_group->coal.int_gl);
76ad4f0e 2855 }
3f97bd23 2856
50477f37 2857 tqp_vector->last_jiffies = jiffies;
3f97bd23 2858 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
76ad4f0e
S		 2859}
2860
2861static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2862{
f22463fd 2863 struct hns3_nic_priv *priv = netdev_priv(napi->dev);
76ad4f0e
S		 2864 struct hns3_enet_ring *ring;
2865 int rx_pkt_total = 0;
2866
2867 struct hns3_enet_tqp_vector *tqp_vector =
2868 container_of(napi, struct hns3_enet_tqp_vector, napi);
2869 bool clean_complete = true;
2870 int rx_budget;
2871
f22463fd
HT		 2872 if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
2873 napi_complete(napi);
2874 return 0;
2875 }
2876
76ad4f0e
S		 2877 /* Since the actual Tx work is minimal, we can give the Tx a larger
2878 * budget and be more aggressive about cleaning up the Tx descriptors.
2879 */
6a38a95f
PL		 2880 hns3_for_each_ring(ring, tqp_vector->tx_group)
2881 hns3_clean_tx_ring(ring);
76ad4f0e
S		 2882
2883 /* make sure rx ring budget not smaller than 1 */
2884 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2885
2886 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2887 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2888 hns3_rx_skb);
76ad4f0e
S		 2889
2890 if (rx_cleaned >= rx_budget)
2891 clean_complete = false;
2892
2893 rx_pkt_total += rx_cleaned;
2894 }
2895
2896 tqp_vector->rx_group.total_packets += rx_pkt_total;
2897
2898 if (!clean_complete)
2899 return budget;
2900
f22463fd
HT		 2901 if (likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) &&
2902 napi_complete(napi)) {
2903 hns3_update_new_int_gl(tqp_vector);
2904 hns3_mask_vector_irq(tqp_vector, 1);
2905 }
76ad4f0e
S		 2906
2907 return rx_pkt_total;
2908}
2909
2910static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2911 struct hnae3_ring_chain_node *head)
2912{
2913 struct pci_dev *pdev = tqp_vector->handle->pdev;
2914 struct hnae3_ring_chain_node *cur_chain = head;
2915 struct hnae3_ring_chain_node *chain;
2916 struct hns3_enet_ring *tx_ring;
2917 struct hns3_enet_ring *rx_ring;
2918
2919 tx_ring = tqp_vector->tx_group.ring;
2920 if (tx_ring) {
2921 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
ccc23ef3
PL		 2922 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2923 HNAE3_RING_TYPE_TX);
2924 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2925 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 2926
2927 cur_chain->next = NULL;
2928
2929 while (tx_ring->next) {
2930 tx_ring = tx_ring->next;
2931
2932 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2933 GFP_KERNEL);
2934 if (!chain)
1129d424 2935 goto err_free_chain;
76ad4f0e
S		 2936
2937 cur_chain->next = chain;
2938 chain->tqp_index = tx_ring->tqp->tqp_index;
ccc23ef3
PL		 2939 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2940 HNAE3_RING_TYPE_TX);
2941 hnae3_set_field(chain->int_gl_idx,
2942 HNAE3_RING_GL_IDX_M,
2943 HNAE3_RING_GL_IDX_S,
2944 HNAE3_RING_GL_TX);
76ad4f0e
S		 2945
2946 cur_chain = chain;
2947 }
2948 }
2949
2950 rx_ring = tqp_vector->rx_group.ring;
2951 if (!tx_ring && rx_ring) {
2952 cur_chain->next = NULL;
2953 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
ccc23ef3
PL		 2954 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2955 HNAE3_RING_TYPE_RX);
2956 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2957 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 2958
2959 rx_ring = rx_ring->next;
2960 }
2961
2962 while (rx_ring) {
2963 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2964 if (!chain)
1129d424 2965 goto err_free_chain;
76ad4f0e
S		 2966
2967 cur_chain->next = chain;
2968 chain->tqp_index = rx_ring->tqp->tqp_index;
ccc23ef3
PL		 2969 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2970 HNAE3_RING_TYPE_RX);
2971 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2972 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
f230c6c5 2973
76ad4f0e
S		 2974 cur_chain = chain;
2975
2976 rx_ring = rx_ring->next;
2977 }
2978
2979 return 0;
1129d424
HT		 2980
2981err_free_chain:
2982 cur_chain = head->next;
2983 while (cur_chain) {
2984 chain = cur_chain->next;
2985 devm_kfree(&pdev->dev, chain);
2986 cur_chain = chain;
2987 }
2988
2989 return -ENOMEM;
76ad4f0e
S		 2990}
2991
2992static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2993 struct hnae3_ring_chain_node *head)
2994{
2995 struct pci_dev *pdev = tqp_vector->handle->pdev;
2996 struct hnae3_ring_chain_node *chain_tmp, *chain;
2997
2998 chain = head->next;
2999
3000 while (chain) {
3001 chain_tmp = chain->next;
3002 devm_kfree(&pdev->dev, chain);
3003 chain = chain_tmp;
3004 }
3005}
3006
3007static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
3008 struct hns3_enet_ring *ring)
3009{
3010 ring->next = group->ring;
3011 group->ring = ring;
3012
3013 group->count++;
3014}
3015
15040788
PL		 3016static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
3017{
3018 struct pci_dev *pdev = priv->ae_handle->pdev;
3019 struct hns3_enet_tqp_vector *tqp_vector;
3020 int num_vectors = priv->vector_num;
3021 int numa_node;
3022 int vector_i;
3023
3024 numa_node = dev_to_node(&pdev->dev);
3025
3026 for (vector_i = 0; vector_i < num_vectors; vector_i++) {
3027 tqp_vector = &priv->tqp_vector[vector_i];
3028 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
3029 &tqp_vector->affinity_mask);
3030 }
3031}
3032
76ad4f0e
S		 3033static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
3034{
3035 struct hnae3_ring_chain_node vector_ring_chain;
3036 struct hnae3_handle *h = priv->ae_handle;
3037 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e 3038 int ret = 0;
3c424930 3039 int i;
76ad4f0e 3040
15040788
PL		 3041 hns3_nic_set_cpumask(priv);
3042
6cbd6d33
YL		 3043 for (i = 0; i < priv->vector_num; i++) {
3044 tqp_vector = &priv->tqp_vector[i];
3045 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
3046 tqp_vector->num_tqps = 0;
3047 }
76ad4f0e 3048
6cbd6d33
YL		 3049 for (i = 0; i < h->kinfo.num_tqps; i++) {
3050 u16 vector_i = i % priv->vector_num;
3051 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 3052
3053 tqp_vector = &priv->tqp_vector[vector_i];
3054
3055 hns3_add_ring_to_group(&tqp_vector->tx_group,
3056 priv->ring_data[i].ring);
3057
3058 hns3_add_ring_to_group(&tqp_vector->rx_group,
3059 priv->ring_data[i + tqp_num].ring);
3060
76ad4f0e
S		 3061 priv->ring_data[i].ring->tqp_vector = tqp_vector;
3062 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
6cbd6d33 3063 tqp_vector->num_tqps++;
76ad4f0e
S		 3064 }
3065
6cbd6d33 3066 for (i = 0; i < priv->vector_num; i++) {
76ad4f0e
S		 3067 tqp_vector = &priv->tqp_vector[i];
3068
3069 tqp_vector->rx_group.total_bytes = 0;
3070 tqp_vector->rx_group.total_packets = 0;
3071 tqp_vector->tx_group.total_bytes = 0;
3072 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 3073 tqp_vector->handle = h;
3074
3075 ret = hns3_get_vector_ring_chain(tqp_vector,
3076 &vector_ring_chain);
3077 if (ret)
6cbd6d33 3078 return ret;
76ad4f0e
S		 3079
3080 ret = h->ae_algo->ops->map_ring_to_vector(h,
3081 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 3082
3083 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3084
6cbd6d33 3085 if (ret)
3c424930 3086 goto map_ring_fail;
6cbd6d33 3087
76ad4f0e
S		 3088 netif_napi_add(priv->netdev, &tqp_vector->napi,
3089 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
3090 }
3091
6cbd6d33 3092 return 0;
3c424930
HT		 3093
3094map_ring_fail:
3095 while (i--)
3096 netif_napi_del(&priv->tqp_vector[i].napi);
3097
3098 return ret;
6cbd6d33
YL		 3099}
3100
3101static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
3102{
3103 struct hnae3_handle *h = priv->ae_handle;
3104 struct hns3_enet_tqp_vector *tqp_vector;
3105 struct hnae3_vector_info *vector;
3106 struct pci_dev *pdev = h->pdev;
3107 u16 tqp_num = h->kinfo.num_tqps;
3108 u16 vector_num;
3109 int ret = 0;
3110 u16 i;
3111
3112 /* RSS size, cpu online and vector_num should be the same */
3113 /* Should consider 2p/4p later */
3114 vector_num = min_t(u16, num_online_cpus(), tqp_num);
3115 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
3116 GFP_KERNEL);
3117 if (!vector)
3118 return -ENOMEM;
3119
3120 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
3121
3122 priv->vector_num = vector_num;
3123 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
3124 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
3125 GFP_KERNEL);
3126 if (!priv->tqp_vector) {
3127 ret = -ENOMEM;
3128 goto out;
3129 }
3130
3131 for (i = 0; i < priv->vector_num; i++) {
3132 tqp_vector = &priv->tqp_vector[i];
3133 tqp_vector->idx = i;
3134 tqp_vector->mask_addr = vector[i].io_addr;
3135 tqp_vector->vector_irq = vector[i].vector;
3136 hns3_vector_gl_rl_init(tqp_vector, priv);
3137 }
3138
76ad4f0e
S		 3139out:
3140 devm_kfree(&pdev->dev, vector);
3141 return ret;
3142}
3143
6cbd6d33
YL		 3144static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
3145{
3146 group->ring = NULL;
3147 group->count = 0;
3148}
3149
76ad4f0e
S		 3150static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
3151{
3152 struct hnae3_ring_chain_node vector_ring_chain;
3153 struct hnae3_handle *h = priv->ae_handle;
3154 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 3155 int i, ret;
3156
3157 for (i = 0; i < priv->vector_num; i++) {
3158 tqp_vector = &priv->tqp_vector[i];
3159
3160 ret = hns3_get_vector_ring_chain(tqp_vector,
3161 &vector_ring_chain);
3162 if (ret)
3163 return ret;
3164
3165 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
3166 tqp_vector->vector_irq, &vector_ring_chain);
3167 if (ret)
3168 return ret;
3169
3170 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3171
3172 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
3173 (void)irq_set_affinity_hint(
3174 priv->tqp_vector[i].vector_irq,
3175 NULL);
ae064e61 3176 free_irq(priv->tqp_vector[i].vector_irq,
3177 &priv->tqp_vector[i]);
76ad4f0e
S		 3178 }
3179
3180 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
6cbd6d33
YL		 3181 hns3_clear_ring_group(&tqp_vector->rx_group);
3182 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 3183 netif_napi_del(&priv->tqp_vector[i].napi);
3184 }
3185
6cbd6d33
YL		 3186 return 0;
3187}
3188
3189static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
3190{
3191 struct hnae3_handle *h = priv->ae_handle;
3192 struct pci_dev *pdev = h->pdev;
3193 int i, ret;
3194
3195 for (i = 0; i < priv->vector_num; i++) {
3196 struct hns3_enet_tqp_vector *tqp_vector;
3197
3198 tqp_vector = &priv->tqp_vector[i];
3199 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3200 if (ret)
3201 return ret;
3202 }
76ad4f0e 3203
6cbd6d33 3204 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 3205 return 0;
3206}
3207
3208static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3209 int ring_type)
3210{
3211 struct hns3_nic_ring_data *ring_data = priv->ring_data;
3212 int queue_num = priv->ae_handle->kinfo.num_tqps;
3213 struct pci_dev *pdev = priv->ae_handle->pdev;
3214 struct hns3_enet_ring *ring;
3215
3216 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
3217 if (!ring)
3218 return -ENOMEM;
3219
3220 if (ring_type == HNAE3_RING_TYPE_TX) {
3221 ring_data[q->tqp_index].ring = ring;
66b44730 3222 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 3223 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3224 } else {
3225 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 3226 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 3227 ring->io_base = q->io_base;
3228 }
3229
ccc23ef3 3230 hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
76ad4f0e 3231
76ad4f0e
S		 3232 ring->tqp = q;
3233 ring->desc = NULL;
3234 ring->desc_cb = NULL;
3235 ring->dev = priv->dev;
3236 ring->desc_dma_addr = 0;
3237 ring->buf_size = q->buf_size;
3238 ring->desc_num = q->desc_num;
3239 ring->next_to_use = 0;
3240 ring->next_to_clean = 0;
3241
3242 return 0;
3243}
3244
3245static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3246 struct hns3_nic_priv *priv)
3247{
3248 int ret;
3249
3250 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3251 if (ret)
3252 return ret;
3253
3254 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
1129d424
HT		 3255 if (ret) {
3256 devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring);
76ad4f0e 3257 return ret;
1129d424 3258 }
76ad4f0e
S		 3259
3260 return 0;
3261}
3262
3263static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3264{
3265 struct hnae3_handle *h = priv->ae_handle;
3266 struct pci_dev *pdev = h->pdev;
3267 int i, ret;
3268
3269 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
3270 sizeof(*priv->ring_data) * 2,
3271 GFP_KERNEL);
3272 if (!priv->ring_data)
3273 return -ENOMEM;
3274
3275 for (i = 0; i < h->kinfo.num_tqps; i++) {
3276 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3277 if (ret)
3278 goto err;
3279 }
3280
3281 return 0;
3282err:
1129d424
HT		 3283 while (i--) {
3284 devm_kfree(priv->dev, priv->ring_data[i].ring);
3285 devm_kfree(priv->dev,
3286 priv->ring_data[i + h->kinfo.num_tqps].ring);
3287 }
3288
76ad4f0e
S		 3289 devm_kfree(&pdev->dev, priv->ring_data);
3290 return ret;
3291}
3292
f1f779ce
PL		 3293static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3294{
3295 struct hnae3_handle *h = priv->ae_handle;
3296 int i;
3297
3298 for (i = 0; i < h->kinfo.num_tqps; i++) {
3299 devm_kfree(priv->dev, priv->ring_data[i].ring);
3300 devm_kfree(priv->dev,
3301 priv->ring_data[i + h->kinfo.num_tqps].ring);
3302 }
3303 devm_kfree(priv->dev, priv->ring_data);
3304}
3305
76ad4f0e
S		 3306static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3307{
3308 int ret;
3309
3310 if (ring->desc_num <= 0 || ring->buf_size <= 0)
3311 return -EINVAL;
3312
3313 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3314 GFP_KERNEL);
3315 if (!ring->desc_cb) {
3316 ret = -ENOMEM;
3317 goto out;
3318 }
3319
3320 ret = hns3_alloc_desc(ring);
3321 if (ret)
3322 goto out_with_desc_cb;
3323
3324 if (!HNAE3_IS_TX_RING(ring)) {
3325 ret = hns3_alloc_ring_buffers(ring);
3326 if (ret)
3327 goto out_with_desc;
3328 }
3329
3330 return 0;
3331
3332out_with_desc:
3333 hns3_free_desc(ring);
3334out_with_desc_cb:
3335 kfree(ring->desc_cb);
3336 ring->desc_cb = NULL;
3337out:
3338 return ret;
3339}
3340
3341static void hns3_fini_ring(struct hns3_enet_ring *ring)
3342{
3343 hns3_free_desc(ring);
3344 kfree(ring->desc_cb);
3345 ring->desc_cb = NULL;
3346 ring->next_to_clean = 0;
3347 ring->next_to_use = 0;
3348}
3349
1db9b1bf 3350static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 3351{
3352 int bd_size_type;
3353
3354 switch (buf_size) {
3355 case 512:
3356 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3357 break;
3358 case 1024:
3359 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3360 break;
3361 case 2048:
3362 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3363 break;
3364 case 4096:
3365 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3366 break;
3367 default:
3368 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3369 }
3370
3371 return bd_size_type;
3372}
3373
3374static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3375{
3376 dma_addr_t dma = ring->desc_dma_addr;
3377 struct hnae3_queue *q = ring->tqp;
3378
3379 if (!HNAE3_IS_TX_RING(ring)) {
3380 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
3381 (u32)dma);
3382 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3383 (u32)((dma >> 31) >> 1));
3384
3385 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3386 hns3_buf_size2type(ring->buf_size));
3387 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3388 ring->desc_num / 8 - 1);
3389
3390 } else {
3391 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3392 (u32)dma);
3393 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3394 (u32)((dma >> 31) >> 1));
3395
76ad4f0e
S		 3396 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3397 ring->desc_num / 8 - 1);
3398 }
3399}
3400
fe46c859
YL		 3401static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3402{
3403 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3404 int i;
3405
3406 for (i = 0; i < HNAE3_MAX_TC; i++) {
3407 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3408 int j;
3409
3410 if (!tc_info->enable)
3411 continue;
3412
3413 for (j = 0; j < tc_info->tqp_count; j++) {
3414 struct hnae3_queue *q;
3415
3416 q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3417 hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3418 tc_info->tc);
3419 }
3420 }
3421}
3422
5668abda 3423int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3424{
3425 struct hnae3_handle *h = priv->ae_handle;
3426 int ring_num = h->kinfo.num_tqps * 2;
3427 int i, j;
3428 int ret;
3429
3430 for (i = 0; i < ring_num; i++) {
3431 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3432 if (ret) {
3433 dev_err(priv->dev,
3434 "Alloc ring memory fail! ret=%d\n", ret);
3435 goto out_when_alloc_ring_memory;
3436 }
3437
76ad4f0e
S		 3438 u64_stats_init(&priv->ring_data[i].ring->syncp);
3439 }
3440
3441 return 0;
3442
3443out_when_alloc_ring_memory:
3444 for (j = i - 1; j >= 0; j--)
ee83f776 3445 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 3446
3447 return -ENOMEM;
3448}
3449
5668abda 3450int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3451{
3452 struct hnae3_handle *h = priv->ae_handle;
3453 int i;
3454
3455 for (i = 0; i < h->kinfo.num_tqps; i++) {
76ad4f0e
S		 3456 hns3_fini_ring(priv->ring_data[i].ring);
3457 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3458 }
76ad4f0e
S		 3459 return 0;
3460}
3461
3462/* Set mac addr if it is configured. or leave it to the AE driver */
abe62a63 3463static int hns3_init_mac_addr(struct net_device *netdev, bool init)
76ad4f0e
S		 3464{
3465 struct hns3_nic_priv *priv = netdev_priv(netdev);
3466 struct hnae3_handle *h = priv->ae_handle;
3467 u8 mac_addr_temp[ETH_ALEN];
abe62a63 3468 int ret = 0;
76ad4f0e 3469
edf76c8e 3470 if (h->ae_algo->ops->get_mac_addr && init) {
76ad4f0e
S		 3471 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3472 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3473 }
3474
3475 /* Check if the MAC address is valid, if not get a random one */
3476 if (!is_valid_ether_addr(netdev->dev_addr)) {
3477 eth_hw_addr_random(netdev);
3478 dev_warn(priv->dev, "using random MAC address %pM\n",
3479 netdev->dev_addr);
76ad4f0e 3480 }
139e8792
L		 3481
3482 if (h->ae_algo->ops->set_mac_addr)
abe62a63 3483 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
139e8792 3484
abe62a63 3485 return ret;
76ad4f0e
S		 3486}
3487
7ce98982
JS		 3488static int hns3_restore_fd_rules(struct net_device *netdev)
3489{
3490 struct hnae3_handle *h = hns3_get_handle(netdev);
3491 int ret = 0;
3492
3493 if (h->ae_algo->ops->restore_fd_rules)
3494 ret = h->ae_algo->ops->restore_fd_rules(h);
3495
3496 return ret;
3497}
3498
3499static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3500{
3501 struct hnae3_handle *h = hns3_get_handle(netdev);
3502
3503 if (h->ae_algo->ops->del_all_fd_entries)
3504 h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3505}
3506
76ad4f0e
S		 3507static void hns3_nic_set_priv_ops(struct net_device *netdev)
3508{
3509 struct hns3_nic_priv *priv = netdev_priv(netdev);
3510
e4c38d6e 3511 priv->ops.fill_desc = hns3_fill_desc;
76ad4f0e 3512 if ((netdev->features & NETIF_F_TSO) ||
e4c38d6e 3513 (netdev->features & NETIF_F_TSO6))
76ad4f0e 3514 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
e4c38d6e 3515 else
76ad4f0e 3516 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
76ad4f0e
S		 3517}
3518
337460de
YL		 3519static int hns3_client_start(struct hnae3_handle *handle)
3520{
3521 if (!handle->ae_algo->ops->client_start)
3522 return 0;
3523
3524 return handle->ae_algo->ops->client_start(handle);
3525}
3526
3527static void hns3_client_stop(struct hnae3_handle *handle)
3528{
3529 if (!handle->ae_algo->ops->client_stop)
3530 return;
3531
3532 handle->ae_algo->ops->client_stop(handle);
3533}
3534
76ad4f0e
S		 3535static int hns3_client_init(struct hnae3_handle *handle)
3536{
3537 struct pci_dev *pdev = handle->pdev;
08ca3d58 3538 u16 alloc_tqps, max_rss_size;
76ad4f0e
S		 3539 struct hns3_nic_priv *priv;
3540 struct net_device *netdev;
3541 int ret;
3542
08ca3d58
HT		 3543 handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
3544 &max_rss_size);
3545 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
76ad4f0e
S		 3546 if (!netdev)
3547 return -ENOMEM;
3548
3549 priv = netdev_priv(netdev);
3550 priv->dev = &pdev->dev;
3551 priv->netdev = netdev;
3552 priv->ae_handle = handle;
f8fa222c 3553 priv->tx_timeout_count = 0;
76ad4f0e
S		 3554
3555 handle->kinfo.netdev = netdev;
3556 handle->priv = (void *)priv;
3557
edf76c8e 3558 hns3_init_mac_addr(netdev, true);
76ad4f0e
S		 3559
3560 hns3_set_default_feature(netdev);
3561
3562 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3563 netdev->priv_flags |= IFF_UNICAST_FLT;
3564 netdev->netdev_ops = &hns3_nic_netdev_ops;
3565 SET_NETDEV_DEV(netdev, &pdev->dev);
3566 hns3_ethtool_set_ops(netdev);
3567 hns3_nic_set_priv_ops(netdev);
3568
3569 /* Carrier off reporting is important to ethtool even BEFORE open */
3570 netif_carrier_off(netdev);
3571
3572 ret = hns3_get_ring_config(priv);
3573 if (ret) {
3574 ret = -ENOMEM;
3575 goto out_get_ring_cfg;
3576 }
3577
6cbd6d33
YL		 3578 ret = hns3_nic_alloc_vector_data(priv);
3579 if (ret) {
3580 ret = -ENOMEM;
3581 goto out_alloc_vector_data;
3582 }
3583
76ad4f0e
S		 3584 ret = hns3_nic_init_vector_data(priv);
3585 if (ret) {
3586 ret = -ENOMEM;
3587 goto out_init_vector_data;
3588 }
3589
3590 ret = hns3_init_all_ring(priv);
3591 if (ret) {
3592 ret = -ENOMEM;
3593 goto out_init_ring_data;
3594 }
3595
3596 ret = register_netdev(netdev);
3597 if (ret) {
3598 dev_err(priv->dev, "probe register netdev fail!\n");
3599 goto out_reg_netdev_fail;
3600 }
3601
337460de
YL		 3602 ret = hns3_client_start(handle);
3603 if (ret) {
3604 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);
3605 goto out_reg_netdev_fail;
3606 }
3607
986743db
YL		 3608 hns3_dcbnl_setup(handle);
3609
a9556d8c 3610 hns3_dbg_init(handle);
3611
bd975002 3612 /* MTU range: (ETH_MIN_MTU(kernel default) - 9702) */
4ee09281 3613 netdev->max_mtu = HNS3_MAX_MTU;
a8e8b7ff 3614
352285f1
HT		 3615 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
3616
76ad4f0e
S		 3617 return ret;
3618
3619out_reg_netdev_fail:
3620out_init_ring_data:
3621 (void)hns3_nic_uninit_vector_data(priv);
76ad4f0e 3622out_init_vector_data:
6cbd6d33
YL		 3623 hns3_nic_dealloc_vector_data(priv);
3624out_alloc_vector_data:
3625 priv->ring_data = NULL;
76ad4f0e
S		 3626out_get_ring_cfg:
3627 priv->ae_handle = NULL;
3628 free_netdev(netdev);
3629 return ret;
3630}
3631
3632static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3633{
3634 struct net_device *netdev = handle->kinfo.netdev;
3635 struct hns3_nic_priv *priv = netdev_priv(netdev);
3636 int ret;
3637
337460de
YL		 3638 hns3_client_stop(handle);
3639
2c794374
JS		 3640 hns3_remove_hw_addr(netdev);
3641
76ad4f0e
S		 3642 if (netdev->reg_state != NETREG_UNINITIALIZED)
3643 unregister_netdev(netdev);
3644
352285f1
HT		 3645 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
3646 netdev_warn(netdev, "already uninitialized\n");
3647 goto out_netdev_free;
3648 }
3649
c27ad61b
JS		 3650 hns3_del_all_fd_rules(netdev, true);
3651
e32a805a
FL		 3652 hns3_force_clear_all_rx_ring(handle);
3653
76ad4f0e
S		 3654 ret = hns3_nic_uninit_vector_data(priv);
3655 if (ret)
3656 netdev_err(netdev, "uninit vector error\n");
3657
6cbd6d33
YL		 3658 ret = hns3_nic_dealloc_vector_data(priv);
3659 if (ret)
3660 netdev_err(netdev, "dealloc vector error\n");
3661
76ad4f0e
S		 3662 ret = hns3_uninit_all_ring(priv);
3663 if (ret)
3664 netdev_err(netdev, "uninit ring error\n");
3665
a2ddee8c
YL		 3666 hns3_put_ring_config(priv);
3667
a9556d8c 3668 hns3_dbg_uninit(handle);
3669
76ad4f0e
S		 3670 priv->ring_data = NULL;
3671
352285f1 3672out_netdev_free:
76ad4f0e
S		 3673 free_netdev(netdev);
3674}
3675
3676static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3677{
3678 struct net_device *netdev = handle->kinfo.netdev;
3679
3680 if (!netdev)
3681 return;
3682
3683 if (linkup) {
3684 netif_carrier_on(netdev);
3685 netif_tx_wake_all_queues(netdev);
3686 netdev_info(netdev, "link up\n");
3687 } else {
3688 netif_carrier_off(netdev);
3689 netif_tx_stop_all_queues(netdev);
3690 netdev_info(netdev, "link down\n");
3691 }
3692}
3693
9df8f79a
YL		 3694static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3695{
3696 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3697 struct net_device *ndev = kinfo->netdev;
075cfdd6 3698 bool if_running;
9df8f79a 3699 int ret;
9df8f79a
YL		 3700
3701 if (tc > HNAE3_MAX_TC)
3702 return -EINVAL;
3703
3704 if (!ndev)
3705 return -ENODEV;
3706
075cfdd6
CIK		 3707 if_running = netif_running(ndev);
3708
9df8f79a
YL		 3709 if (if_running) {
3710 (void)hns3_nic_net_stop(ndev);
3711 msleep(100);
3712 }
3713
3714 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3715 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3716 if (ret)
3717 goto err_out;
3718
9df8f79a
YL		 3719 ret = hns3_nic_set_real_num_queue(ndev);
3720
3721err_out:
3722 if (if_running)
3723 (void)hns3_nic_net_open(ndev);
3724
3725 return ret;
3726}
3727
abe62a63 3728static int hns3_recover_hw_addr(struct net_device *ndev)
bb6b94a8
L		 3729{
3730 struct netdev_hw_addr_list *list;
3731 struct netdev_hw_addr *ha, *tmp;
abe62a63 3732 int ret = 0;
bb6b94a8
L		 3733
3734 /* go through and sync uc_addr entries to the device */
3735 list = &ndev->uc;
abe62a63
HT		 3736 list_for_each_entry_safe(ha, tmp, &list->list, list) {
3737 ret = hns3_nic_uc_sync(ndev, ha->addr);
3738 if (ret)
3739 return ret;
3740 }
bb6b94a8
L		 3741
3742 /* go through and sync mc_addr entries to the device */
3743 list = &ndev->mc;
abe62a63
HT		 3744 list_for_each_entry_safe(ha, tmp, &list->list, list) {
3745 ret = hns3_nic_mc_sync(ndev, ha->addr);
3746 if (ret)
3747 return ret;
3748 }
3749
3750 return ret;
bb6b94a8
L		 3751}
3752
2c794374
JS		 3753static void hns3_remove_hw_addr(struct net_device *netdev)
3754{
3755 struct netdev_hw_addr_list *list;
3756 struct netdev_hw_addr *ha, *tmp;
3757
3758 hns3_nic_uc_unsync(netdev, netdev->dev_addr);
3759
3760 /* go through and unsync uc_addr entries to the device */
3761 list = &netdev->uc;
3762 list_for_each_entry_safe(ha, tmp, &list->list, list)
3763 hns3_nic_uc_unsync(netdev, ha->addr);
3764
3765 /* go through and unsync mc_addr entries to the device */
3766 list = &netdev->mc;
3767 list_for_each_entry_safe(ha, tmp, &list->list, list)
3768 if (ha->refcount > 1)
3769 hns3_nic_mc_unsync(netdev, ha->addr);
3770}
3771
82172ec9 3772static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
bb6b94a8 3773{
82172ec9 3774 while (ring->next_to_clean != ring->next_to_use) {
e32a805a 3775 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
82172ec9
YL		 3776 hns3_free_buffer_detach(ring, ring->next_to_clean);
3777 ring_ptr_move_fw(ring, next_to_clean);
3778 }
3779}
3780
e32a805a
FL		 3781static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3782{
3783 struct hns3_desc_cb res_cbs;
3784 int ret;
3785
3786 while (ring->next_to_use != ring->next_to_clean) {
3787 /* When a buffer is not reused, it's memory has been
3788 * freed in hns3_handle_rx_bd or will be freed by
3789 * stack, so we need to replace the buffer here.
3790 */
3791 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3792 ret = hns3_reserve_buffer_map(ring, &res_cbs);
3793 if (ret) {
3794 u64_stats_update_begin(&ring->syncp);
3795 ring->stats.sw_err_cnt++;
3796 u64_stats_update_end(&ring->syncp);
3797 /* if alloc new buffer fail, exit directly
3798 * and reclear in up flow.
3799 */
3800 netdev_warn(ring->tqp->handle->kinfo.netdev,
3801 "reserve buffer map failed, ret = %d\n",
3802 ret);
3803 return ret;
3804 }
3805 hns3_replace_buffer(ring, ring->next_to_use,
3806 &res_cbs);
3807 }
3808 ring_ptr_move_fw(ring, next_to_use);
3809 }
3810
3811 return 0;
3812}
3813
3814static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
82172ec9 3815{
82172ec9
YL		 3816 while (ring->next_to_use != ring->next_to_clean) {
3817 /* When a buffer is not reused, it's memory has been
3818 * freed in hns3_handle_rx_bd or will be freed by
3819 * stack, so only need to unmap the buffer here.
3820 */
3821 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3822 hns3_unmap_buffer(ring,
3823 &ring->desc_cb[ring->next_to_use]);
3824 ring->desc_cb[ring->next_to_use].dma = 0;
3825 }
3826
3827 ring_ptr_move_fw(ring, next_to_use);
3828 }
bb6b94a8
L		 3829}
3830
e32a805a
FL		 3831static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3832{
3833 struct net_device *ndev = h->kinfo.netdev;
3834 struct hns3_nic_priv *priv = netdev_priv(ndev);
3835 struct hns3_enet_ring *ring;
3836 u32 i;
3837
3838 for (i = 0; i < h->kinfo.num_tqps; i++) {
3839 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3840 hns3_force_clear_rx_ring(ring);
3841 }
3842}
3843
bb6b94a8
L		 3844static void hns3_clear_all_ring(struct hnae3_handle *h)
3845{
3846 struct net_device *ndev = h->kinfo.netdev;
3847 struct hns3_nic_priv *priv = netdev_priv(ndev);
3848 u32 i;
3849
3850 for (i = 0; i < h->kinfo.num_tqps; i++) {
3851 struct netdev_queue *dev_queue;
3852 struct hns3_enet_ring *ring;
3853
3854 ring = priv->ring_data[i].ring;
82172ec9 3855 hns3_clear_tx_ring(ring);
bb6b94a8
L		 3856 dev_queue = netdev_get_tx_queue(ndev,
3857 priv->ring_data[i].queue_index);
3858 netdev_tx_reset_queue(dev_queue);
3859
3860 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
e32a805a
FL		 3861 /* Continue to clear other rings even if clearing some
3862 * rings failed.
3863 */
82172ec9 3864 hns3_clear_rx_ring(ring);
bb6b94a8
L		 3865 }
3866}
3867
e32a805a
FL		 3868int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3869{
3870 struct net_device *ndev = h->kinfo.netdev;
3871 struct hns3_nic_priv *priv = netdev_priv(ndev);
3872 struct hns3_enet_ring *rx_ring;
3873 int i, j;
3874 int ret;
3875
3876 for (i = 0; i < h->kinfo.num_tqps; i++) {
abe62a63
HT		 3877 ret = h->ae_algo->ops->reset_queue(h, i);
3878 if (ret)
3879 return ret;
3880
e32a805a
FL		 3881 hns3_init_ring_hw(priv->ring_data[i].ring);
3882
3883 /* We need to clear tx ring here because self test will
3884 * use the ring and will not run down before up
3885 */
3886 hns3_clear_tx_ring(priv->ring_data[i].ring);
3887 priv->ring_data[i].ring->next_to_clean = 0;
3888 priv->ring_data[i].ring->next_to_use = 0;
3889
3890 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3891 hns3_init_ring_hw(rx_ring);
3892 ret = hns3_clear_rx_ring(rx_ring);
3893 if (ret)
3894 return ret;
3895
3896 /* We can not know the hardware head and tail when this
3897 * function is called in reset flow, so we reuse all desc.
3898 */
3899 for (j = 0; j < rx_ring->desc_num; j++)
3900 hns3_reuse_buffer(rx_ring, j);
3901
3902 rx_ring->next_to_clean = 0;
3903 rx_ring->next_to_use = 0;
3904 }
3905
fe46c859
YL		 3906 hns3_init_tx_ring_tc(priv);
3907
e32a805a
FL		 3908 return 0;
3909}
3910
0378eccc
HT		 3911static void hns3_store_coal(struct hns3_nic_priv *priv)
3912{
3913 /* ethtool only support setting and querying one coal
3914 * configuation for now, so save the vector 0' coal
3915 * configuation here in order to restore it.
3916 */
3917 memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3918 sizeof(struct hns3_enet_coalesce));
3919 memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3920 sizeof(struct hns3_enet_coalesce));
3921}
3922
3923static void hns3_restore_coal(struct hns3_nic_priv *priv)
3924{
3925 u16 vector_num = priv->vector_num;
3926 int i;
3927
3928 for (i = 0; i < vector_num; i++) {
3929 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3930 sizeof(struct hns3_enet_coalesce));
3931 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3932 sizeof(struct hns3_enet_coalesce));
3933 }
3934}
3935
bb6b94a8
L		 3936static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3937{
7e68177b 3938 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
bb6b94a8
L		 3939 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3940 struct net_device *ndev = kinfo->netdev;
3a11f446
HT		 3941 struct hns3_nic_priv *priv = netdev_priv(ndev);
3942
3943 if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
3944 return 0;
bb6b94a8 3945
7e68177b
HT		 3946 /* it is cumbersome for hardware to pick-and-choose entries for deletion
3947 * from table space. Hence, for function reset software intervention is
3948 * required to delete the entries
3949 */
3950 if (hns3_dev_ongoing_func_reset(ae_dev)) {
3951 hns3_remove_hw_addr(ndev);
3952 hns3_del_all_fd_rules(ndev, false);
3953 }
3954
bb6b94a8 3955 if (!netif_running(ndev))
196b8760 3956 return 0;
bb6b94a8
L		 3957
3958 return hns3_nic_net_stop(ndev);
3959}
3960
3961static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3962{
3963 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3a11f446 3964 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
bb6b94a8
L		 3965 int ret = 0;
3966
3967 if (netif_running(kinfo->netdev)) {
3968 ret = hns3_nic_net_up(kinfo->netdev);
3969 if (ret) {
3970 netdev_err(kinfo->netdev,
3971 "hns net up fail, ret=%d!\n", ret);
3972 return ret;
3973 }
bb6b94a8
L		 3974 }
3975
3a11f446
HT		 3976 clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
3977
bb6b94a8
L		 3978 return ret;
3979}
3980
3981static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3982{
3983 struct net_device *netdev = handle->kinfo.netdev;
3984 struct hns3_nic_priv *priv = netdev_priv(netdev);
ad94a397 3985 bool vlan_filter_enable;
bb6b94a8
L		 3986 int ret;
3987
abe62a63
HT		 3988 ret = hns3_init_mac_addr(netdev, false);
3989 if (ret)
3990 return ret;
3991
3992 ret = hns3_recover_hw_addr(netdev);
3993 if (ret)
3994 return ret;
3995
3996 ret = hns3_update_promisc_mode(netdev, handle->netdev_flags);
3997 if (ret)
3998 return ret;
3999
ad94a397
JS		 4000 vlan_filter_enable = netdev->flags & IFF_PROMISC ? false : true;
4001 hns3_enable_vlan_filter(netdev, vlan_filter_enable);
4002
103ce052 4003 /* Hardware table is only clear when pf resets */
abe62a63
HT		 4004 if (!(handle->flags & HNAE3_SUPPORT_VF)) {
4005 ret = hns3_restore_vlan(netdev);
8eb0e623
HT		 4006 if (ret)
4007 return ret;
abe62a63 4008 }
103ce052 4009
abe62a63
HT		 4010 ret = hns3_restore_fd_rules(netdev);
4011 if (ret)
4012 return ret;
7ce98982 4013
bb6b94a8
L		 4014 /* Carrier off reporting is important to ethtool even BEFORE open */
4015 netif_carrier_off(netdev);
4016
1c92981c
HT		 4017 ret = hns3_nic_alloc_vector_data(priv);
4018 if (ret)
4019 return ret;
4020
0378eccc
HT		 4021 hns3_restore_coal(priv);
4022
bb6b94a8
L		 4023 ret = hns3_nic_init_vector_data(priv);
4024 if (ret)
1c92981c 4025 goto err_dealloc_vector;
bb6b94a8
L		 4026
4027 ret = hns3_init_all_ring(priv);
1c92981c
HT		 4028 if (ret)
4029 goto err_uninit_vector;
bb6b94a8 4030
352285f1
HT		 4031 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
4032
1c92981c
HT		 4033 return ret;
4034
4035err_uninit_vector:
4036 hns3_nic_uninit_vector_data(priv);
4037 priv->ring_data = NULL;
4038err_dealloc_vector:
4039 hns3_nic_dealloc_vector_data(priv);
4040
bb6b94a8
L		 4041 return ret;
4042}
4043
4044static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
4045{
4046 struct net_device *netdev = handle->kinfo.netdev;
4047 struct hns3_nic_priv *priv = netdev_priv(netdev);
4048 int ret;
4049
352285f1
HT		 4050 if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
4051 netdev_warn(netdev, "already uninitialized\n");
4052 return 0;
4053 }
4054
e32a805a 4055 hns3_force_clear_all_rx_ring(handle);
bb6b94a8
L		 4056
4057 ret = hns3_nic_uninit_vector_data(priv);
4058 if (ret) {
4059 netdev_err(netdev, "uninit vector error\n");
4060 return ret;
4061 }
4062
0378eccc
HT		 4063 hns3_store_coal(priv);
4064
1c92981c
HT		 4065 ret = hns3_nic_dealloc_vector_data(priv);
4066 if (ret)
4067 netdev_err(netdev, "dealloc vector error\n");
4068
bb6b94a8
L		 4069 ret = hns3_uninit_all_ring(priv);
4070 if (ret)
4071 netdev_err(netdev, "uninit ring error\n");
4072
352285f1
HT		 4073 clear_bit(HNS3_NIC_STATE_INITED, &priv->state);
4074
bb6b94a8
L		 4075 return ret;
4076}
4077
4078static int hns3_reset_notify(struct hnae3_handle *handle,
4079 enum hnae3_reset_notify_type type)
4080{
4081 int ret = 0;
4082
4083 switch (type) {
4084 case HNAE3_UP_CLIENT:
741e1778
SM		 4085 ret = hns3_reset_notify_up_enet(handle);
4086 break;
bb6b94a8
L		 4087 case HNAE3_DOWN_CLIENT:
4088 ret = hns3_reset_notify_down_enet(handle);
4089 break;
4090 case HNAE3_INIT_CLIENT:
4091 ret = hns3_reset_notify_init_enet(handle);
4092 break;
4093 case HNAE3_UNINIT_CLIENT:
4094 ret = hns3_reset_notify_uninit_enet(handle);
4095 break;
4096 default:
4097 break;
4098 }
4099
4100 return ret;
4101}
4102
0378eccc 4103static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
f1f779ce
PL		 4104{
4105 struct hns3_nic_priv *priv = netdev_priv(netdev);
4106 struct hnae3_handle *h = hns3_get_handle(netdev);
4107 int ret;
4108
4109 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
4110 if (ret)
4111 return ret;
4112
4113 ret = hns3_get_ring_config(priv);
4114 if (ret)
4115 return ret;
4116
6cbd6d33
YL		 4117 ret = hns3_nic_alloc_vector_data(priv);
4118 if (ret)
4119 goto err_alloc_vector;
4120
0378eccc 4121 hns3_restore_coal(priv);
351dad5e 4122
f1f779ce
PL		 4123 ret = hns3_nic_init_vector_data(priv);
4124 if (ret)
4125 goto err_uninit_vector;
4126
4127 ret = hns3_init_all_ring(priv);
4128 if (ret)
4129 goto err_put_ring;
4130
4131 return 0;
4132
4133err_put_ring:
4134 hns3_put_ring_config(priv);
4135err_uninit_vector:
4136 hns3_nic_uninit_vector_data(priv);
6cbd6d33
YL		 4137err_alloc_vector:
4138 hns3_nic_dealloc_vector_data(priv);
f1f779ce
PL		 4139 return ret;
4140}
4141
4142static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
4143{
4144 return (new_tqp_num / num_tc) * num_tc;
4145}
4146
4147int hns3_set_channels(struct net_device *netdev,
4148 struct ethtool_channels *ch)
4149{
4150 struct hns3_nic_priv *priv = netdev_priv(netdev);
4151 struct hnae3_handle *h = hns3_get_handle(netdev);
4152 struct hnae3_knic_private_info *kinfo = &h->kinfo;
4153 bool if_running = netif_running(netdev);
4154 u32 new_tqp_num = ch->combined_count;
4155 u16 org_tqp_num;
4156 int ret;
4157
4158 if (ch->rx_count || ch->tx_count)
4159 return -EINVAL;
4160
2d7187ce 4161 if (new_tqp_num > hns3_get_max_available_channels(h) ||
f1f779ce
PL		 4162 new_tqp_num < kinfo->num_tc) {
4163 dev_err(&netdev->dev,
4164 "Change tqps fail, the tqp range is from %d to %d",
4165 kinfo->num_tc,
2d7187ce 4166 hns3_get_max_available_channels(h));
f1f779ce
PL		 4167 return -EINVAL;
4168 }
4169
4170 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
4171 if (kinfo->num_tqps == new_tqp_num)
4172 return 0;
4173
4174 if (if_running)
41efd6b1 4175 hns3_nic_net_stop(netdev);
f1f779ce 4176
f1f779ce
PL		 4177 ret = hns3_nic_uninit_vector_data(priv);
4178 if (ret) {
4179 dev_err(&netdev->dev,
4180 "Unbind vector with tqp fail, nothing is changed");
4181 goto open_netdev;
4182 }
4183
0378eccc 4184 hns3_store_coal(priv);
351dad5e 4185
6cbd6d33
YL		 4186 hns3_nic_dealloc_vector_data(priv);
4187
f1f779ce 4188 hns3_uninit_all_ring(priv);
a2ddee8c 4189 hns3_put_ring_config(priv);
f1f779ce
PL		 4190
4191 org_tqp_num = h->kinfo.num_tqps;
0378eccc 4192 ret = hns3_modify_tqp_num(netdev, new_tqp_num);
f1f779ce 4193 if (ret) {
0378eccc 4194 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
f1f779ce
PL		 4195 if (ret) {
4196 /* If revert to old tqp failed, fatal error occurred */
4197 dev_err(&netdev->dev,
4198 "Revert to old tqp num fail, ret=%d", ret);
4199 return ret;
4200 }
4201 dev_info(&netdev->dev,
4202 "Change tqp num fail, Revert to old tqp num");
4203 }
4204
4205open_netdev:
4206 if (if_running)
41efd6b1 4207 hns3_nic_net_open(netdev);
f1f779ce
PL		 4208
4209 return ret;
4210}
4211
1db9b1bf 4212static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 4213 .init_instance = hns3_client_init,
4214 .uninit_instance = hns3_client_uninit,
4215 .link_status_change = hns3_link_status_change,
9df8f79a 4216 .setup_tc = hns3_client_setup_tc,
bb6b94a8 4217 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 4218};
4219
4220/* hns3_init_module - Driver registration routine
4221 * hns3_init_module is the first routine called when the driver is
4222 * loaded. All it does is register with the PCI subsystem.
4223 */
4224static int __init hns3_init_module(void)
4225{
4226 int ret;
4227
4228 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
4229 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
4230
4231 client.type = HNAE3_CLIENT_KNIC;
4232 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
4233 hns3_driver_name);
4234
4235 client.ops = &client_ops;
4236
dadc9935
XW		 4237 INIT_LIST_HEAD(&client.node);
4238
a9556d8c 4239 hns3_dbg_register_debugfs(hns3_driver_name);
4240
76ad4f0e
S		 4241 ret = hnae3_register_client(&client);
4242 if (ret)
a9556d8c 4243 goto err_reg_client;
76ad4f0e
S		 4244
4245 ret = pci_register_driver(&hns3_driver);
4246 if (ret)
a9556d8c 4247 goto err_reg_driver;
76ad4f0e
S		 4248
4249 return ret;
a9556d8c 4250
4251err_reg_driver:
4252 hnae3_unregister_client(&client);
4253err_reg_client:
4254 hns3_dbg_unregister_debugfs();
4255 return ret;
76ad4f0e
S		 4256}
4257module_init(hns3_init_module);
4258
4259/* hns3_exit_module - Driver exit cleanup routine
4260 * hns3_exit_module is called just before the driver is removed
4261 * from memory.
4262 */
4263static void __exit hns3_exit_module(void)
4264{
4265 pci_unregister_driver(&hns3_driver);
4266 hnae3_unregister_client(&client);
a9556d8c 4267 hns3_dbg_unregister_debugfs();
76ad4f0e
S		 4268}
4269module_exit(hns3_exit_module);
4270
4271MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4272MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4273MODULE_LICENSE("GPL");
4274MODULE_ALIAS("pci:hns-nic");
4786ad87 4275MODULE_VERSION(HNS3_MOD_VERSION);
ubuntu-bionic-kernel mirror