]> git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blame - drivers/net/ethernet/qualcomm/emac/emac-mac.c
projects / mirror_ubuntu-eoan-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mac80211-next-for-davem-2018-03-29' of git://git.kernel.org/pub/scm/linux...
[mirror_ubuntu-eoan-kernel.git] / drivers / net / ethernet / qualcomm / emac / emac-mac.c
CommitLineData
b9b17deb
TT		 1/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 */
12
13/* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
14 */
15
16#include <linux/tcp.h>
17#include <linux/ip.h>
18#include <linux/ipv6.h>
19#include <linux/crc32.h>
20#include <linux/if_vlan.h>
21#include <linux/jiffies.h>
22#include <linux/phy.h>
23#include <linux/of.h>
24#include <net/ip6_checksum.h>
25#include "emac.h"
26#include "emac-sgmii.h"
27
b9b17deb
TT		 28/* EMAC_MAC_CTRL */
29#define SINGLE_PAUSE_MODE 0x10000000
30#define DEBUG_MODE 0x08000000
31#define BROAD_EN 0x04000000
32#define MULTI_ALL 0x02000000
33#define RX_CHKSUM_EN 0x01000000
34#define HUGE 0x00800000
35#define SPEED(x) (((x) & 0x3) << 20)
36#define SPEED_MASK SPEED(0x3)
37#define SIMR 0x00080000
38#define TPAUSE 0x00010000
39#define PROM_MODE 0x00008000
40#define VLAN_STRIP 0x00004000
41#define PRLEN_BMSK 0x00003c00
42#define PRLEN_SHFT 10
43#define HUGEN 0x00000200
44#define FLCHK 0x00000100
45#define PCRCE 0x00000080
46#define CRCE 0x00000040
47#define FULLD 0x00000020
48#define MAC_LP_EN 0x00000010
49#define RXFC 0x00000008
50#define TXFC 0x00000004
51#define RXEN 0x00000002
52#define TXEN 0x00000001
53
b9b17deb
TT		 54/* EMAC_DESC_CTRL_3 */
55#define RFD_RING_SIZE_BMSK 0xfff
56
57/* EMAC_DESC_CTRL_4 */
58#define RX_BUFFER_SIZE_BMSK 0xffff
59
60/* EMAC_DESC_CTRL_6 */
61#define RRD_RING_SIZE_BMSK 0xfff
62
63/* EMAC_DESC_CTRL_9 */
64#define TPD_RING_SIZE_BMSK 0xffff
65
66/* EMAC_TXQ_CTRL_0 */
67#define NUM_TXF_BURST_PREF_BMSK 0xffff0000
68#define NUM_TXF_BURST_PREF_SHFT 16
69#define LS_8023_SP 0x80
70#define TXQ_MODE 0x40
71#define TXQ_EN 0x20
72#define IP_OP_SP 0x10
73#define NUM_TPD_BURST_PREF_BMSK 0xf
74#define NUM_TPD_BURST_PREF_SHFT 0
75
76/* EMAC_TXQ_CTRL_1 */
77#define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff
78
79/* EMAC_TXQ_CTRL_2 */
80#define TXF_HWM_BMSK 0xfff0000
81#define TXF_LWM_BMSK 0xfff
82
83/* EMAC_RXQ_CTRL_0 */
84#define RXQ_EN BIT(31)
85#define CUT_THRU_EN BIT(30)
86#define RSS_HASH_EN BIT(29)
87#define NUM_RFD_BURST_PREF_BMSK 0x3f00000
88#define NUM_RFD_BURST_PREF_SHFT 20
89#define IDT_TABLE_SIZE_BMSK 0x1ff00
90#define IDT_TABLE_SIZE_SHFT 8
91#define SP_IPV6 0x80
92
93/* EMAC_RXQ_CTRL_1 */
94#define JUMBO_1KAH_BMSK 0xf000
95#define JUMBO_1KAH_SHFT 12
96#define RFD_PREF_LOW_TH 0x10
97#define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0
98#define RFD_PREF_LOW_THRESHOLD_SHFT 6
99#define RFD_PREF_UP_TH 0x10
100#define RFD_PREF_UP_THRESHOLD_BMSK 0x3f
101#define RFD_PREF_UP_THRESHOLD_SHFT 0
102
103/* EMAC_RXQ_CTRL_2 */
104#define RXF_DOF_THRESFHOLD 0x1a0
105#define RXF_DOF_THRESHOLD_BMSK 0xfff0000
106#define RXF_DOF_THRESHOLD_SHFT 16
107#define RXF_UOF_THRESFHOLD 0xbe
108#define RXF_UOF_THRESHOLD_BMSK 0xfff
109#define RXF_UOF_THRESHOLD_SHFT 0
110
111/* EMAC_RXQ_CTRL_3 */
112#define RXD_TIMER_BMSK 0xffff0000
113#define RXD_THRESHOLD_BMSK 0xfff
114#define RXD_THRESHOLD_SHFT 0
115
116/* EMAC_DMA_CTRL */
117#define DMAW_DLY_CNT_BMSK 0xf0000
118#define DMAW_DLY_CNT_SHFT 16
119#define DMAR_DLY_CNT_BMSK 0xf800
120#define DMAR_DLY_CNT_SHFT 11
121#define DMAR_REQ_PRI 0x400
122#define REGWRBLEN_BMSK 0x380
123#define REGWRBLEN_SHFT 7
124#define REGRDBLEN_BMSK 0x70
125#define REGRDBLEN_SHFT 4
126#define OUT_ORDER_MODE 0x4
127#define ENH_ORDER_MODE 0x2
128#define IN_ORDER_MODE 0x1
129
130/* EMAC_MAILBOX_13 */
131#define RFD3_PROC_IDX_BMSK 0xfff0000
132#define RFD3_PROC_IDX_SHFT 16
133#define RFD3_PROD_IDX_BMSK 0xfff
134#define RFD3_PROD_IDX_SHFT 0
135
136/* EMAC_MAILBOX_2 */
137#define NTPD_CONS_IDX_BMSK 0xffff0000
138#define NTPD_CONS_IDX_SHFT 16
139
140/* EMAC_MAILBOX_3 */
141#define RFD0_CONS_IDX_BMSK 0xfff
142#define RFD0_CONS_IDX_SHFT 0
143
144/* EMAC_MAILBOX_11 */
145#define H3TPD_PROD_IDX_BMSK 0xffff0000
146#define H3TPD_PROD_IDX_SHFT 16
147
148/* EMAC_AXI_MAST_CTRL */
149#define DATA_BYTE_SWAP 0x8
150#define MAX_BOUND 0x2
151#define MAX_BTYPE 0x1
152
153/* EMAC_MAILBOX_12 */
154#define H3TPD_CONS_IDX_BMSK 0xffff0000
155#define H3TPD_CONS_IDX_SHFT 16
156
157/* EMAC_MAILBOX_9 */
158#define H2TPD_PROD_IDX_BMSK 0xffff
159#define H2TPD_PROD_IDX_SHFT 0
160
161/* EMAC_MAILBOX_10 */
162#define H1TPD_CONS_IDX_BMSK 0xffff0000
163#define H1TPD_CONS_IDX_SHFT 16
164#define H2TPD_CONS_IDX_BMSK 0xffff
165#define H2TPD_CONS_IDX_SHFT 0
166
167/* EMAC_ATHR_HEADER_CTRL */
168#define HEADER_CNT_EN 0x2
169#define HEADER_ENABLE 0x1
170
171/* EMAC_MAILBOX_0 */
172#define RFD0_PROC_IDX_BMSK 0xfff0000
173#define RFD0_PROC_IDX_SHFT 16
174#define RFD0_PROD_IDX_BMSK 0xfff
175#define RFD0_PROD_IDX_SHFT 0
176
177/* EMAC_MAILBOX_5 */
178#define RFD1_PROC_IDX_BMSK 0xfff0000
179#define RFD1_PROC_IDX_SHFT 16
180#define RFD1_PROD_IDX_BMSK 0xfff
181#define RFD1_PROD_IDX_SHFT 0
182
183/* EMAC_MISC_CTRL */
184#define RX_UNCPL_INT_EN 0x1
185
186/* EMAC_MAILBOX_7 */
187#define RFD2_CONS_IDX_BMSK 0xfff0000
188#define RFD2_CONS_IDX_SHFT 16
189#define RFD1_CONS_IDX_BMSK 0xfff
190#define RFD1_CONS_IDX_SHFT 0
191
192/* EMAC_MAILBOX_8 */
193#define RFD3_CONS_IDX_BMSK 0xfff
194#define RFD3_CONS_IDX_SHFT 0
195
196/* EMAC_MAILBOX_15 */
197#define NTPD_PROD_IDX_BMSK 0xffff
198#define NTPD_PROD_IDX_SHFT 0
199
200/* EMAC_MAILBOX_16 */
201#define H1TPD_PROD_IDX_BMSK 0xffff
202#define H1TPD_PROD_IDX_SHFT 0
203
204#define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20
205#define RXQ0_RSS_HSTYP_IPV6_EN 0x10
206#define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8
207#define RXQ0_RSS_HSTYP_IPV4_EN 0x4
208
209/* EMAC_EMAC_WRAPPER_TX_TS_INX */
210#define EMAC_WRAPPER_TX_TS_EMPTY BIT(31)
211#define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff
212
213struct emac_skb_cb {
214 u32 tpd_idx;
215 unsigned long jiffies;
216};
217
218#define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)
219#define EMAC_RSS_IDT_SIZE 256
220#define JUMBO_1KAH 0x4
221#define RXD_TH 0x100
222#define EMAC_TPD_LAST_FRAGMENT 0x80000000
223#define EMAC_TPD_TSTAMP_SAVE 0x80000000
224
225/* EMAC Errors in emac_rrd.word[3] */
226#define EMAC_RRD_L4F BIT(14)
227#define EMAC_RRD_IPF BIT(15)
228#define EMAC_RRD_CRC BIT(21)
229#define EMAC_RRD_FAE BIT(22)
230#define EMAC_RRD_TRN BIT(23)
231#define EMAC_RRD_RNT BIT(24)
232#define EMAC_RRD_INC BIT(25)
233#define EMAC_RRD_FOV BIT(29)
234#define EMAC_RRD_LEN BIT(30)
235
236/* Error bits that will result in a received frame being discarded */
237#define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
238 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
239 EMAC_RRD_FOV | EMAC_RRD_LEN)
240#define EMAC_RRD_STATS_DW_IDX 3
241
242#define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
243#define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
244#define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
245
246#define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))
247#define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))
248
249#define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8
250
251#define ISR_RX_PKT (\
252 RX_PKT_INT0 |\
253 RX_PKT_INT1 |\
254 RX_PKT_INT2 |\
255 RX_PKT_INT3)
256
b9b17deb
TT		 257void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
258{
259 u32 crc32, bit, reg, mta;
260
261 /* Calculate the CRC of the MAC address */
262 crc32 = ether_crc(ETH_ALEN, addr);
263
264 /* The HASH Table is an array of 2 32-bit registers. It is
265 * treated like an array of 64 bits (BitArray[hash_value]).
266 * Use the upper 6 bits of the above CRC as the hash value.
267 */
268 reg = (crc32 >> 31) & 0x1;
269 bit = (crc32 >> 26) & 0x1F;
270
271 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
272 mta |= BIT(bit);
273 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
274}
275
276void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
277{
278 writel(0, adpt->base + EMAC_HASH_TAB_REG0);
279 writel(0, adpt->base + EMAC_HASH_TAB_REG1);
280}
281
282/* definitions for RSS */
283#define EMAC_RSS_KEY(_i, _type) \
284 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
285#define EMAC_RSS_TBL(_i, _type) \
286 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
287
288/* Config MAC modes */
289void emac_mac_mode_config(struct emac_adapter *adpt)
290{
291 struct net_device *netdev = adpt->netdev;
292 u32 mac;
293
294 mac = readl(adpt->base + EMAC_MAC_CTRL);
295 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
296
297 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
298 mac |= VLAN_STRIP;
299
300 if (netdev->flags & IFF_PROMISC)
301 mac |= PROM_MODE;
302
303 if (netdev->flags & IFF_ALLMULTI)
304 mac |= MULTI_ALL;
305
306 writel(mac, adpt->base + EMAC_MAC_CTRL);
307}
308
309/* Config descriptor rings */
310static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
311{
b9b17deb
TT		 312 /* TPD (Transmit Packet Descriptor) */
313 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
314 adpt->base + EMAC_DESC_CTRL_1);
315
316 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
3958ffcd 317 adpt->base + EMAC_DESC_CTRL_8);
b9b17deb
TT		 318
319 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
320 adpt->base + EMAC_DESC_CTRL_9);
321
322 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
323 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
324 adpt->base + EMAC_DESC_CTRL_0);
325
326 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
3958ffcd 327 adpt->base + EMAC_DESC_CTRL_2);
b9b17deb 328 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
3958ffcd 329 adpt->base + EMAC_DESC_CTRL_5);
b9b17deb
TT		 330
331 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
332 adpt->base + EMAC_DESC_CTRL_3);
333 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
334 adpt->base + EMAC_DESC_CTRL_6);
335
336 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
337 adpt->base + EMAC_DESC_CTRL_4);
338
339 writel(0, adpt->base + EMAC_DESC_CTRL_11);
340
341 /* Load all of the base addresses above and ensure that triggering HW to
342 * read ring pointers is flushed
343 */
344 writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
345}
346
347/* Config transmit parameters */
348static void emac_mac_tx_config(struct emac_adapter *adpt)
349{
350 u32 val;
351
352 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
353 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
354
355 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
356 NUM_TPD_BURST_PREF_BMSK;
357
358 val |= TXQ_MODE | LS_8023_SP;
359 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
360 NUM_TXF_BURST_PREF_BMSK;
361
362 writel(val, adpt->base + EMAC_TXQ_CTRL_0);
363 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
364 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
365}
366
367/* Config receive parameters */
368static void emac_mac_rx_config(struct emac_adapter *adpt)
369{
370 u32 val;
371
372 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
373 NUM_RFD_BURST_PREF_BMSK;
374 val |= (SP_IPV6 | CUT_THRU_EN);
375
376 writel(val, adpt->base + EMAC_RXQ_CTRL_0);
377
378 val = readl(adpt->base + EMAC_RXQ_CTRL_1);
379 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
380 RFD_PREF_UP_THRESHOLD_BMSK);
381 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
382 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
383 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT);
384 writel(val, adpt->base + EMAC_RXQ_CTRL_1);
385
386 val = readl(adpt->base + EMAC_RXQ_CTRL_2);
387 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
388 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) |
389 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
390 writel(val, adpt->base + EMAC_RXQ_CTRL_2);
391
392 val = readl(adpt->base + EMAC_RXQ_CTRL_3);
393 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
394 val |= RXD_TH << RXD_THRESHOLD_SHFT;
395 writel(val, adpt->base + EMAC_RXQ_CTRL_3);
396}
397
398/* Config dma */
399static void emac_mac_dma_config(struct emac_adapter *adpt)
400{
401 u32 dma_ctrl = DMAR_REQ_PRI;
402
403 switch (adpt->dma_order) {
404 case emac_dma_ord_in:
405 dma_ctrl |= IN_ORDER_MODE;
406 break;
407 case emac_dma_ord_enh:
408 dma_ctrl |= ENH_ORDER_MODE;
409 break;
410 case emac_dma_ord_out:
411 dma_ctrl |= OUT_ORDER_MODE;
412 break;
413 default:
414 break;
415 }
416
417 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
418 REGRDBLEN_BMSK;
419 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
420 REGWRBLEN_BMSK;
421 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
422 DMAR_DLY_CNT_BMSK;
423 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
424 DMAW_DLY_CNT_BMSK;
425
426 /* config DMA and ensure that configuration is flushed to HW */
427 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
428}
429
430/* set MAC address */
431static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
432{
433 u32 sta;
434
435 /* for example: 00-A0-C6-11-22-33
436 * 0<-->C6112233, 1<-->00A0.
437 */
438
439 /* low 32bit word */
440 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
441 (((u32)addr[4]) << 8) | (((u32)addr[5]));
442 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
443
444 /* hight 32bit word */
445 sta = (((u32)addr[0]) << 8) | (u32)addr[1];
446 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
447}
448
449static void emac_mac_config(struct emac_adapter *adpt)
450{
451 struct net_device *netdev = adpt->netdev;
452 unsigned int max_frame;
453 u32 val;
454
455 emac_set_mac_address(adpt, netdev->dev_addr);
456
457 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
458 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
459 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
460
461 emac_mac_dma_rings_config(adpt);
462
463 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
464 adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
465
466 emac_mac_tx_config(adpt);
467 emac_mac_rx_config(adpt);
468 emac_mac_dma_config(adpt);
469
470 val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
471 val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
472 val |= MAX_BTYPE;
473 writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
474 writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
475 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
476}
477
478void emac_mac_reset(struct emac_adapter *adpt)
479{
480 emac_mac_stop(adpt);
481
482 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
483 usleep_range(100, 150); /* reset may take up to 100usec */
484
485 /* interrupt clear-on-read */
486 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
487}
488
0f20276d 489static void emac_mac_start(struct emac_adapter *adpt)
b9b17deb
TT		 490{
491 struct phy_device *phydev = adpt->phydev;
492 u32 mac, csr1;
493
494 /* enable tx queue */
495 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
496
497 /* enable rx queue */
498 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
499
500 /* enable mac control */
501 mac = readl(adpt->base + EMAC_MAC_CTRL);
502 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
503
504 mac |= TXEN | RXEN; /* enable RX/TX */
505
b44700e9
TT		 506 /* Configure MAC flow control. If set to automatic, then match
507 * whatever the PHY does. Otherwise, enable or disable it, depending
508 * on what the user configured via ethtool.
509 */
510 mac &= ~(RXFC | TXFC);
511
512 if (adpt->automatic) {
513 /* If it's set to automatic, then update our local values */
514 adpt->rx_flow_control = phydev->pause;
515 adpt->tx_flow_control = phydev->pause != phydev->asym_pause;
516 }
517 mac |= adpt->rx_flow_control ? RXFC : 0;
518 mac |= adpt->tx_flow_control ? TXFC : 0;
b9b17deb
TT		 519
520 /* setup link speed */
521 mac &= ~SPEED_MASK;
522 if (phydev->speed == SPEED_1000) {
523 mac |= SPEED(2);
524 csr1 |= FREQ_MODE;
525 } else {
526 mac |= SPEED(1);
527 csr1 &= ~FREQ_MODE;
528 }
529
530 if (phydev->duplex == DUPLEX_FULL)
531 mac |= FULLD;
532 else
533 mac &= ~FULLD;
534
535 /* other parameters */
536 mac |= (CRCE | PCRCE);
537 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
538 mac |= BROAD_EN;
539 mac |= FLCHK;
540 mac &= ~RX_CHKSUM_EN;
541 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
542 DEBUG_MODE | SINGLE_PAUSE_MODE);
543
4a7a3860
TT		 544 /* Enable single-pause-frame mode if requested.
545 *
546 * If enabled, the EMAC will send a single pause frame when the RX
547 * queue is full. This normally leads to packet loss because
548 * the pause frame disables the remote MAC only for 33ms (the quanta),
549 * and then the remote MAC continues sending packets even though
550 * the RX queue is still full.
551 *
552 * If disabled, the EMAC sends a pause frame every 31ms until the RX
553 * queue is no longer full. Normally, this is the preferred
554 * method of operation. However, when the system is hung (e.g.
555 * cores are halted), the EMAC interrupt handler is never called
556 * and so the RX queue fills up quickly and stays full. The resuling
557 * non-stop "flood" of pause frames sometimes has the effect of
558 * disabling nearby switches. In some cases, other nearby switches
559 * are also affected, shutting down the entire network.
560 *
561 * The user can enable or disable single-pause-frame mode
562 * via ethtool.
563 */
564 mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0;
565
b9b17deb
TT		 566 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
567
568 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
569
570 /* enable interrupt read clear, low power sleep mode and
571 * the irq moderators
572 */
573
574 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
575 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
576 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
577
578 emac_mac_mode_config(adpt);
579
580 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
581 (HEADER_ENABLE | HEADER_CNT_EN), 0);
b9b17deb
TT		 582}
583
584void emac_mac_stop(struct emac_adapter *adpt)
585{
586 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
587 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
588 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
589 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
590}
591
592/* Free all descriptors of given transmit queue */
593static void emac_tx_q_descs_free(struct emac_adapter *adpt)
594{
595 struct emac_tx_queue *tx_q = &adpt->tx_q;
596 unsigned int i;
597 size_t size;
598
599 /* ring already cleared, nothing to do */
600 if (!tx_q->tpd.tpbuff)
601 return;
602
603 for (i = 0; i < tx_q->tpd.count; i++) {
604 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
605
606 if (tpbuf->dma_addr) {
607 dma_unmap_single(adpt->netdev->dev.parent,
608 tpbuf->dma_addr, tpbuf->length,
609 DMA_TO_DEVICE);
610 tpbuf->dma_addr = 0;
611 }
612 if (tpbuf->skb) {
613 dev_kfree_skb_any(tpbuf->skb);
614 tpbuf->skb = NULL;
615 }
616 }
617
618 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
619 memset(tx_q->tpd.tpbuff, 0, size);
620
621 /* clear the descriptor ring */
622 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
623
624 tx_q->tpd.consume_idx = 0;
625 tx_q->tpd.produce_idx = 0;
626}
627
628/* Free all descriptors of given receive queue */
629static void emac_rx_q_free_descs(struct emac_adapter *adpt)
630{
631 struct device *dev = adpt->netdev->dev.parent;
632 struct emac_rx_queue *rx_q = &adpt->rx_q;
633 unsigned int i;
634 size_t size;
635
636 /* ring already cleared, nothing to do */
637 if (!rx_q->rfd.rfbuff)
638 return;
639
640 for (i = 0; i < rx_q->rfd.count; i++) {
641 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
642
643 if (rfbuf->dma_addr) {
644 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
645 DMA_FROM_DEVICE);
646 rfbuf->dma_addr = 0;
647 }
648 if (rfbuf->skb) {
649 dev_kfree_skb(rfbuf->skb);
650 rfbuf->skb = NULL;
651 }
652 }
653
654 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
655 memset(rx_q->rfd.rfbuff, 0, size);
656
657 /* clear the descriptor rings */
658 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
659 rx_q->rrd.produce_idx = 0;
660 rx_q->rrd.consume_idx = 0;
661
662 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
663 rx_q->rfd.produce_idx = 0;
664 rx_q->rfd.consume_idx = 0;
665}
666
667/* Free all buffers associated with given transmit queue */
668static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
669{
670 struct emac_tx_queue *tx_q = &adpt->tx_q;
671
672 emac_tx_q_descs_free(adpt);
673
674 kfree(tx_q->tpd.tpbuff);
675 tx_q->tpd.tpbuff = NULL;
676 tx_q->tpd.v_addr = NULL;
677 tx_q->tpd.dma_addr = 0;
678 tx_q->tpd.size = 0;
679}
680
681/* Allocate TX descriptor ring for the given transmit queue */
682static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
683 struct emac_tx_queue *tx_q)
684{
685 struct emac_ring_header *ring_header = &adpt->ring_header;
686 size_t size;
687
688 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
689 tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
690 if (!tx_q->tpd.tpbuff)
691 return -ENOMEM;
692
693 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
694 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
695 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
696 ring_header->used += ALIGN(tx_q->tpd.size, 8);
697 tx_q->tpd.produce_idx = 0;
698 tx_q->tpd.consume_idx = 0;
699
700 return 0;
701}
702
703/* Free all buffers associated with given transmit queue */
704static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
705{
706 struct emac_rx_queue *rx_q = &adpt->rx_q;
707
708 emac_rx_q_free_descs(adpt);
709
710 kfree(rx_q->rfd.rfbuff);
711 rx_q->rfd.rfbuff = NULL;
712
713 rx_q->rfd.v_addr = NULL;
714 rx_q->rfd.dma_addr = 0;
715 rx_q->rfd.size = 0;
716
717 rx_q->rrd.v_addr = NULL;
718 rx_q->rrd.dma_addr = 0;
719 rx_q->rrd.size = 0;
720}
721
722/* Allocate RX descriptor rings for the given receive queue */
723static int emac_rx_descs_alloc(struct emac_adapter *adpt)
724{
725 struct emac_ring_header *ring_header = &adpt->ring_header;
726 struct emac_rx_queue *rx_q = &adpt->rx_q;
727 size_t size;
728
729 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
730 rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
731 if (!rx_q->rfd.rfbuff)
732 return -ENOMEM;
733
734 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
735 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
736
737 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
738 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used;
739 ring_header->used += ALIGN(rx_q->rrd.size, 8);
740
741 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
742 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used;
743 ring_header->used += ALIGN(rx_q->rfd.size, 8);
744
745 rx_q->rrd.produce_idx = 0;
746 rx_q->rrd.consume_idx = 0;
747
748 rx_q->rfd.produce_idx = 0;
749 rx_q->rfd.consume_idx = 0;
750
751 return 0;
752}
753
754/* Allocate all TX and RX descriptor rings */
755int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
756{
757 struct emac_ring_header *ring_header = &adpt->ring_header;
758 struct device *dev = adpt->netdev->dev.parent;
759 unsigned int num_tx_descs = adpt->tx_desc_cnt;
760 unsigned int num_rx_descs = adpt->rx_desc_cnt;
761 int ret;
762
763 adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
764
765 adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
766 adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
767
768 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
769 * hence the additional padding bytes are allocated.
770 */
3f7832c2
TT		 771 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
772 num_rx_descs * (adpt->rfd_size * 4) +
773 num_rx_descs * (adpt->rrd_size * 4) +
774 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
b9b17deb
TT		 775
776 ring_header->used = 0;
777 ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
778 &ring_header->dma_addr,
779 GFP_KERNEL);
780 if (!ring_header->v_addr)
781 return -ENOMEM;
782
3f7832c2
TT		 783 ring_header->used = ALIGN(ring_header->dma_addr, 8) -
784 ring_header->dma_addr;
b9b17deb 785
df1ec1b9
TT		 786 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
787 if (ret) {
3f7832c2 788 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
df1ec1b9
TT		 789 goto err_alloc_tx;
790 }
791
3f7832c2
TT		 792 ret = emac_rx_descs_alloc(adpt);
793 if (ret) {
794 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
795 goto err_alloc_rx;
796 }
797
b9b17deb
TT		 798 return 0;
799
df1ec1b9 800err_alloc_rx:
3f7832c2
TT		 801 emac_tx_q_bufs_free(adpt);
802err_alloc_tx:
b9b17deb
TT		 803 dma_free_coherent(dev, ring_header->size,
804 ring_header->v_addr, ring_header->dma_addr);
805
806 ring_header->v_addr = NULL;
807 ring_header->dma_addr = 0;
808 ring_header->size = 0;
809 ring_header->used = 0;
810
811 return ret;
812}
813
814/* Free all TX and RX descriptor rings */
815void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
816{
817 struct emac_ring_header *ring_header = &adpt->ring_header;
818 struct device *dev = adpt->netdev->dev.parent;
819
820 emac_tx_q_bufs_free(adpt);
821 emac_rx_q_bufs_free(adpt);
822
823 dma_free_coherent(dev, ring_header->size,
824 ring_header->v_addr, ring_header->dma_addr);
825
826 ring_header->v_addr = NULL;
827 ring_header->dma_addr = 0;
828 ring_header->size = 0;
829 ring_header->used = 0;
830}
831
832/* Initialize descriptor rings */
833static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
834{
835 unsigned int i;
836
837 adpt->tx_q.tpd.produce_idx = 0;
838 adpt->tx_q.tpd.consume_idx = 0;
839 for (i = 0; i < adpt->tx_q.tpd.count; i++)
840 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
841
842 adpt->rx_q.rrd.produce_idx = 0;
843 adpt->rx_q.rrd.consume_idx = 0;
844 adpt->rx_q.rfd.produce_idx = 0;
845 adpt->rx_q.rfd.consume_idx = 0;
846 for (i = 0; i < adpt->rx_q.rfd.count; i++)
847 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
848}
849
850/* Produce new receive free descriptor */
851static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
852 struct emac_rx_queue *rx_q,
853 dma_addr_t addr)
854{
855 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
856
857 *(hw_rfd++) = lower_32_bits(addr);
858 *hw_rfd = upper_32_bits(addr);
859
860 if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
861 rx_q->rfd.produce_idx = 0;
862}
863
864/* Fill up receive queue's RFD with preallocated receive buffers */
865static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
866 struct emac_rx_queue *rx_q)
867{
868 struct emac_buffer *curr_rxbuf;
869 struct emac_buffer *next_rxbuf;
870 unsigned int count = 0;
871 u32 next_produce_idx;
872
873 next_produce_idx = rx_q->rfd.produce_idx + 1;
874 if (next_produce_idx == rx_q->rfd.count)
875 next_produce_idx = 0;
876
877 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
878 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
879
880 /* this always has a blank rx_buffer*/
881 while (!next_rxbuf->dma_addr) {
882 struct sk_buff *skb;
883 int ret;
884
885 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
886 if (!skb)
887 break;
888
889 curr_rxbuf->dma_addr =
890 dma_map_single(adpt->netdev->dev.parent, skb->data,
a93ad944
TT		 891 adpt->rxbuf_size, DMA_FROM_DEVICE);
892
b9b17deb
TT		 893 ret = dma_mapping_error(adpt->netdev->dev.parent,
894 curr_rxbuf->dma_addr);
895 if (ret) {
896 dev_kfree_skb(skb);
897 break;
898 }
899 curr_rxbuf->skb = skb;
900 curr_rxbuf->length = adpt->rxbuf_size;
901
902 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
903 next_produce_idx = rx_q->rfd.produce_idx + 1;
904 if (next_produce_idx == rx_q->rfd.count)
905 next_produce_idx = 0;
906
907 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
908 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
909 count++;
910 }
911
912 if (count) {
913 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
914 rx_q->produce_mask;
915 emac_reg_update32(adpt->base + rx_q->produce_reg,
916 rx_q->produce_mask, prod_idx);
917 }
918}
919
920static void emac_adjust_link(struct net_device *netdev)
921{
922 struct emac_adapter *adpt = netdev_priv(netdev);
fd0e97b8 923 struct emac_sgmii *sgmii = &adpt->phy;
b9b17deb
TT		 924 struct phy_device *phydev = netdev->phydev;
925
fd0e97b8 926 if (phydev->link) {
b9b17deb 927 emac_mac_start(adpt);
fd0e97b8
TT		 928 sgmii->link_up(adpt);
929 } else {
930 sgmii->link_down(adpt);
b9b17deb 931 emac_mac_stop(adpt);
fd0e97b8 932 }
b9b17deb
TT		 933
934 phy_print_status(phydev);
935}
936
937/* Bringup the interface/HW */
938int emac_mac_up(struct emac_adapter *adpt)
939{
940 struct net_device *netdev = adpt->netdev;
b9b17deb
TT		 941 int ret;
942
943 emac_mac_rx_tx_ring_reset_all(adpt);
944 emac_mac_config(adpt);
b9b17deb
TT		 945 emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
946
24609669 947 adpt->phydev->irq = PHY_POLL;
b9b17deb
TT		 948 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
949 PHY_INTERFACE_MODE_SGMII);
950 if (ret) {
951 netdev_err(adpt->netdev, "could not connect phy\n");
b9b17deb
TT		 952 return ret;
953 }
954
9da34f27
TT		 955 phy_attached_print(adpt->phydev, NULL);
956
b9b17deb
TT		 957 /* enable mac irq */
958 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
959 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
960
b9b17deb
TT		 961 phy_start(adpt->phydev);
962
963 napi_enable(&adpt->rx_q.napi);
964 netif_start_queue(netdev);
965
966 return 0;
967}
968
969/* Bring down the interface/HW */
970void emac_mac_down(struct emac_adapter *adpt)
971{
972 struct net_device *netdev = adpt->netdev;
973
974 netif_stop_queue(netdev);
975 napi_disable(&adpt->rx_q.napi);
976
977 phy_stop(adpt->phydev);
b9b17deb 978
93966b71
TT		 979 /* Interrupts must be disabled before the PHY is disconnected, to
980 * avoid a race condition where adjust_link is null when we get
981 * an interrupt.
982 */
b9b17deb
TT		 983 writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
984 writel(0, adpt->base + EMAC_INT_MASK);
985 synchronize_irq(adpt->irq.irq);
b9b17deb 986
93966b71
TT		 987 phy_disconnect(adpt->phydev);
988
b9b17deb
TT		 989 emac_mac_reset(adpt);
990
991 emac_tx_q_descs_free(adpt);
992 netdev_reset_queue(adpt->netdev);
993 emac_rx_q_free_descs(adpt);
994}
995
996/* Consume next received packet descriptor */
997static bool emac_rx_process_rrd(struct emac_adapter *adpt,
998 struct emac_rx_queue *rx_q,
999 struct emac_rrd *rrd)
1000{
1001 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
1002
1003 rrd->word[3] = *(hw_rrd + 3);
1004
1005 if (!RRD_UPDT(rrd))
1006 return false;
1007
1008 rrd->word[4] = 0;
1009 rrd->word[5] = 0;
1010
1011 rrd->word[0] = *(hw_rrd++);
1012 rrd->word[1] = *(hw_rrd++);
1013 rrd->word[2] = *(hw_rrd++);
1014
1015 if (unlikely(RRD_NOR(rrd) != 1)) {
1016 netdev_err(adpt->netdev,
1017 "error: multi-RFD not support yet! nor:%lu\n",
1018 RRD_NOR(rrd));
1019 }
1020
1021 /* mark rrd as processed */
1022 RRD_UPDT_SET(rrd, 0);
1023 *hw_rrd = rrd->word[3];
1024
1025 if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1026 rx_q->rrd.consume_idx = 0;
1027
1028 return true;
1029}
1030
1031/* Produce new transmit descriptor */
1032static void emac_tx_tpd_create(struct emac_adapter *adpt,
1033 struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1034{
1035 u32 *hw_tpd;
1036
1037 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1038 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1039
1040 if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1041 tx_q->tpd.produce_idx = 0;
1042
1043 *(hw_tpd++) = tpd->word[0];
1044 *(hw_tpd++) = tpd->word[1];
1045 *(hw_tpd++) = tpd->word[2];
1046 *hw_tpd = tpd->word[3];
1047}
1048
1049/* Mark the last transmit descriptor as such (for the transmit packet) */
1050static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1051 struct emac_tx_queue *tx_q)
1052{
1053 u32 *hw_tpd =
1054 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1055 u32 tmp_tpd;
1056
1057 tmp_tpd = *(hw_tpd + 1);
1058 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1059 *(hw_tpd + 1) = tmp_tpd;
1060}
1061
1062static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1063{
1064 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1065 u32 consume_idx = RRD_SI(rrd);
1066 unsigned int i;
1067
1068 for (i = 0; i < RRD_NOR(rrd); i++) {
1069 rfbuf[consume_idx].skb = NULL;
1070 if (++consume_idx == rx_q->rfd.count)
1071 consume_idx = 0;
1072 }
1073
1074 rx_q->rfd.consume_idx = consume_idx;
1075 rx_q->rfd.process_idx = consume_idx;
1076}
1077
1078/* Push the received skb to upper layers */
1079static void emac_receive_skb(struct emac_rx_queue *rx_q,
1080 struct sk_buff *skb,
1081 u16 vlan_tag, bool vlan_flag)
1082{
1083 if (vlan_flag) {
1084 u16 vlan;
1085
1086 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1087 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1088 }
1089
1090 napi_gro_receive(&rx_q->napi, skb);
1091}
1092
1093/* Process receive event */
1094void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1095 int *num_pkts, int max_pkts)
1096{
1097 u32 proc_idx, hw_consume_idx, num_consume_pkts;
1098 struct net_device *netdev = adpt->netdev;
1099 struct emac_buffer *rfbuf;
1100 unsigned int count = 0;
1101 struct emac_rrd rrd;
1102 struct sk_buff *skb;
1103 u32 reg;
1104
1105 reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1106
1107 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1108 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1109 (hw_consume_idx - rx_q->rrd.consume_idx) :
1110 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1111
1112 do {
1113 if (!num_consume_pkts)
1114 break;
1115
1116 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1117 break;
1118
1119 if (likely(RRD_NOR(&rrd) == 1)) {
1120 /* good receive */
1121 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1122 dma_unmap_single(adpt->netdev->dev.parent,
1123 rfbuf->dma_addr, rfbuf->length,
1124 DMA_FROM_DEVICE);
1125 rfbuf->dma_addr = 0;
1126 skb = rfbuf->skb;
1127 } else {
1128 netdev_err(adpt->netdev,
1129 "error: multi-RFD not support yet!\n");
1130 break;
1131 }
1132 emac_rx_rfd_clean(rx_q, &rrd);
1133 num_consume_pkts--;
1134 count++;
1135
1136 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1137 * with DF set are marked as error), drop packets based on the
1138 * error mask rather than the summary bit (ignoring L4F errors)
1139 */
1140 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1141 netif_dbg(adpt, rx_status, adpt->netdev,
1142 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1143 rrd.word[0], rrd.word[1],
1144 rrd.word[2], rrd.word[3]);
1145
1146 dev_kfree_skb(skb);
1147 continue;
1148 }
1149
1150 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1151 skb->dev = netdev;
1152 skb->protocol = eth_type_trans(skb, skb->dev);
1153 if (netdev->features & NETIF_F_RXCSUM)
1154 skb->ip_summed = RRD_L4F(&rrd) ?
1155 CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1156 else
1157 skb_checksum_none_assert(skb);
1158
1159 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1160 (bool)RRD_CVTAG(&rrd));
1161
b9b17deb
TT		 1162 (*num_pkts)++;
1163 } while (*num_pkts < max_pkts);
1164
1165 if (count) {
1166 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1167 rx_q->process_mask;
1168 emac_reg_update32(adpt->base + rx_q->process_reg,
1169 rx_q->process_mask, proc_idx);
1170 emac_mac_rx_descs_refill(adpt, rx_q);
1171 }
1172}
1173
1174/* get the number of free transmit descriptors */
1175static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1176{
1177 u32 produce_idx = tx_q->tpd.produce_idx;
1178 u32 consume_idx = tx_q->tpd.consume_idx;
1179
1180 return (consume_idx > produce_idx) ?
1181 (consume_idx - produce_idx - 1) :
1182 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1183}
1184
1185/* Process transmit event */
1186void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1187{
1188 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1189 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1190 struct emac_buffer *tpbuf;
1191
1192 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1193
1194 while (tx_q->tpd.consume_idx != hw_consume_idx) {
1195 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1196 if (tpbuf->dma_addr) {
cc5db315
HP		 1197 dma_unmap_page(adpt->netdev->dev.parent,
1198 tpbuf->dma_addr, tpbuf->length,
1199 DMA_TO_DEVICE);
b9b17deb
TT		 1200 tpbuf->dma_addr = 0;
1201 }
1202
1203 if (tpbuf->skb) {
1204 pkts_compl++;
1205 bytes_compl += tpbuf->skb->len;
1206 dev_kfree_skb_irq(tpbuf->skb);
1207 tpbuf->skb = NULL;
1208 }
1209
1210 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1211 tx_q->tpd.consume_idx = 0;
1212 }
1213
1214 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1215
1216 if (netif_queue_stopped(adpt->netdev))
1217 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1218 netif_wake_queue(adpt->netdev);
1219}
1220
1221/* Initialize all queue data structures */
1222void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1223 struct emac_adapter *adpt)
1224{
1225 adpt->rx_q.netdev = adpt->netdev;
1226
1227 adpt->rx_q.produce_reg = EMAC_MAILBOX_0;
1228 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1229 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1230
1231 adpt->rx_q.process_reg = EMAC_MAILBOX_0;
1232 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1233 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1234
1235 adpt->rx_q.consume_reg = EMAC_MAILBOX_3;
1236 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1237 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1238
1239 adpt->rx_q.irq = &adpt->irq;
1240 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT;
1241
1242 adpt->tx_q.produce_reg = EMAC_MAILBOX_15;
1243 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1244 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1245
1246 adpt->tx_q.consume_reg = EMAC_MAILBOX_2;
1247 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1248 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1249}
1250
1251/* Fill up transmit descriptors with TSO and Checksum offload information */
1252static int emac_tso_csum(struct emac_adapter *adpt,
1253 struct emac_tx_queue *tx_q,
1254 struct sk_buff *skb,
1255 struct emac_tpd *tpd)
1256{
1257 unsigned int hdr_len;
1258 int ret;
1259
1260 if (skb_is_gso(skb)) {
1261 if (skb_header_cloned(skb)) {
1262 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1263 if (unlikely(ret))
1264 return ret;
1265 }
1266
1267 if (skb->protocol == htons(ETH_P_IP)) {
1268 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1269 + ntohs(ip_hdr(skb)->tot_len);
1270 if (skb->len > pkt_len)
1271 pskb_trim(skb, pkt_len);
1272 }
1273
1274 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1275 if (unlikely(skb->len == hdr_len)) {
1276 /* we only need to do csum */
1277 netif_warn(adpt, tx_err, adpt->netdev,
1278 "tso not needed for packet with 0 data\n");
1279 goto do_csum;
1280 }
1281
1282 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1283 ip_hdr(skb)->check = 0;
1284 tcp_hdr(skb)->check =
1285 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1286 ip_hdr(skb)->daddr,
1287 0, IPPROTO_TCP, 0);
1288 TPD_IPV4_SET(tpd, 1);
1289 }
1290
1291 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1292 /* ipv6 tso need an extra tpd */
1293 struct emac_tpd extra_tpd;
1294
1295 memset(tpd, 0, sizeof(*tpd));
1296 memset(&extra_tpd, 0, sizeof(extra_tpd));
1297
1298 ipv6_hdr(skb)->payload_len = 0;
1299 tcp_hdr(skb)->check =
1300 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1301 &ipv6_hdr(skb)->daddr,
1302 0, IPPROTO_TCP, 0);
1303 TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1304 TPD_LSO_SET(&extra_tpd, 1);
1305 TPD_LSOV_SET(&extra_tpd, 1);
1306 emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1307 TPD_LSOV_SET(tpd, 1);
1308 }
1309
1310 TPD_LSO_SET(tpd, 1);
1311 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1312 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1313 return 0;
1314 }
1315
1316do_csum:
1317 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1318 unsigned int css, cso;
1319
1320 cso = skb_transport_offset(skb);
1321 if (unlikely(cso & 0x1)) {
1322 netdev_err(adpt->netdev,
1323 "error: payload offset should be even\n");
1324 return -EINVAL;
1325 }
1326 css = cso + skb->csum_offset;
1327
1328 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1329 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1330 TPD_CSX_SET(tpd, 1);
1331 }
1332
1333 return 0;
1334}
1335
1336/* Fill up transmit descriptors */
1337static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1338 struct emac_tx_queue *tx_q, struct sk_buff *skb,
1339 struct emac_tpd *tpd)
1340{
1341 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1342 unsigned int first = tx_q->tpd.produce_idx;
1343 unsigned int len = skb_headlen(skb);
1344 struct emac_buffer *tpbuf = NULL;
1345 unsigned int mapped_len = 0;
1346 unsigned int i;
1347 int count = 0;
1348 int ret;
1349
1350 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1351 if (TPD_LSO(tpd)) {
1352 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1353
1354 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1355 tpbuf->length = mapped_len;
cc5db315
HP		 1356 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1357 virt_to_page(skb->data),
1358 offset_in_page(skb->data),
1359 tpbuf->length,
1360 DMA_TO_DEVICE);
b9b17deb
TT		 1361 ret = dma_mapping_error(adpt->netdev->dev.parent,
1362 tpbuf->dma_addr);
1363 if (ret)
1364 goto error;
1365
1366 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1367 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1368 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1369 emac_tx_tpd_create(adpt, tx_q, tpd);
1370 count++;
1371 }
1372
1373 if (mapped_len < len) {
1374 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1375 tpbuf->length = len - mapped_len;
cc5db315
HP		 1376 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1377 virt_to_page(skb->data +
1378 mapped_len),
1379 offset_in_page(skb->data +
1380 mapped_len),
1381 tpbuf->length, DMA_TO_DEVICE);
b9b17deb
TT		 1382 ret = dma_mapping_error(adpt->netdev->dev.parent,
1383 tpbuf->dma_addr);
1384 if (ret)
1385 goto error;
1386
1387 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1388 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1389 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1390 emac_tx_tpd_create(adpt, tx_q, tpd);
1391 count++;
1392 }
1393
1394 for (i = 0; i < nr_frags; i++) {
1395 struct skb_frag_struct *frag;
1396
1397 frag = &skb_shinfo(skb)->frags[i];
1398
1399 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1400 tpbuf->length = frag->size;
1401 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1402 frag->page.p, frag->page_offset,
1403 tpbuf->length, DMA_TO_DEVICE);
1404 ret = dma_mapping_error(adpt->netdev->dev.parent,
1405 tpbuf->dma_addr);
1406 if (ret)
1407 goto error;
1408
1409 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1410 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1411 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1412 emac_tx_tpd_create(adpt, tx_q, tpd);
1413 count++;
1414 }
1415
1416 /* The last tpd */
1417 wmb();
1418 emac_tx_tpd_mark_last(adpt, tx_q);
1419
1420 /* The last buffer info contain the skb address,
1421 * so it will be freed after unmap
1422 */
1423 tpbuf->skb = skb;
1424
1425 return;
1426
1427error:
1428 /* One of the memory mappings failed, so undo everything */
1429 tx_q->tpd.produce_idx = first;
1430
1431 while (count--) {
1432 tpbuf = GET_TPD_BUFFER(tx_q, first);
1433 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1434 tpbuf->length, DMA_TO_DEVICE);
1435 tpbuf->dma_addr = 0;
1436 tpbuf->length = 0;
1437
1438 if (++first == tx_q->tpd.count)
1439 first = 0;
1440 }
1441
1442 dev_kfree_skb(skb);
1443}
1444
1445/* Transmit the packet using specified transmit queue */
1446int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1447 struct sk_buff *skb)
1448{
1449 struct emac_tpd tpd;
1450 u32 prod_idx;
1451
1452 memset(&tpd, 0, sizeof(tpd));
1453
1454 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1455 dev_kfree_skb_any(skb);
1456 return NETDEV_TX_OK;
1457 }
1458
1459 if (skb_vlan_tag_present(skb)) {
1460 u16 tag;
1461
1462 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1463 TPD_CVLAN_TAG_SET(&tpd, tag);
1464 TPD_INSTC_SET(&tpd, 1);
1465 }
1466
1467 if (skb_network_offset(skb) != ETH_HLEN)
1468 TPD_TYP_SET(&tpd, 1);
1469
1470 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1471
1472 netdev_sent_queue(adpt->netdev, skb->len);
1473
1474 /* Make sure the are enough free descriptors to hold one
1475 * maximum-sized SKB. We need one desc for each fragment,
1476 * one for the checksum (emac_tso_csum), one for TSO, and
1477 * and one for the SKB header.
1478 */
1479 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1480 netif_stop_queue(adpt->netdev);
1481
1482 /* update produce idx */
1483 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1484 tx_q->produce_mask;
1485 emac_reg_update32(adpt->base + tx_q->produce_reg,
1486 tx_q->produce_mask, prod_idx);
1487
1488 return NETDEV_TX_OK;
1489}
Ubuntu Eoan Linux kernel mirror