]> git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blame - drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
projects / mirror_ubuntu-focal-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sh-pfc-for-v5.1-tag2' of git://git.kernel.org/pub/scm/linux/kernel/git...
[mirror_ubuntu-focal-kernel.git] / drivers / net / ethernet / marvell / mvpp2 / mvpp2_main.c
CommitLineData
f1e37e31 1// SPDX-License-Identifier: GPL-2.0
3f518509
MW		 2/*
3 * Driver for Marvell PPv2 network controller for Armada 375 SoC.
4 *
5 * Copyright (C) 2014 Marvell
6 *
7 * Marcin Wojtas <mw@semihalf.com>
3f518509
MW		 8 */
9
a75edc7c 10#include <linux/acpi.h>
3f518509
MW		 11#include <linux/kernel.h>
12#include <linux/netdevice.h>
13#include <linux/etherdevice.h>
14#include <linux/platform_device.h>
15#include <linux/skbuff.h>
16#include <linux/inetdevice.h>
17#include <linux/mbus.h>
18#include <linux/module.h>
f84bf386 19#include <linux/mfd/syscon.h>
3f518509
MW		 20#include <linux/interrupt.h>
21#include <linux/cpumask.h>
22#include <linux/of.h>
23#include <linux/of_irq.h>
24#include <linux/of_mdio.h>
25#include <linux/of_net.h>
26#include <linux/of_address.h>
faca9247 27#include <linux/of_device.h>
3f518509 28#include <linux/phy.h>
4bb04326 29#include <linux/phylink.h>
542897d9 30#include <linux/phy/phy.h>
3f518509 31#include <linux/clk.h>
edc660fa
MW		 32#include <linux/hrtimer.h>
33#include <linux/ktime.h>
f84bf386 34#include <linux/regmap.h>
3f518509
MW		 35#include <uapi/linux/ppp_defs.h>
36#include <net/ip.h>
37#include <net/ipv6.h>
186cd4d4 38#include <net/tso.h>
3f518509 39
db9d7d36
MC		 40#include "mvpp2.h"
41#include "mvpp2_prs.h"
42#include "mvpp2_cls.h"
a786841d 43
01d04936
SC		 44enum mvpp2_bm_pool_log_num {
45 MVPP2_BM_SHORT,
46 MVPP2_BM_LONG,
576193f2 47 MVPP2_BM_JUMBO,
01d04936 48 MVPP2_BM_POOLS_NUM
3f518509
MW		 49};
50
db9d7d36
MC		 51static struct {
52 int pkt_size;
53 int buf_num;
54} mvpp2_pools[MVPP2_BM_POOLS_NUM];
3f518509 55
db9d7d36
MC		 56/* The prototype is added here to be used in start_dev when using ACPI. This
57 * will be removed once phylink is used for all modes (dt+ACPI).
58 */
59static void mvpp2_mac_config(struct net_device *dev, unsigned int mode,
60 const struct phylink_link_state *state);
41948ccb
AT		 61static void mvpp2_mac_link_up(struct net_device *dev, unsigned int mode,
62 phy_interface_t interface, struct phy_device *phy);
10fea26c 63
db9d7d36
MC		 64/* Queue modes */
65#define MVPP2_QDIST_SINGLE_MODE 0
66#define MVPP2_QDIST_MULTI_MODE 1
3f518509 67
3f6aaf72 68static int queue_mode = MVPP2_QDIST_MULTI_MODE;
3f518509 69
db9d7d36
MC		 70module_param(queue_mode, int, 0444);
71MODULE_PARM_DESC(queue_mode, "Set queue_mode (single=0, multi=1)");
3f518509 72
db9d7d36 73/* Utility/helper methods */
3f518509 74
db9d7d36
MC		 75void mvpp2_write(struct mvpp2 *priv, u32 offset, u32 data)
76{
77 writel(data, priv->swth_base[0] + offset);
3f518509
MW		 78}
79
db9d7d36 80u32 mvpp2_read(struct mvpp2 *priv, u32 offset)
3f518509 81{
db9d7d36 82 return readl(priv->swth_base[0] + offset);
3f518509
MW		 83}
84
16274427 85static u32 mvpp2_read_relaxed(struct mvpp2 *priv, u32 offset)
3f518509 86{
db9d7d36 87 return readl_relaxed(priv->swth_base[0] + offset);
3f518509 88}
543ec376 89
e531f767 90static inline u32 mvpp2_cpu_to_thread(struct mvpp2 *priv, int cpu)
543ec376 91{
e531f767 92 return cpu % priv->nthreads;
543ec376
AT		 93}
94
db9d7d36
MC		 95/* These accessors should be used to access:
96 *
543ec376 97 * - per-thread registers, where each thread has its own copy of the
db9d7d36
MC		 98 * register.
99 *
100 * MVPP2_BM_VIRT_ALLOC_REG
101 * MVPP2_BM_ADDR_HIGH_ALLOC
102 * MVPP22_BM_ADDR_HIGH_RLS_REG
103 * MVPP2_BM_VIRT_RLS_REG
104 * MVPP2_ISR_RX_TX_CAUSE_REG
105 * MVPP2_ISR_RX_TX_MASK_REG
106 * MVPP2_TXQ_NUM_REG
107 * MVPP2_AGGR_TXQ_UPDATE_REG
108 * MVPP2_TXQ_RSVD_REQ_REG
109 * MVPP2_TXQ_RSVD_RSLT_REG
110 * MVPP2_TXQ_SENT_REG
111 * MVPP2_RXQ_NUM_REG
112 *
543ec376
AT		 113 * - global registers that must be accessed through a specific thread
114 * window, because they are related to an access to a per-thread
db9d7d36
MC		 115 * register
116 *
117 * MVPP2_BM_PHY_ALLOC_REG (related to MVPP2_BM_VIRT_ALLOC_REG)
118 * MVPP2_BM_PHY_RLS_REG (related to MVPP2_BM_VIRT_RLS_REG)
119 * MVPP2_RXQ_THRESH_REG (related to MVPP2_RXQ_NUM_REG)
120 * MVPP2_RXQ_DESC_ADDR_REG (related to MVPP2_RXQ_NUM_REG)
121 * MVPP2_RXQ_DESC_SIZE_REG (related to MVPP2_RXQ_NUM_REG)
122 * MVPP2_RXQ_INDEX_REG (related to MVPP2_RXQ_NUM_REG)
123 * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
124 * MVPP2_TXQ_DESC_ADDR_REG (related to MVPP2_TXQ_NUM_REG)
125 * MVPP2_TXQ_DESC_SIZE_REG (related to MVPP2_TXQ_NUM_REG)
126 * MVPP2_TXQ_INDEX_REG (related to MVPP2_TXQ_NUM_REG)
127 * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
128 * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
129 * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
130 */
1068549c 131static void mvpp2_thread_write(struct mvpp2 *priv, unsigned int thread,
db9d7d36
MC		 132 u32 offset, u32 data)
133{
543ec376 134 writel(data, priv->swth_base[thread] + offset);
3f518509
MW		 135}
136
1068549c 137static u32 mvpp2_thread_read(struct mvpp2 *priv, unsigned int thread,
db9d7d36 138 u32 offset)
3f518509 139{
543ec376 140 return readl(priv->swth_base[thread] + offset);
db9d7d36 141}
3f518509 142
1068549c 143static void mvpp2_thread_write_relaxed(struct mvpp2 *priv, unsigned int thread,
db9d7d36
MC		 144 u32 offset, u32 data)
145{
543ec376 146 writel_relaxed(data, priv->swth_base[thread] + offset);
db9d7d36 147}
0c6d9b44 148
1068549c 149static u32 mvpp2_thread_read_relaxed(struct mvpp2 *priv, unsigned int thread,
db9d7d36
MC		 150 u32 offset)
151{
543ec376 152 return readl_relaxed(priv->swth_base[thread] + offset);
db9d7d36 153}
3f518509 154
db9d7d36
MC		 155static dma_addr_t mvpp2_txdesc_dma_addr_get(struct mvpp2_port *port,
156 struct mvpp2_tx_desc *tx_desc)
157{
158 if (port->priv->hw_version == MVPP21)
7b9c7d7d 159 return le32_to_cpu(tx_desc->pp21.buf_dma_addr);
db9d7d36 160 else
7b9c7d7d
MC		 161 return le64_to_cpu(tx_desc->pp22.buf_dma_addr_ptp) &
162 MVPP2_DESC_DMA_MASK;
db9d7d36 163}
3f518509 164
db9d7d36
MC		 165static void mvpp2_txdesc_dma_addr_set(struct mvpp2_port *port,
166 struct mvpp2_tx_desc *tx_desc,
167 dma_addr_t dma_addr)
168{
169 dma_addr_t addr, offset;
3f518509 170
db9d7d36
MC		 171 addr = dma_addr & ~MVPP2_TX_DESC_ALIGN;
172 offset = dma_addr & MVPP2_TX_DESC_ALIGN;
3f518509 173
db9d7d36 174 if (port->priv->hw_version == MVPP21) {
7b9c7d7d 175 tx_desc->pp21.buf_dma_addr = cpu_to_le32(addr);
db9d7d36 176 tx_desc->pp21.packet_offset = offset;
0c6d9b44 177 } else {
7b9c7d7d 178 __le64 val = cpu_to_le64(addr);
3f518509 179
7b9c7d7d 180 tx_desc->pp22.buf_dma_addr_ptp &= ~cpu_to_le64(MVPP2_DESC_DMA_MASK);
db9d7d36
MC		 181 tx_desc->pp22.buf_dma_addr_ptp |= val;
182 tx_desc->pp22.packet_offset = offset;
183 }
3f518509
MW		 184}
185
db9d7d36
MC		 186static size_t mvpp2_txdesc_size_get(struct mvpp2_port *port,
187 struct mvpp2_tx_desc *tx_desc)
3f518509 188{
db9d7d36 189 if (port->priv->hw_version == MVPP21)
7b9c7d7d 190 return le16_to_cpu(tx_desc->pp21.data_size);
db9d7d36 191 else
7b9c7d7d 192 return le16_to_cpu(tx_desc->pp22.data_size);
3f518509
MW		 193}
194
db9d7d36
MC		 195static void mvpp2_txdesc_size_set(struct mvpp2_port *port,
196 struct mvpp2_tx_desc *tx_desc,
197 size_t size)
3f518509 198{
db9d7d36 199 if (port->priv->hw_version == MVPP21)
7b9c7d7d 200 tx_desc->pp21.data_size = cpu_to_le16(size);
db9d7d36 201 else
7b9c7d7d 202 tx_desc->pp22.data_size = cpu_to_le16(size);
3f518509
MW		 203}
204
db9d7d36
MC		 205static void mvpp2_txdesc_txq_set(struct mvpp2_port *port,
206 struct mvpp2_tx_desc *tx_desc,
207 unsigned int txq)
3f518509 208{
db9d7d36
MC		 209 if (port->priv->hw_version == MVPP21)
210 tx_desc->pp21.phys_txq = txq;
211 else
212 tx_desc->pp22.phys_txq = txq;
3f518509
MW		 213}
214
db9d7d36
MC		 215static void mvpp2_txdesc_cmd_set(struct mvpp2_port *port,
216 struct mvpp2_tx_desc *tx_desc,
217 unsigned int command)
3f518509 218{
db9d7d36 219 if (port->priv->hw_version == MVPP21)
7b9c7d7d 220 tx_desc->pp21.command = cpu_to_le32(command);
db9d7d36 221 else
7b9c7d7d 222 tx_desc->pp22.command = cpu_to_le32(command);
db9d7d36 223}
3f518509 224
db9d7d36
MC		 225static unsigned int mvpp2_txdesc_offset_get(struct mvpp2_port *port,
226 struct mvpp2_tx_desc *tx_desc)
227{
228 if (port->priv->hw_version == MVPP21)
229 return tx_desc->pp21.packet_offset;
230 else
231 return tx_desc->pp22.packet_offset;
232}
3f518509 233
db9d7d36
MC		 234static dma_addr_t mvpp2_rxdesc_dma_addr_get(struct mvpp2_port *port,
235 struct mvpp2_rx_desc *rx_desc)
236{
237 if (port->priv->hw_version == MVPP21)
7b9c7d7d 238 return le32_to_cpu(rx_desc->pp21.buf_dma_addr);
db9d7d36 239 else
7b9c7d7d
MC		 240 return le64_to_cpu(rx_desc->pp22.buf_dma_addr_key_hash) &
241 MVPP2_DESC_DMA_MASK;
db9d7d36 242}
3f518509 243
db9d7d36
MC		 244static unsigned long mvpp2_rxdesc_cookie_get(struct mvpp2_port *port,
245 struct mvpp2_rx_desc *rx_desc)
246{
247 if (port->priv->hw_version == MVPP21)
7b9c7d7d 248 return le32_to_cpu(rx_desc->pp21.buf_cookie);
db9d7d36 249 else
7b9c7d7d
MC		 250 return le64_to_cpu(rx_desc->pp22.buf_cookie_misc) &
251 MVPP2_DESC_DMA_MASK;
db9d7d36 252}
3f518509 253
db9d7d36
MC		 254static size_t mvpp2_rxdesc_size_get(struct mvpp2_port *port,
255 struct mvpp2_rx_desc *rx_desc)
256{
257 if (port->priv->hw_version == MVPP21)
7b9c7d7d 258 return le16_to_cpu(rx_desc->pp21.data_size);
db9d7d36 259 else
7b9c7d7d 260 return le16_to_cpu(rx_desc->pp22.data_size);
db9d7d36 261}
3f518509 262
db9d7d36
MC		 263static u32 mvpp2_rxdesc_status_get(struct mvpp2_port *port,
264 struct mvpp2_rx_desc *rx_desc)
265{
266 if (port->priv->hw_version == MVPP21)
7b9c7d7d 267 return le32_to_cpu(rx_desc->pp21.status);
db9d7d36 268 else
7b9c7d7d 269 return le32_to_cpu(rx_desc->pp22.status);
3f518509
MW		 270}
271
db9d7d36 272static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)
3f518509 273{
db9d7d36
MC		 274 txq_pcpu->txq_get_index++;
275 if (txq_pcpu->txq_get_index == txq_pcpu->size)
276 txq_pcpu->txq_get_index = 0;
277}
3f518509 278
db9d7d36
MC		 279static void mvpp2_txq_inc_put(struct mvpp2_port *port,
280 struct mvpp2_txq_pcpu *txq_pcpu,
281 struct sk_buff *skb,
282 struct mvpp2_tx_desc *tx_desc)
283{
284 struct mvpp2_txq_pcpu_buf *tx_buf =
285 txq_pcpu->buffs + txq_pcpu->txq_put_index;
286 tx_buf->skb = skb;
287 tx_buf->size = mvpp2_txdesc_size_get(port, tx_desc);
288 tx_buf->dma = mvpp2_txdesc_dma_addr_get(port, tx_desc) +
289 mvpp2_txdesc_offset_get(port, tx_desc);
290 txq_pcpu->txq_put_index++;
291 if (txq_pcpu->txq_put_index == txq_pcpu->size)
292 txq_pcpu->txq_put_index = 0;
293}
3f518509 294
db9d7d36
MC		 295/* Get number of physical egress port */
296static inline int mvpp2_egress_port(struct mvpp2_port *port)
297{
298 return MVPP2_MAX_TCONT + port->id;
299}
3f518509 300
db9d7d36
MC		 301/* Get number of physical TXQ */
302static inline int mvpp2_txq_phys(int port, int txq)
303{
304 return (MVPP2_MAX_TCONT + port) * MVPP2_MAX_TXQ + txq;
3f518509
MW		 305}
306
0e037281
TP		 307static void *mvpp2_frag_alloc(const struct mvpp2_bm_pool *pool)
308{
309 if (likely(pool->frag_size <= PAGE_SIZE))
310 return netdev_alloc_frag(pool->frag_size);
311 else
312 return kmalloc(pool->frag_size, GFP_ATOMIC);
313}
314
315static void mvpp2_frag_free(const struct mvpp2_bm_pool *pool, void *data)
316{
317 if (likely(pool->frag_size <= PAGE_SIZE))
318 skb_free_frag(data);
319 else
320 kfree(data);
321}
322
3f518509
MW		 323/* Buffer Manager configuration routines */
324
325/* Create pool */
326static int mvpp2_bm_pool_create(struct platform_device *pdev,
327 struct mvpp2 *priv,
328 struct mvpp2_bm_pool *bm_pool, int size)
329{
3f518509
MW		 330 u32 val;
331
d01524d8
TP		 332 /* Number of buffer pointers must be a multiple of 16, as per
333 * hardware constraints
334 */
335 if (!IS_ALIGNED(size, 16))
336 return -EINVAL;
337
338 /* PPv2.1 needs 8 bytes per buffer pointer, PPv2.2 needs 16
339 * bytes per buffer pointer
340 */
341 if (priv->hw_version == MVPP21)
342 bm_pool->size_bytes = 2 * sizeof(u32) * size;
343 else
344 bm_pool->size_bytes = 2 * sizeof(u64) * size;
345
346 bm_pool->virt_addr = dma_alloc_coherent(&pdev->dev, bm_pool->size_bytes,
20396136 347 &bm_pool->dma_addr,
3f518509
MW		 348 GFP_KERNEL);
349 if (!bm_pool->virt_addr)
350 return -ENOMEM;
351
d3158807
TP		 352 if (!IS_ALIGNED((unsigned long)bm_pool->virt_addr,
353 MVPP2_BM_POOL_PTR_ALIGN)) {
d01524d8
TP		 354 dma_free_coherent(&pdev->dev, bm_pool->size_bytes,
355 bm_pool->virt_addr, bm_pool->dma_addr);
3f518509
MW		 356 dev_err(&pdev->dev, "BM pool %d is not %d bytes aligned\n",
357 bm_pool->id, MVPP2_BM_POOL_PTR_ALIGN);
358 return -ENOMEM;
359 }
360
361 mvpp2_write(priv, MVPP2_BM_POOL_BASE_REG(bm_pool->id),
d01524d8 362 lower_32_bits(bm_pool->dma_addr));
3f518509
MW		 363 mvpp2_write(priv, MVPP2_BM_POOL_SIZE_REG(bm_pool->id), size);
364
365 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id));
366 val |= MVPP2_BM_START_MASK;
367 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val);
368
3f518509
MW		 369 bm_pool->size = size;
370 bm_pool->pkt_size = 0;
371 bm_pool->buf_num = 0;
3f518509
MW		 372
373 return 0;
374}
375
376/* Set pool buffer size */
377static void mvpp2_bm_pool_bufsize_set(struct mvpp2 *priv,
378 struct mvpp2_bm_pool *bm_pool,
379 int buf_size)
380{
381 u32 val;
382
383 bm_pool->buf_size = buf_size;
384
385 val = ALIGN(buf_size, 1 << MVPP2_POOL_BUF_SIZE_OFFSET);
386 mvpp2_write(priv, MVPP2_POOL_BUF_SIZE_REG(bm_pool->id), val);
387}
388
d01524d8
TP		 389static void mvpp2_bm_bufs_get_addrs(struct device *dev, struct mvpp2 *priv,
390 struct mvpp2_bm_pool *bm_pool,
391 dma_addr_t *dma_addr,
392 phys_addr_t *phys_addr)
393{
e531f767 394 unsigned int thread = mvpp2_cpu_to_thread(priv, get_cpu());
a786841d 395
1068549c 396 *dma_addr = mvpp2_thread_read(priv, thread,
a786841d 397 MVPP2_BM_PHY_ALLOC_REG(bm_pool->id));
1068549c 398 *phys_addr = mvpp2_thread_read(priv, thread, MVPP2_BM_VIRT_ALLOC_REG);
d01524d8
TP		 399
400 if (priv->hw_version == MVPP22) {
401 u32 val;
402 u32 dma_addr_highbits, phys_addr_highbits;
403
1068549c 404 val = mvpp2_thread_read(priv, thread, MVPP22_BM_ADDR_HIGH_ALLOC);
d01524d8
TP		 405 dma_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_PHYS_MASK);
406 phys_addr_highbits = (val & MVPP22_BM_ADDR_HIGH_VIRT_MASK) >>
407 MVPP22_BM_ADDR_HIGH_VIRT_SHIFT;
408
409 if (sizeof(dma_addr_t) == 8)
410 *dma_addr |= (u64)dma_addr_highbits << 32;
411
412 if (sizeof(phys_addr_t) == 8)
413 *phys_addr |= (u64)phys_addr_highbits << 32;
414 }
a704bb5c
TP		 415
416 put_cpu();
d01524d8
TP		 417}
418
7861f12b 419/* Free all buffers from the pool */
4229d502 420static void mvpp2_bm_bufs_free(struct device *dev, struct mvpp2 *priv,
effbf5f5 421 struct mvpp2_bm_pool *bm_pool, int buf_num)
3f518509
MW		 422{
423 int i;
424
effbf5f5
SC		 425 if (buf_num > bm_pool->buf_num) {
426 WARN(1, "Pool does not have so many bufs pool(%d) bufs(%d)\n",
427 bm_pool->id, buf_num);
428 buf_num = bm_pool->buf_num;
429 }
430
431 for (i = 0; i < buf_num; i++) {
20396136 432 dma_addr_t buf_dma_addr;
4e4a105f
TP		 433 phys_addr_t buf_phys_addr;
434 void *data;
3f518509 435
d01524d8
TP		 436 mvpp2_bm_bufs_get_addrs(dev, priv, bm_pool,
437 &buf_dma_addr, &buf_phys_addr);
4229d502 438
20396136 439 dma_unmap_single(dev, buf_dma_addr,
4229d502
MW		 440 bm_pool->buf_size, DMA_FROM_DEVICE);
441
4e4a105f
TP		 442 data = (void *)phys_to_virt(buf_phys_addr);
443 if (!data)
3f518509 444 break;
0e037281 445
4e4a105f 446 mvpp2_frag_free(bm_pool, data);
3f518509
MW		 447 }
448
449 /* Update BM driver with number of buffers removed from pool */
450 bm_pool->buf_num -= i;
3f518509
MW		 451}
452
effbf5f5 453/* Check number of buffers in BM pool */
6e61e10a 454static int mvpp2_check_hw_buf_num(struct mvpp2 *priv, struct mvpp2_bm_pool *bm_pool)
effbf5f5
SC		 455{
456 int buf_num = 0;
457
458 buf_num += mvpp2_read(priv, MVPP2_BM_POOL_PTRS_NUM_REG(bm_pool->id)) &
459 MVPP22_BM_POOL_PTRS_NUM_MASK;
460 buf_num += mvpp2_read(priv, MVPP2_BM_BPPI_PTRS_NUM_REG(bm_pool->id)) &
461 MVPP2_BM_BPPI_PTR_NUM_MASK;
462
463 /* HW has one buffer ready which is not reflected in the counters */
464 if (buf_num)
465 buf_num += 1;
466
467 return buf_num;
468}
469
3f518509
MW		 470/* Cleanup pool */
471static int mvpp2_bm_pool_destroy(struct platform_device *pdev,
472 struct mvpp2 *priv,
473 struct mvpp2_bm_pool *bm_pool)
474{
effbf5f5 475 int buf_num;
3f518509
MW		 476 u32 val;
477
effbf5f5
SC		 478 buf_num = mvpp2_check_hw_buf_num(priv, bm_pool);
479 mvpp2_bm_bufs_free(&pdev->dev, priv, bm_pool, buf_num);
480
481 /* Check buffer counters after free */
482 buf_num = mvpp2_check_hw_buf_num(priv, bm_pool);
483 if (buf_num) {
484 WARN(1, "cannot free all buffers in pool %d, buf_num left %d\n",
485 bm_pool->id, bm_pool->buf_num);
3f518509
MW		 486 return 0;
487 }
488
489 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id));
490 val |= MVPP2_BM_STOP_MASK;
491 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val);
492
d01524d8 493 dma_free_coherent(&pdev->dev, bm_pool->size_bytes,
3f518509 494 bm_pool->virt_addr,
20396136 495 bm_pool->dma_addr);
3f518509
MW		 496 return 0;
497}
498
499static int mvpp2_bm_pools_init(struct platform_device *pdev,
500 struct mvpp2 *priv)
501{
502 int i, err, size;
503 struct mvpp2_bm_pool *bm_pool;
504
505 /* Create all pools with maximum size */
506 size = MVPP2_BM_POOL_SIZE_MAX;
507 for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
508 bm_pool = &priv->bm_pools[i];
509 bm_pool->id = i;
510 err = mvpp2_bm_pool_create(pdev, priv, bm_pool, size);
511 if (err)
512 goto err_unroll_pools;
513 mvpp2_bm_pool_bufsize_set(priv, bm_pool, 0);
514 }
515 return 0;
516
517err_unroll_pools:
518 dev_err(&pdev->dev, "failed to create BM pool %d, size %d\n", i, size);
519 for (i = i - 1; i >= 0; i--)
520 mvpp2_bm_pool_destroy(pdev, priv, &priv->bm_pools[i]);
521 return err;
522}
523
524static int mvpp2_bm_init(struct platform_device *pdev, struct mvpp2 *priv)
525{
526 int i, err;
527
528 for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
529 /* Mask BM all interrupts */
530 mvpp2_write(priv, MVPP2_BM_INTR_MASK_REG(i), 0);
531 /* Clear BM cause register */
532 mvpp2_write(priv, MVPP2_BM_INTR_CAUSE_REG(i), 0);
533 }
534
535 /* Allocate and initialize BM pools */
536 priv->bm_pools = devm_kcalloc(&pdev->dev, MVPP2_BM_POOLS_NUM,
81f915eb 537 sizeof(*priv->bm_pools), GFP_KERNEL);
3f518509
MW		 538 if (!priv->bm_pools)
539 return -ENOMEM;
540
541 err = mvpp2_bm_pools_init(pdev, priv);
542 if (err < 0)
543 return err;
544 return 0;
545}
546
01d04936
SC		 547static void mvpp2_setup_bm_pool(void)
548{
549 /* Short pool */
550 mvpp2_pools[MVPP2_BM_SHORT].buf_num = MVPP2_BM_SHORT_BUF_NUM;
551 mvpp2_pools[MVPP2_BM_SHORT].pkt_size = MVPP2_BM_SHORT_PKT_SIZE;
552
553 /* Long pool */
554 mvpp2_pools[MVPP2_BM_LONG].buf_num = MVPP2_BM_LONG_BUF_NUM;
555 mvpp2_pools[MVPP2_BM_LONG].pkt_size = MVPP2_BM_LONG_PKT_SIZE;
576193f2
SC		 556
557 /* Jumbo pool */
558 mvpp2_pools[MVPP2_BM_JUMBO].buf_num = MVPP2_BM_JUMBO_BUF_NUM;
559 mvpp2_pools[MVPP2_BM_JUMBO].pkt_size = MVPP2_BM_JUMBO_PKT_SIZE;
01d04936
SC		 560}
561
3f518509
MW		 562/* Attach long pool to rxq */
563static void mvpp2_rxq_long_pool_set(struct mvpp2_port *port,
564 int lrxq, int long_pool)
565{
5eac892a 566 u32 val, mask;
3f518509
MW		 567 int prxq;
568
569 /* Get queue physical ID */
570 prxq = port->rxqs[lrxq]->id;
571
5eac892a
TP		 572 if (port->priv->hw_version == MVPP21)
573 mask = MVPP21_RXQ_POOL_LONG_MASK;
574 else
575 mask = MVPP22_RXQ_POOL_LONG_MASK;
3f518509 576
5eac892a
TP		 577 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
578 val &= ~mask;
579 val |= (long_pool << MVPP2_RXQ_POOL_LONG_OFFS) & mask;
3f518509
MW		 580 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
581}
582
583/* Attach short pool to rxq */
584static void mvpp2_rxq_short_pool_set(struct mvpp2_port *port,
585 int lrxq, int short_pool)
586{
5eac892a 587 u32 val, mask;
3f518509
MW		 588 int prxq;
589
590 /* Get queue physical ID */
591 prxq = port->rxqs[lrxq]->id;
592
5eac892a
TP		 593 if (port->priv->hw_version == MVPP21)
594 mask = MVPP21_RXQ_POOL_SHORT_MASK;
595 else
596 mask = MVPP22_RXQ_POOL_SHORT_MASK;
3f518509 597
5eac892a
TP		 598 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
599 val &= ~mask;
600 val |= (short_pool << MVPP2_RXQ_POOL_SHORT_OFFS) & mask;
3f518509
MW		 601 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
602}
603
0e037281
TP		 604static void *mvpp2_buf_alloc(struct mvpp2_port *port,
605 struct mvpp2_bm_pool *bm_pool,
20396136 606 dma_addr_t *buf_dma_addr,
4e4a105f 607 phys_addr_t *buf_phys_addr,
0e037281 608 gfp_t gfp_mask)
3f518509 609{
20396136 610 dma_addr_t dma_addr;
0e037281 611 void *data;
3f518509 612
0e037281
TP		 613 data = mvpp2_frag_alloc(bm_pool);
614 if (!data)
3f518509
MW		 615 return NULL;
616
20396136
TP		 617 dma_addr = dma_map_single(port->dev->dev.parent, data,
618 MVPP2_RX_BUF_SIZE(bm_pool->pkt_size),
619 DMA_FROM_DEVICE);
620 if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) {
0e037281 621 mvpp2_frag_free(bm_pool, data);
3f518509
MW		 622 return NULL;
623 }
20396136 624 *buf_dma_addr = dma_addr;
4e4a105f 625 *buf_phys_addr = virt_to_phys(data);
3f518509 626
0e037281 627 return data;
3f518509
MW		 628}
629
3f518509
MW		 630/* Release buffer to BM */
631static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool,
20396136 632 dma_addr_t buf_dma_addr,
4e4a105f 633 phys_addr_t buf_phys_addr)
3f518509 634{
e531f767
AT		 635 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
636 unsigned long flags = 0;
637
638 if (test_bit(thread, &port->priv->lock_map))
639 spin_lock_irqsave(&port->bm_lock[thread], flags);
a786841d 640
d01524d8
TP		 641 if (port->priv->hw_version == MVPP22) {
642 u32 val = 0;
643
644 if (sizeof(dma_addr_t) == 8)
645 val |= upper_32_bits(buf_dma_addr) &
646 MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK;
647
648 if (sizeof(phys_addr_t) == 8)
649 val |= (upper_32_bits(buf_phys_addr)
650 << MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT) &
651 MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK;
652
1068549c 653 mvpp2_thread_write_relaxed(port->priv, thread,
cdcfeb0f 654 MVPP22_BM_ADDR_HIGH_RLS_REG, val);
d01524d8
TP		 655 }
656
4e4a105f
TP		 657 /* MVPP2_BM_VIRT_RLS_REG is not interpreted by HW, and simply
658 * returned in the "cookie" field of the RX
659 * descriptor. Instead of storing the virtual address, we
660 * store the physical address
661 */
1068549c 662 mvpp2_thread_write_relaxed(port->priv, thread,
cdcfeb0f 663 MVPP2_BM_VIRT_RLS_REG, buf_phys_addr);
1068549c 664 mvpp2_thread_write_relaxed(port->priv, thread,
cdcfeb0f 665 MVPP2_BM_PHY_RLS_REG(pool), buf_dma_addr);
a704bb5c 666
e531f767
AT		 667 if (test_bit(thread, &port->priv->lock_map))
668 spin_unlock_irqrestore(&port->bm_lock[thread], flags);
669
a704bb5c 670 put_cpu();
3f518509
MW		 671}
672
3f518509
MW		 673/* Allocate buffers for the pool */
674static int mvpp2_bm_bufs_add(struct mvpp2_port *port,
675 struct mvpp2_bm_pool *bm_pool, int buf_num)
676{
3f518509 677 int i, buf_size, total_size;
20396136 678 dma_addr_t dma_addr;
4e4a105f 679 phys_addr_t phys_addr;
0e037281 680 void *buf;
3f518509
MW		 681
682 buf_size = MVPP2_RX_BUF_SIZE(bm_pool->pkt_size);
683 total_size = MVPP2_RX_TOTAL_SIZE(buf_size);
684
685 if (buf_num < 0 ||
686 (buf_num + bm_pool->buf_num > bm_pool->size)) {
687 netdev_err(port->dev,
688 "cannot allocate %d buffers for pool %d\n",
689 buf_num, bm_pool->id);
690 return 0;
691 }
692
3f518509 693 for (i = 0; i < buf_num; i++) {
4e4a105f
TP		 694 buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr,
695 &phys_addr, GFP_KERNEL);
0e037281 696 if (!buf)
3f518509
MW		 697 break;
698
20396136 699 mvpp2_bm_pool_put(port, bm_pool->id, dma_addr,
4e4a105f 700 phys_addr);
3f518509
MW		 701 }
702
703 /* Update BM driver with number of buffers added to pool */
704 bm_pool->buf_num += i;
3f518509
MW		 705
706 netdev_dbg(port->dev,
01d04936 707 "pool %d: pkt_size=%4d, buf_size=%4d, total_size=%4d\n",
3f518509
MW		 708 bm_pool->id, bm_pool->pkt_size, buf_size, total_size);
709
710 netdev_dbg(port->dev,
01d04936 711 "pool %d: %d of %d buffers added\n",
3f518509
MW		 712 bm_pool->id, i, buf_num);
713 return i;
714}
715
716/* Notify the driver that BM pool is being used as specific type and return the
717 * pool pointer on success
718 */
719static struct mvpp2_bm_pool *
01d04936 720mvpp2_bm_pool_use(struct mvpp2_port *port, unsigned pool, int pkt_size)
3f518509 721{
3f518509
MW		 722 struct mvpp2_bm_pool *new_pool = &port->priv->bm_pools[pool];
723 int num;
724
01d04936
SC		 725 if (pool >= MVPP2_BM_POOLS_NUM) {
726 netdev_err(port->dev, "Invalid pool %d\n", pool);
3f518509
MW		 727 return NULL;
728 }
729
3f518509
MW		 730 /* Allocate buffers in case BM pool is used as long pool, but packet
731 * size doesn't match MTU or BM pool hasn't being used yet
732 */
01d04936 733 if (new_pool->pkt_size == 0) {
3f518509
MW		 734 int pkts_num;
735
736 /* Set default buffer number or free all the buffers in case
737 * the pool is not empty
738 */
739 pkts_num = new_pool->buf_num;
740 if (pkts_num == 0)
01d04936 741 pkts_num = mvpp2_pools[pool].buf_num;
3f518509 742 else
4229d502 743 mvpp2_bm_bufs_free(port->dev->dev.parent,
effbf5f5 744 port->priv, new_pool, pkts_num);
3f518509
MW		 745
746 new_pool->pkt_size = pkt_size;
0e037281
TP		 747 new_pool->frag_size =
748 SKB_DATA_ALIGN(MVPP2_RX_BUF_SIZE(pkt_size)) +
749 MVPP2_SKB_SHINFO_SIZE;
3f518509
MW		 750
751 /* Allocate buffers for this pool */
752 num = mvpp2_bm_bufs_add(port, new_pool, pkts_num);
753 if (num != pkts_num) {
754 WARN(1, "pool %d: %d of %d allocated\n",
755 new_pool->id, num, pkts_num);
3f518509
MW		 756 return NULL;
757 }
758 }
759
760 mvpp2_bm_pool_bufsize_set(port->priv, new_pool,
761 MVPP2_RX_BUF_SIZE(new_pool->pkt_size));
762
3f518509
MW		 763 return new_pool;
764}
765
766/* Initialize pools for swf */
767static int mvpp2_swf_bm_pool_init(struct mvpp2_port *port)
768{
3f518509 769 int rxq;
576193f2
SC		 770 enum mvpp2_bm_pool_log_num long_log_pool, short_log_pool;
771
772 /* If port pkt_size is higher than 1518B:
773 * HW Long pool - SW Jumbo pool, HW Short pool - SW Long pool
774 * else: HW Long pool - SW Long pool, HW Short pool - SW Short pool
775 */
776 if (port->pkt_size > MVPP2_BM_LONG_PKT_SIZE) {
777 long_log_pool = MVPP2_BM_JUMBO;
778 short_log_pool = MVPP2_BM_LONG;
779 } else {
780 long_log_pool = MVPP2_BM_LONG;
781 short_log_pool = MVPP2_BM_SHORT;
782 }
3f518509
MW		 783
784 if (!port->pool_long) {
785 port->pool_long =
576193f2
SC		 786 mvpp2_bm_pool_use(port, long_log_pool,
787 mvpp2_pools[long_log_pool].pkt_size);
3f518509
MW		 788 if (!port->pool_long)
789 return -ENOMEM;
790
576193f2 791 port->pool_long->port_map |= BIT(port->id);
3f518509 792
09f83975 793 for (rxq = 0; rxq < port->nrxqs; rxq++)
3f518509
MW		 794 mvpp2_rxq_long_pool_set(port, rxq, port->pool_long->id);
795 }
796
797 if (!port->pool_short) {
798 port->pool_short =
576193f2 799 mvpp2_bm_pool_use(port, short_log_pool,
e2e03164 800 mvpp2_pools[short_log_pool].pkt_size);
3f518509
MW		 801 if (!port->pool_short)
802 return -ENOMEM;
803
576193f2 804 port->pool_short->port_map |= BIT(port->id);
3f518509 805
09f83975 806 for (rxq = 0; rxq < port->nrxqs; rxq++)
3f518509
MW		 807 mvpp2_rxq_short_pool_set(port, rxq,
808 port->pool_short->id);
809 }
810
811 return 0;
812}
813
814static int mvpp2_bm_update_mtu(struct net_device *dev, int mtu)
815{
816 struct mvpp2_port *port = netdev_priv(dev);
576193f2
SC		 817 enum mvpp2_bm_pool_log_num new_long_pool;
818 int pkt_size = MVPP2_RX_PKT_SIZE(mtu);
3f518509 819
576193f2
SC		 820 /* If port MTU is higher than 1518B:
821 * HW Long pool - SW Jumbo pool, HW Short pool - SW Long pool
822 * else: HW Long pool - SW Long pool, HW Short pool - SW Short pool
823 */
824 if (pkt_size > MVPP2_BM_LONG_PKT_SIZE)
825 new_long_pool = MVPP2_BM_JUMBO;
826 else
827 new_long_pool = MVPP2_BM_LONG;
828
829 if (new_long_pool != port->pool_long->id) {
830 /* Remove port from old short & long pool */
831 port->pool_long = mvpp2_bm_pool_use(port, port->pool_long->id,
832 port->pool_long->pkt_size);
833 port->pool_long->port_map &= ~BIT(port->id);
834 port->pool_long = NULL;
835
836 port->pool_short = mvpp2_bm_pool_use(port, port->pool_short->id,
837 port->pool_short->pkt_size);
838 port->pool_short->port_map &= ~BIT(port->id);
839 port->pool_short = NULL;
840
841 port->pkt_size = pkt_size;
842
843 /* Add port to new short & long pool */
844 mvpp2_swf_bm_pool_init(port);
845
846 /* Update L4 checksum when jumbo enable/disable on port */
847 if (new_long_pool == MVPP2_BM_JUMBO && port->id != 0) {
848 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
849 dev->hw_features &= ~(NETIF_F_IP_CSUM |
850 NETIF_F_IPV6_CSUM);
851 } else {
852 dev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
853 dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
854 }
3f518509
MW		 855 }
856
3f518509 857 dev->mtu = mtu;
576193f2
SC		 858 dev->wanted_features = dev->features;
859
3f518509
MW		 860 netdev_update_features(dev);
861 return 0;
862}
863
864static inline void mvpp2_interrupts_enable(struct mvpp2_port *port)
865{
591f4cfa
TP		 866 int i, sw_thread_mask = 0;
867
868 for (i = 0; i < port->nqvecs; i++)
869 sw_thread_mask |= port->qvecs[i].sw_thread_mask;
3f518509 870
3f518509 871 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
591f4cfa 872 MVPP2_ISR_ENABLE_INTERRUPT(sw_thread_mask));
3f518509
MW		 873}
874
875static inline void mvpp2_interrupts_disable(struct mvpp2_port *port)
876{
591f4cfa
TP		 877 int i, sw_thread_mask = 0;
878
879 for (i = 0; i < port->nqvecs; i++)
880 sw_thread_mask |= port->qvecs[i].sw_thread_mask;
881
882 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
883 MVPP2_ISR_DISABLE_INTERRUPT(sw_thread_mask));
884}
885
886static inline void mvpp2_qvec_interrupt_enable(struct mvpp2_queue_vector *qvec)
887{
888 struct mvpp2_port *port = qvec->port;
3f518509 889
3f518509 890 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
591f4cfa
TP		 891 MVPP2_ISR_ENABLE_INTERRUPT(qvec->sw_thread_mask));
892}
893
894static inline void mvpp2_qvec_interrupt_disable(struct mvpp2_queue_vector *qvec)
895{
896 struct mvpp2_port *port = qvec->port;
897
898 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id),
899 MVPP2_ISR_DISABLE_INTERRUPT(qvec->sw_thread_mask));
3f518509
MW		 900}
901
543ec376 902/* Mask the current thread's Rx/Tx interrupts
e0af22d9
TP		 903 * Called by on_each_cpu(), guaranteed to run with migration disabled,
904 * using smp_processor_id() is OK.
905 */
3f518509
MW		 906static void mvpp2_interrupts_mask(void *arg)
907{
908 struct mvpp2_port *port = arg;
909
e531f767
AT		 910 /* If the thread isn't used, don't do anything */
911 if (smp_processor_id() > port->priv->nthreads)
912 return;
913
1068549c 914 mvpp2_thread_write(port->priv,
e531f767 915 mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
a786841d 916 MVPP2_ISR_RX_TX_MASK_REG(port->id), 0);
3f518509
MW		 917}
918
543ec376 919/* Unmask the current thread's Rx/Tx interrupts.
e0af22d9
TP		 920 * Called by on_each_cpu(), guaranteed to run with migration disabled,
921 * using smp_processor_id() is OK.
922 */
3f518509
MW		 923static void mvpp2_interrupts_unmask(void *arg)
924{
925 struct mvpp2_port *port = arg;
213f428f
TP		 926 u32 val;
927
e531f767
AT		 928 /* If the thread isn't used, don't do anything */
929 if (smp_processor_id() > port->priv->nthreads)
930 return;
931
213f428f 932 val = MVPP2_CAUSE_MISC_SUM_MASK |
70afb58e 933 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version);
213f428f
TP		 934 if (port->has_tx_irqs)
935 val |= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
3f518509 936
1068549c 937 mvpp2_thread_write(port->priv,
e531f767 938 mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
213f428f
TP		 939 MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
940}
941
942static void
943mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask)
944{
945 u32 val;
946 int i;
947
948 if (port->priv->hw_version != MVPP22)
949 return;
950
951 if (mask)
952 val = 0;
953 else
70afb58e 954 val = MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(MVPP22);
213f428f
TP		 955
956 for (i = 0; i < port->nqvecs; i++) {
957 struct mvpp2_queue_vector *v = port->qvecs + i;
958
959 if (v->type != MVPP2_QUEUE_VECTOR_SHARED)
960 continue;
961
1068549c 962 mvpp2_thread_write(port->priv, v->sw_thread_id,
213f428f
TP		 963 MVPP2_ISR_RX_TX_MASK_REG(port->id), val);
964 }
3f518509
MW		 965}
966
967/* Port configuration routines */
968
f84bf386
AT		 969static void mvpp22_gop_init_rgmii(struct mvpp2_port *port)
970{
971 struct mvpp2 *priv = port->priv;
972 u32 val;
973
974 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
975 val |= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT;
976 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
977
978 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
979 if (port->gop_id == 2)
980 val |= GENCONF_CTRL0_PORT0_RGMII | GENCONF_CTRL0_PORT1_RGMII;
981 else if (port->gop_id == 3)
982 val |= GENCONF_CTRL0_PORT1_RGMII_MII;
983 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
984}
985
986static void mvpp22_gop_init_sgmii(struct mvpp2_port *port)
987{
988 struct mvpp2 *priv = port->priv;
989 u32 val;
990
991 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
992 val |= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT |
993 GENCONF_PORT_CTRL0_RX_DATA_SAMPLE;
994 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
995
996 if (port->gop_id > 1) {
997 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val);
998 if (port->gop_id == 2)
999 val &= ~GENCONF_CTRL0_PORT0_RGMII;
1000 else if (port->gop_id == 3)
1001 val &= ~GENCONF_CTRL0_PORT1_RGMII_MII;
1002 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val);
1003 }
1004}
1005
1006static void mvpp22_gop_init_10gkr(struct mvpp2_port *port)
1007{
1008 struct mvpp2 *priv = port->priv;
1009 void __iomem *mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id);
1010 void __iomem *xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id);
1011 u32 val;
1012
1013 /* XPCS */
1014 val = readl(xpcs + MVPP22_XPCS_CFG0);
1015 val &= ~(MVPP22_XPCS_CFG0_PCS_MODE(0x3) |
1016 MVPP22_XPCS_CFG0_ACTIVE_LANE(0x3));
1017 val |= MVPP22_XPCS_CFG0_ACTIVE_LANE(2);
1018 writel(val, xpcs + MVPP22_XPCS_CFG0);
1019
1020 /* MPCS */
1021 val = readl(mpcs + MVPP22_MPCS_CTRL);
1022 val &= ~MVPP22_MPCS_CTRL_FWD_ERR_CONN;
1023 writel(val, mpcs + MVPP22_MPCS_CTRL);
1024
1025 val = readl(mpcs + MVPP22_MPCS_CLK_RESET);
1026 val &= ~(MVPP22_MPCS_CLK_RESET_DIV_RATIO(0x7) | MAC_CLK_RESET_MAC |
1027 MAC_CLK_RESET_SD_RX | MAC_CLK_RESET_SD_TX);
1028 val |= MVPP22_MPCS_CLK_RESET_DIV_RATIO(1);
1029 writel(val, mpcs + MVPP22_MPCS_CLK_RESET);
1030
1031 val &= ~MVPP22_MPCS_CLK_RESET_DIV_SET;
1032 val |= MAC_CLK_RESET_MAC | MAC_CLK_RESET_SD_RX | MAC_CLK_RESET_SD_TX;
1033 writel(val, mpcs + MVPP22_MPCS_CLK_RESET);
1034}
1035
1036static int mvpp22_gop_init(struct mvpp2_port *port)
1037{
1038 struct mvpp2 *priv = port->priv;
1039 u32 val;
1040
1041 if (!priv->sysctrl_base)
1042 return 0;
1043
1044 switch (port->phy_interface) {
1045 case PHY_INTERFACE_MODE_RGMII:
1046 case PHY_INTERFACE_MODE_RGMII_ID:
1047 case PHY_INTERFACE_MODE_RGMII_RXID:
1048 case PHY_INTERFACE_MODE_RGMII_TXID:
1049 if (port->gop_id == 0)
1050 goto invalid_conf;
1051 mvpp22_gop_init_rgmii(port);
1052 break;
1053 case PHY_INTERFACE_MODE_SGMII:
d97c9f4a 1054 case PHY_INTERFACE_MODE_1000BASEX:
a6fe31de 1055 case PHY_INTERFACE_MODE_2500BASEX:
f84bf386
AT		 1056 mvpp22_gop_init_sgmii(port);
1057 break;
1058 case PHY_INTERFACE_MODE_10GKR:
1059 if (port->gop_id != 0)
1060 goto invalid_conf;
1061 mvpp22_gop_init_10gkr(port);
1062 break;
1063 default:
1064 goto unsupported_conf;
1065 }
1066
1067 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL1, &val);
1068 val |= GENCONF_PORT_CTRL1_RESET(port->gop_id) |
1069 GENCONF_PORT_CTRL1_EN(port->gop_id);
1070 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL1, val);
1071
1072 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val);
1073 val |= GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR;
1074 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val);
1075
1076 regmap_read(priv->sysctrl_base, GENCONF_SOFT_RESET1, &val);
1077 val |= GENCONF_SOFT_RESET1_GOP;
1078 regmap_write(priv->sysctrl_base, GENCONF_SOFT_RESET1, val);
1079
1080unsupported_conf:
1081 return 0;
1082
1083invalid_conf:
1084 netdev_err(port->dev, "Invalid port configuration\n");
1085 return -EINVAL;
1086}
1087
fd3651b2
AT		 1088static void mvpp22_gop_unmask_irq(struct mvpp2_port *port)
1089{
1090 u32 val;
1091
1092 if (phy_interface_mode_is_rgmii(port->phy_interface) ||
d97c9f4a 1093 port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 1094 port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
1095 port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
fd3651b2
AT		 1096 /* Enable the GMAC link status irq for this port */
1097 val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK);
1098 val |= MVPP22_GMAC_INT_SUM_MASK_LINK_STAT;
1099 writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK);
1100 }
1101
1102 if (port->gop_id == 0) {
1103 /* Enable the XLG/GIG irqs for this port */
1104 val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
1105 if (port->phy_interface == PHY_INTERFACE_MODE_10GKR)
1106 val |= MVPP22_XLG_EXT_INT_MASK_XLG;
1107 else
1108 val |= MVPP22_XLG_EXT_INT_MASK_GIG;
1109 writel(val, port->base + MVPP22_XLG_EXT_INT_MASK);
1110 }
1111}
1112
1113static void mvpp22_gop_mask_irq(struct mvpp2_port *port)
1114{
1115 u32 val;
1116
1117 if (port->gop_id == 0) {
1118 val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
1119 val &= ~(MVPP22_XLG_EXT_INT_MASK_XLG |
a3302baa 1120 MVPP22_XLG_EXT_INT_MASK_GIG);
fd3651b2
AT		 1121 writel(val, port->base + MVPP22_XLG_EXT_INT_MASK);
1122 }
1123
1124 if (phy_interface_mode_is_rgmii(port->phy_interface) ||
d97c9f4a 1125 port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 1126 port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
1127 port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
fd3651b2
AT		 1128 val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK);
1129 val &= ~MVPP22_GMAC_INT_SUM_MASK_LINK_STAT;
1130 writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK);
1131 }
1132}
1133
1134static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
1135{
1136 u32 val;
1137
1138 if (phy_interface_mode_is_rgmii(port->phy_interface) ||
d97c9f4a 1139 port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 1140 port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
1141 port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
fd3651b2
AT		 1142 val = readl(port->base + MVPP22_GMAC_INT_MASK);
1143 val |= MVPP22_GMAC_INT_MASK_LINK_STAT;
1144 writel(val, port->base + MVPP22_GMAC_INT_MASK);
1145 }
1146
1147 if (port->gop_id == 0) {
1148 val = readl(port->base + MVPP22_XLG_INT_MASK);
1149 val |= MVPP22_XLG_INT_MASK_LINK;
1150 writel(val, port->base + MVPP22_XLG_INT_MASK);
1151 }
1152
1153 mvpp22_gop_unmask_irq(port);
1154}
1155
a6fe31de
AT		 1156/* Sets the PHY mode of the COMPHY (which configures the serdes lanes).
1157 *
1158 * The PHY mode used by the PPv2 driver comes from the network subsystem, while
1159 * the one given to the COMPHY comes from the generic PHY subsystem. Hence they
1160 * differ.
1161 *
1162 * The COMPHY configures the serdes lanes regardless of the actual use of the
1163 * lanes by the physical layer. This is why configurations like
1164 * "PPv2 (2500BaseX) - COMPHY (2500SGMII)" are valid.
1165 */
542897d9
AT		 1166static int mvpp22_comphy_init(struct mvpp2_port *port)
1167{
542897d9
AT		 1168 int ret;
1169
1170 if (!port->comphy)
1171 return 0;
1172
cccc43b8
GS		 1173 ret = phy_set_mode_ext(port->comphy, PHY_MODE_ETHERNET,
1174 port->phy_interface);
542897d9
AT		 1175 if (ret)
1176 return ret;
1177
1178 return phy_power_on(port->comphy);
1179}
1180
3f518509
MW		 1181static void mvpp2_port_enable(struct mvpp2_port *port)
1182{
1183 u32 val;
1184
725757ae
AT		 1185 /* Only GOP port 0 has an XLG MAC */
1186 if (port->gop_id == 0 &&
1187 (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
1188 port->phy_interface == PHY_INTERFACE_MODE_10GKR)) {
1189 val = readl(port->base + MVPP22_XLG_CTRL0_REG);
1190 val |= MVPP22_XLG_CTRL0_PORT_EN |
1191 MVPP22_XLG_CTRL0_MAC_RESET_DIS;
1192 val &= ~MVPP22_XLG_CTRL0_MIB_CNT_DIS;
1193 writel(val, port->base + MVPP22_XLG_CTRL0_REG);
1194 } else {
1195 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
1196 val |= MVPP2_GMAC_PORT_EN_MASK;
1197 val |= MVPP2_GMAC_MIB_CNTR_EN_MASK;
1198 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
1199 }
3f518509
MW		 1200}
1201
1202static void mvpp2_port_disable(struct mvpp2_port *port)
1203{
1204 u32 val;
1205
725757ae
AT		 1206 /* Only GOP port 0 has an XLG MAC */
1207 if (port->gop_id == 0 &&
1208 (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
1209 port->phy_interface == PHY_INTERFACE_MODE_10GKR)) {
1210 val = readl(port->base + MVPP22_XLG_CTRL0_REG);
4bb04326
AT		 1211 val &= ~MVPP22_XLG_CTRL0_PORT_EN;
1212 writel(val, port->base + MVPP22_XLG_CTRL0_REG);
1213
1214 /* Disable & reset should be done separately */
1215 val &= ~MVPP22_XLG_CTRL0_MAC_RESET_DIS;
725757ae
AT		 1216 writel(val, port->base + MVPP22_XLG_CTRL0_REG);
1217 } else {
1218 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
1219 val &= ~(MVPP2_GMAC_PORT_EN_MASK);
1220 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
1221 }
3f518509
MW		 1222}
1223
1224/* Set IEEE 802.3x Flow Control Xon Packet Transmission Mode */
1225static void mvpp2_port_periodic_xon_disable(struct mvpp2_port *port)
1226{
1227 u32 val;
1228
1229 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG) &
1230 ~MVPP2_GMAC_PERIODIC_XON_EN_MASK;
1231 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG);
1232}
1233
1234/* Configure loopback port */
4bb04326
AT		 1235static void mvpp2_port_loopback_set(struct mvpp2_port *port,
1236 const struct phylink_link_state *state)
3f518509
MW		 1237{
1238 u32 val;
1239
1240 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG);
1241
4bb04326 1242 if (state->speed == 1000)
3f518509
MW		 1243 val |= MVPP2_GMAC_GMII_LB_EN_MASK;
1244 else
1245 val &= ~MVPP2_GMAC_GMII_LB_EN_MASK;
1246
d97c9f4a 1247 if (port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 1248 port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
1249 port->phy_interface == PHY_INTERFACE_MODE_2500BASEX)
3f518509
MW		 1250 val |= MVPP2_GMAC_PCS_LB_EN_MASK;
1251 else
1252 val &= ~MVPP2_GMAC_PCS_LB_EN_MASK;
1253
1254 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG);
1255}
1256
118d6298
MR		 1257struct mvpp2_ethtool_counter {
1258 unsigned int offset;
1259 const char string[ETH_GSTRING_LEN];
1260 bool reg_is_64b;
1261};
1262
1263static u64 mvpp2_read_count(struct mvpp2_port *port,
1264 const struct mvpp2_ethtool_counter *counter)
1265{
1266 u64 val;
1267
1268 val = readl(port->stats_base + counter->offset);
1269 if (counter->reg_is_64b)
1270 val += (u64)readl(port->stats_base + counter->offset + 4) << 32;
1271
1272 return val;
1273}
1274
1275/* Due to the fact that software statistics and hardware statistics are, by
1276 * design, incremented at different moments in the chain of packet processing,
1277 * it is very likely that incoming packets could have been dropped after being
1278 * counted by hardware but before reaching software statistics (most probably
1279 * multicast packets), and in the oppposite way, during transmission, FCS bytes
1280 * are added in between as well as TSO skb will be split and header bytes added.
1281 * Hence, statistics gathered from userspace with ifconfig (software) and
1282 * ethtool (hardware) cannot be compared.
1283 */
1284static const struct mvpp2_ethtool_counter mvpp2_ethtool_regs[] = {
1285 { MVPP2_MIB_GOOD_OCTETS_RCVD, "good_octets_received", true },
1286 { MVPP2_MIB_BAD_OCTETS_RCVD, "bad_octets_received" },
1287 { MVPP2_MIB_CRC_ERRORS_SENT, "crc_errors_sent" },
1288 { MVPP2_MIB_UNICAST_FRAMES_RCVD, "unicast_frames_received" },
1289 { MVPP2_MIB_BROADCAST_FRAMES_RCVD, "broadcast_frames_received" },
1290 { MVPP2_MIB_MULTICAST_FRAMES_RCVD, "multicast_frames_received" },
1291 { MVPP2_MIB_FRAMES_64_OCTETS, "frames_64_octets" },
1292 { MVPP2_MIB_FRAMES_65_TO_127_OCTETS, "frames_65_to_127_octet" },
1293 { MVPP2_MIB_FRAMES_128_TO_255_OCTETS, "frames_128_to_255_octet" },
1294 { MVPP2_MIB_FRAMES_256_TO_511_OCTETS, "frames_256_to_511_octet" },
1295 { MVPP2_MIB_FRAMES_512_TO_1023_OCTETS, "frames_512_to_1023_octet" },
1296 { MVPP2_MIB_FRAMES_1024_TO_MAX_OCTETS, "frames_1024_to_max_octet" },
1297 { MVPP2_MIB_GOOD_OCTETS_SENT, "good_octets_sent", true },
1298 { MVPP2_MIB_UNICAST_FRAMES_SENT, "unicast_frames_sent" },
1299 { MVPP2_MIB_MULTICAST_FRAMES_SENT, "multicast_frames_sent" },
1300 { MVPP2_MIB_BROADCAST_FRAMES_SENT, "broadcast_frames_sent" },
1301 { MVPP2_MIB_FC_SENT, "fc_sent" },
1302 { MVPP2_MIB_FC_RCVD, "fc_received" },
1303 { MVPP2_MIB_RX_FIFO_OVERRUN, "rx_fifo_overrun" },
1304 { MVPP2_MIB_UNDERSIZE_RCVD, "undersize_received" },
1305 { MVPP2_MIB_FRAGMENTS_RCVD, "fragments_received" },
1306 { MVPP2_MIB_OVERSIZE_RCVD, "oversize_received" },
1307 { MVPP2_MIB_JABBER_RCVD, "jabber_received" },
1308 { MVPP2_MIB_MAC_RCV_ERROR, "mac_receive_error" },
1309 { MVPP2_MIB_BAD_CRC_EVENT, "bad_crc_event" },
1310 { MVPP2_MIB_COLLISION, "collision" },
1311 { MVPP2_MIB_LATE_COLLISION, "late_collision" },
1312};
1313
1314static void mvpp2_ethtool_get_strings(struct net_device *netdev, u32 sset,
1315 u8 *data)
1316{
1317 if (sset == ETH_SS_STATS) {
1318 int i;
1319
1320 for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_regs); i++)
1321 memcpy(data + i * ETH_GSTRING_LEN,
1322 &mvpp2_ethtool_regs[i].string, ETH_GSTRING_LEN);
1323 }
1324}
1325
1326static void mvpp2_gather_hw_statistics(struct work_struct *work)
1327{
1328 struct delayed_work *del_work = to_delayed_work(work);
e5c500eb
MR		 1329 struct mvpp2_port *port = container_of(del_work, struct mvpp2_port,
1330 stats_work);
118d6298 1331 u64 *pstats;
e5c500eb 1332 int i;
118d6298 1333
e5c500eb 1334 mutex_lock(&port->gather_stats_lock);
118d6298 1335
e5c500eb
MR		 1336 pstats = port->ethtool_stats;
1337 for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_regs); i++)
1338 *pstats++ += mvpp2_read_count(port, &mvpp2_ethtool_regs[i]);
118d6298
MR		 1339
1340 /* No need to read again the counters right after this function if it
1341 * was called asynchronously by the user (ie. use of ethtool).
1342 */
e5c500eb
MR		 1343 cancel_delayed_work(&port->stats_work);
1344 queue_delayed_work(port->priv->stats_queue, &port->stats_work,
118d6298
MR		 1345 MVPP2_MIB_COUNTERS_STATS_DELAY);
1346
e5c500eb 1347 mutex_unlock(&port->gather_stats_lock);
118d6298
MR		 1348}
1349
1350static void mvpp2_ethtool_get_stats(struct net_device *dev,
1351 struct ethtool_stats *stats, u64 *data)
1352{
1353 struct mvpp2_port *port = netdev_priv(dev);
1354
e5c500eb
MR		 1355 /* Update statistics for the given port, then take the lock to avoid
1356 * concurrent accesses on the ethtool_stats structure during its copy.
1357 */
1358 mvpp2_gather_hw_statistics(&port->stats_work.work);
118d6298 1359
e5c500eb 1360 mutex_lock(&port->gather_stats_lock);
118d6298
MR		 1361 memcpy(data, port->ethtool_stats,
1362 sizeof(u64) * ARRAY_SIZE(mvpp2_ethtool_regs));
e5c500eb 1363 mutex_unlock(&port->gather_stats_lock);
118d6298
MR		 1364}
1365
1366static int mvpp2_ethtool_get_sset_count(struct net_device *dev, int sset)
1367{
1368 if (sset == ETH_SS_STATS)
1369 return ARRAY_SIZE(mvpp2_ethtool_regs);
1370
1371 return -EOPNOTSUPP;
1372}
1373
3f518509
MW		 1374static void mvpp2_port_reset(struct mvpp2_port *port)
1375{
1376 u32 val;
118d6298
MR		 1377 unsigned int i;
1378
1379 /* Read the GOP statistics to reset the hardware counters */
1380 for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_regs); i++)
1381 mvpp2_read_count(port, &mvpp2_ethtool_regs[i]);
3f518509
MW		 1382
1383 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG) &
1384 ~MVPP2_GMAC_PORT_RESET_MASK;
1385 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG);
1386
1387 while (readl(port->base + MVPP2_GMAC_CTRL_2_REG) &
1388 MVPP2_GMAC_PORT_RESET_MASK)
1389 continue;
1390}
1391
1392/* Change maximum receive size of the port */
1393static inline void mvpp2_gmac_max_rx_size_set(struct mvpp2_port *port)
1394{
1395 u32 val;
1396
1397 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
1398 val &= ~MVPP2_GMAC_MAX_RX_SIZE_MASK;
1399 val |= (((port->pkt_size - MVPP2_MH_SIZE) / 2) <<
1400 MVPP2_GMAC_MAX_RX_SIZE_OFFS);
1401 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG);
1402}
1403
76eb1b1d
SC		 1404/* Change maximum receive size of the port */
1405static inline void mvpp2_xlg_max_rx_size_set(struct mvpp2_port *port)
1406{
1407 u32 val;
1408
1409 val = readl(port->base + MVPP22_XLG_CTRL1_REG);
1410 val &= ~MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK;
1411 val |= ((port->pkt_size - MVPP2_MH_SIZE) / 2) <<
ec15ecde 1412 MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS;
76eb1b1d
SC		 1413 writel(val, port->base + MVPP22_XLG_CTRL1_REG);
1414}
1415
3f518509
MW		 1416/* Set defaults to the MVPP2 port */
1417static void mvpp2_defaults_set(struct mvpp2_port *port)
1418{
1419 int tx_port_num, val, queue, ptxq, lrxq;
1420
3d9017d9 1421 if (port->priv->hw_version == MVPP21) {
3d9017d9
TP		 1422 /* Update TX FIFO MIN Threshold */
1423 val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
1424 val &= ~MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK;
1425 /* Min. TX threshold must be less than minimal packet length */
1426 val |= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2);
1427 writel(val, port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG);
1428 }
3f518509
MW		 1429
1430 /* Disable Legacy WRR, Disable EJP, Release from reset */
1431 tx_port_num = mvpp2_egress_port(port);
1432 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG,
1433 tx_port_num);
1434 mvpp2_write(port->priv, MVPP2_TXP_SCHED_CMD_1_REG, 0);
1435
4251ea5b
MC		 1436 /* Set TXQ scheduling to Round-Robin */
1437 mvpp2_write(port->priv, MVPP2_TXP_SCHED_FIXED_PRIO_REG, 0);
1438
3f518509
MW		 1439 /* Close bandwidth for all queues */
1440 for (queue = 0; queue < MVPP2_MAX_TXQ; queue++) {
1441 ptxq = mvpp2_txq_phys(port->id, queue);
1442 mvpp2_write(port->priv,
1443 MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(ptxq), 0);
1444 }
1445
1446 /* Set refill period to 1 usec, refill tokens
1447 * and bucket size to maximum
1448 */
1449 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PERIOD_REG,
1450 port->priv->tclk / USEC_PER_SEC);
1451 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_REFILL_REG);
1452 val &= ~MVPP2_TXP_REFILL_PERIOD_ALL_MASK;
1453 val |= MVPP2_TXP_REFILL_PERIOD_MASK(1);
1454 val |= MVPP2_TXP_REFILL_TOKENS_ALL_MASK;
1455 mvpp2_write(port->priv, MVPP2_TXP_SCHED_REFILL_REG, val);
1456 val = MVPP2_TXP_TOKEN_SIZE_MAX;
1457 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val);
1458
1459 /* Set MaximumLowLatencyPacketSize value to 256 */
1460 mvpp2_write(port->priv, MVPP2_RX_CTRL_REG(port->id),
1461 MVPP2_RX_USE_PSEUDO_FOR_CSUM_MASK |
1462 MVPP2_RX_LOW_LATENCY_PKT_SIZE(256));
1463
1464 /* Enable Rx cache snoop */
09f83975 1465 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
3f518509
MW		 1466 queue = port->rxqs[lrxq]->id;
1467 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
1468 val |= MVPP2_SNOOP_PKT_SIZE_MASK |
1469 MVPP2_SNOOP_BUF_HDR_MASK;
1470 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val);
1471 }
1472
1473 /* At default, mask all interrupts to all present cpus */
1474 mvpp2_interrupts_disable(port);
1475}
1476
1477/* Enable/disable receiving packets */
1478static void mvpp2_ingress_enable(struct mvpp2_port *port)
1479{
1480 u32 val;
1481 int lrxq, queue;
1482
09f83975 1483 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
3f518509
MW		 1484 queue = port->rxqs[lrxq]->id;
1485 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
1486 val &= ~MVPP2_RXQ_DISABLE_MASK;
1487 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val);
1488 }
1489}
1490
1491static void mvpp2_ingress_disable(struct mvpp2_port *port)
1492{
1493 u32 val;
1494 int lrxq, queue;
1495
09f83975 1496 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) {
3f518509
MW		 1497 queue = port->rxqs[lrxq]->id;
1498 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue));
1499 val |= MVPP2_RXQ_DISABLE_MASK;
1500 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val);
1501 }
1502}
1503
1504/* Enable transmit via physical egress queue
1505 * - HW starts take descriptors from DRAM
1506 */
1507static void mvpp2_egress_enable(struct mvpp2_port *port)
1508{
1509 u32 qmap;
1510 int queue;
1511 int tx_port_num = mvpp2_egress_port(port);
1512
1513 /* Enable all initialized TXs. */
1514 qmap = 0;
09f83975 1515 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 1516 struct mvpp2_tx_queue *txq = port->txqs[queue];
1517
dbbb2f03 1518 if (txq->descs)
3f518509
MW		 1519 qmap |= (1 << queue);
1520 }
1521
1522 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
1523 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG, qmap);
1524}
1525
1526/* Disable transmit via physical egress queue
1527 * - HW doesn't take descriptors from DRAM
1528 */
1529static void mvpp2_egress_disable(struct mvpp2_port *port)
1530{
1531 u32 reg_data;
1532 int delay;
1533 int tx_port_num = mvpp2_egress_port(port);
1534
1535 /* Issue stop command for active channels only */
1536 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
1537 reg_data = (mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG)) &
1538 MVPP2_TXP_SCHED_ENQ_MASK;
1539 if (reg_data != 0)
1540 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG,
1541 (reg_data << MVPP2_TXP_SCHED_DISQ_OFFSET));
1542
1543 /* Wait for all Tx activity to terminate. */
1544 delay = 0;
1545 do {
1546 if (delay >= MVPP2_TX_DISABLE_TIMEOUT_MSEC) {
1547 netdev_warn(port->dev,
1548 "Tx stop timed out, status=0x%08x\n",
1549 reg_data);
1550 break;
1551 }
1552 mdelay(1);
1553 delay++;
1554
1555 /* Check port TX Command register that all
1556 * Tx queues are stopped
1557 */
1558 reg_data = mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG);
1559 } while (reg_data & MVPP2_TXP_SCHED_ENQ_MASK);
1560}
1561
1562/* Rx descriptors helper methods */
1563
1564/* Get number of Rx descriptors occupied by received packets */
1565static inline int
1566mvpp2_rxq_received(struct mvpp2_port *port, int rxq_id)
1567{
1568 u32 val = mvpp2_read(port->priv, MVPP2_RXQ_STATUS_REG(rxq_id));
1569
1570 return val & MVPP2_RXQ_OCCUPIED_MASK;
1571}
1572
1573/* Update Rx queue status with the number of occupied and available
1574 * Rx descriptor slots.
1575 */
1576static inline void
1577mvpp2_rxq_status_update(struct mvpp2_port *port, int rxq_id,
1578 int used_count, int free_count)
1579{
1580 /* Decrement the number of used descriptors and increment count
1581 * increment the number of free descriptors.
1582 */
1583 u32 val = used_count | (free_count << MVPP2_RXQ_NUM_NEW_OFFSET);
1584
1585 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_UPDATE_REG(rxq_id), val);
1586}
1587
1588/* Get pointer to next RX descriptor to be processed by SW */
1589static inline struct mvpp2_rx_desc *
1590mvpp2_rxq_next_desc_get(struct mvpp2_rx_queue *rxq)
1591{
1592 int rx_desc = rxq->next_desc_to_proc;
1593
1594 rxq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(rxq, rx_desc);
1595 prefetch(rxq->descs + rxq->next_desc_to_proc);
1596 return rxq->descs + rx_desc;
1597}
1598
1599/* Set rx queue offset */
1600static void mvpp2_rxq_offset_set(struct mvpp2_port *port,
1601 int prxq, int offset)
1602{
1603 u32 val;
1604
1605 /* Convert offset from bytes to units of 32 bytes */
1606 offset = offset >> 5;
1607
1608 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq));
1609 val &= ~MVPP2_RXQ_PACKET_OFFSET_MASK;
1610
1611 /* Offset is in */
1612 val |= ((offset << MVPP2_RXQ_PACKET_OFFSET_OFFS) &
1613 MVPP2_RXQ_PACKET_OFFSET_MASK);
1614
1615 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val);
1616}
1617
3f518509
MW		 1618/* Tx descriptors helper methods */
1619
3f518509
MW		 1620/* Get pointer to next Tx descriptor to be processed (send) by HW */
1621static struct mvpp2_tx_desc *
1622mvpp2_txq_next_desc_get(struct mvpp2_tx_queue *txq)
1623{
1624 int tx_desc = txq->next_desc_to_proc;
1625
1626 txq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(txq, tx_desc);
1627 return txq->descs + tx_desc;
1628}
1629
e0af22d9
TP		 1630/* Update HW with number of aggregated Tx descriptors to be sent
1631 *
1632 * Called only from mvpp2_tx(), so migration is disabled, using
1633 * smp_processor_id() is OK.
1634 */
3f518509
MW		 1635static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending)
1636{
1637 /* aggregated access - relevant TXQ number is written in TX desc */
1068549c 1638 mvpp2_thread_write(port->priv,
e531f767 1639 mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
a786841d 1640 MVPP2_AGGR_TXQ_UPDATE_REG, pending);
3f518509
MW		 1641}
1642
3f518509
MW		 1643/* Check if there are enough free descriptors in aggregated txq.
1644 * If not, update the number of occupied descriptors and repeat the check.
e0af22d9
TP		 1645 *
1646 * Called only from mvpp2_tx(), so migration is disabled, using
1647 * smp_processor_id() is OK.
3f518509 1648 */
e531f767 1649static int mvpp2_aggr_desc_num_check(struct mvpp2_port *port,
3f518509
MW		 1650 struct mvpp2_tx_queue *aggr_txq, int num)
1651{
02856a3b 1652 if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE) {
3f518509 1653 /* Update number of occupied aggregated Tx descriptors */
e531f767
AT		 1654 unsigned int thread =
1655 mvpp2_cpu_to_thread(port->priv, smp_processor_id());
1656 u32 val = mvpp2_read_relaxed(port->priv,
543ec376 1657 MVPP2_AGGR_TXQ_STATUS_REG(thread));
3f518509
MW		 1658
1659 aggr_txq->count = val & MVPP2_AGGR_TXQ_PENDING_MASK;
3f518509 1660
914365f1
YM		 1661 if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE)
1662 return -ENOMEM;
1663 }
3f518509
MW		 1664 return 0;
1665}
1666
e0af22d9
TP		 1667/* Reserved Tx descriptors allocation request
1668 *
1669 * Called only from mvpp2_txq_reserved_desc_num_proc(), itself called
1670 * only by mvpp2_tx(), so migration is disabled, using
1671 * smp_processor_id() is OK.
1672 */
e531f767 1673static int mvpp2_txq_alloc_reserved_desc(struct mvpp2_port *port,
3f518509
MW		 1674 struct mvpp2_tx_queue *txq, int num)
1675{
e531f767
AT		 1676 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
1677 struct mvpp2 *priv = port->priv;
3f518509
MW		 1678 u32 val;
1679
1680 val = (txq->id << MVPP2_TXQ_RSVD_REQ_Q_OFFSET) | num;
1068549c 1681 mvpp2_thread_write_relaxed(priv, thread, MVPP2_TXQ_RSVD_REQ_REG, val);
3f518509 1682
1068549c 1683 val = mvpp2_thread_read_relaxed(priv, thread, MVPP2_TXQ_RSVD_RSLT_REG);
3f518509
MW		 1684
1685 return val & MVPP2_TXQ_RSVD_RSLT_MASK;
1686}
1687
1688/* Check if there are enough reserved descriptors for transmission.
1689 * If not, request chunk of reserved descriptors and check again.
1690 */
074c74df 1691static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2_port *port,
3f518509
MW		 1692 struct mvpp2_tx_queue *txq,
1693 struct mvpp2_txq_pcpu *txq_pcpu,
1694 int num)
1695{
850623b3 1696 int req, desc_count;
074c74df 1697 unsigned int thread;
3f518509
MW		 1698
1699 if (txq_pcpu->reserved_num >= num)
1700 return 0;
1701
1702 /* Not enough descriptors reserved! Update the reserved descriptor
1703 * count and check again.
1704 */
1705
1706 desc_count = 0;
1707 /* Compute total of used descriptors */
e531f767 1708 for (thread = 0; thread < port->priv->nthreads; thread++) {
3f518509
MW		 1709 struct mvpp2_txq_pcpu *txq_pcpu_aux;
1710
074c74df 1711 txq_pcpu_aux = per_cpu_ptr(txq->pcpu, thread);
3f518509
MW		 1712 desc_count += txq_pcpu_aux->count;
1713 desc_count += txq_pcpu_aux->reserved_num;
1714 }
1715
1716 req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num);
1717 desc_count += req;
1718
1719 if (desc_count >
074c74df 1720 (txq->size - (MVPP2_MAX_THREADS * MVPP2_CPU_DESC_CHUNK)))
3f518509
MW		 1721 return -ENOMEM;
1722
e531f767 1723 txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req);
3f518509 1724
a3302baa 1725 /* OK, the descriptor could have been updated: check again. */
3f518509
MW		 1726 if (txq_pcpu->reserved_num < num)
1727 return -ENOMEM;
1728 return 0;
1729}
1730
1731/* Release the last allocated Tx descriptor. Useful to handle DMA
1732 * mapping failures in the Tx path.
1733 */
1734static void mvpp2_txq_desc_put(struct mvpp2_tx_queue *txq)
1735{
1736 if (txq->next_desc_to_proc == 0)
1737 txq->next_desc_to_proc = txq->last_desc - 1;
1738 else
1739 txq->next_desc_to_proc--;
1740}
1741
1742/* Set Tx descriptors fields relevant for CSUM calculation */
35f3625c 1743static u32 mvpp2_txq_desc_csum(int l3_offs, __be16 l3_proto,
3f518509
MW		 1744 int ip_hdr_len, int l4_proto)
1745{
1746 u32 command;
1747
1748 /* fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk,
1749 * G_L4_chk, L4_type required only for checksum calculation
1750 */
1751 command = (l3_offs << MVPP2_TXD_L3_OFF_SHIFT);
1752 command |= (ip_hdr_len << MVPP2_TXD_IP_HLEN_SHIFT);
1753 command |= MVPP2_TXD_IP_CSUM_DISABLE;
1754
dc734dbe 1755 if (l3_proto == htons(ETH_P_IP)) {
3f518509
MW		 1756 command &= ~MVPP2_TXD_IP_CSUM_DISABLE; /* enable IPv4 csum */
1757 command &= ~MVPP2_TXD_L3_IP6; /* enable IPv4 */
1758 } else {
1759 command |= MVPP2_TXD_L3_IP6; /* enable IPv6 */
1760 }
1761
1762 if (l4_proto == IPPROTO_TCP) {
1763 command &= ~MVPP2_TXD_L4_UDP; /* enable TCP */
1764 command &= ~MVPP2_TXD_L4_CSUM_FRAG; /* generate L4 csum */
1765 } else if (l4_proto == IPPROTO_UDP) {
1766 command |= MVPP2_TXD_L4_UDP; /* enable UDP */
1767 command &= ~MVPP2_TXD_L4_CSUM_FRAG; /* generate L4 csum */
1768 } else {
1769 command |= MVPP2_TXD_L4_CSUM_NOT;
1770 }
1771
1772 return command;
1773}
1774
1775/* Get number of sent descriptors and decrement counter.
1776 * The number of sent descriptors is returned.
543ec376 1777 * Per-thread access
e0af22d9
TP		 1778 *
1779 * Called only from mvpp2_txq_done(), called from mvpp2_tx()
1780 * (migration disabled) and from the TX completion tasklet (migration
1781 * disabled) so using smp_processor_id() is OK.
3f518509
MW		 1782 */
1783static inline int mvpp2_txq_sent_desc_proc(struct mvpp2_port *port,
1784 struct mvpp2_tx_queue *txq)
1785{
1786 u32 val;
1787
1788 /* Reading status reg resets transmitted descriptor counter */
1068549c 1789 val = mvpp2_thread_read_relaxed(port->priv,
e531f767 1790 mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
cdcfeb0f 1791 MVPP2_TXQ_SENT_REG(txq->id));
3f518509
MW		 1792
1793 return (val & MVPP2_TRANSMITTED_COUNT_MASK) >>
1794 MVPP2_TRANSMITTED_COUNT_OFFSET;
1795}
1796
e0af22d9
TP		 1797/* Called through on_each_cpu(), so runs on all CPUs, with migration
1798 * disabled, therefore using smp_processor_id() is OK.
1799 */
3f518509
MW		 1800static void mvpp2_txq_sent_counter_clear(void *arg)
1801{
1802 struct mvpp2_port *port = arg;
1803 int queue;
1804
e531f767
AT		 1805 /* If the thread isn't used, don't do anything */
1806 if (smp_processor_id() > port->priv->nthreads)
1807 return;
1808
09f83975 1809 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 1810 int id = port->txqs[queue]->id;
1811
1068549c 1812 mvpp2_thread_read(port->priv,
e531f767 1813 mvpp2_cpu_to_thread(port->priv, smp_processor_id()),
a786841d 1814 MVPP2_TXQ_SENT_REG(id));
3f518509
MW		 1815 }
1816}
1817
1818/* Set max sizes for Tx queues */
1819static void mvpp2_txp_max_tx_size_set(struct mvpp2_port *port)
1820{
1821 u32 val, size, mtu;
1822 int txq, tx_port_num;
1823
1824 mtu = port->pkt_size * 8;
1825 if (mtu > MVPP2_TXP_MTU_MAX)
1826 mtu = MVPP2_TXP_MTU_MAX;
1827
1828 /* WA for wrong Token bucket update: Set MTU value = 3*real MTU value */
1829 mtu = 3 * mtu;
1830
1831 /* Indirect access to registers */
1832 tx_port_num = mvpp2_egress_port(port);
1833 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
1834
1835 /* Set MTU */
1836 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_MTU_REG);
1837 val &= ~MVPP2_TXP_MTU_MAX;
1838 val |= mtu;
1839 mvpp2_write(port->priv, MVPP2_TXP_SCHED_MTU_REG, val);
1840
1841 /* TXP token size and all TXQs token size must be larger that MTU */
1842 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG);
1843 size = val & MVPP2_TXP_TOKEN_SIZE_MAX;
1844 if (size < mtu) {
1845 size = mtu;
1846 val &= ~MVPP2_TXP_TOKEN_SIZE_MAX;
1847 val |= size;
1848 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val);
1849 }
1850
09f83975 1851 for (txq = 0; txq < port->ntxqs; txq++) {
3f518509
MW		 1852 val = mvpp2_read(port->priv,
1853 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq));
1854 size = val & MVPP2_TXQ_TOKEN_SIZE_MAX;
1855
1856 if (size < mtu) {
1857 size = mtu;
1858 val &= ~MVPP2_TXQ_TOKEN_SIZE_MAX;
1859 val |= size;
1860 mvpp2_write(port->priv,
1861 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq),
1862 val);
1863 }
1864 }
1865}
1866
1867/* Set the number of packets that will be received before Rx interrupt
1868 * will be generated by HW.
1869 */
1870static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port,
d63f9e41 1871 struct mvpp2_rx_queue *rxq)
3f518509 1872{
e531f767 1873 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
a786841d 1874
f8b0d5f8
TP		 1875 if (rxq->pkts_coal > MVPP2_OCCUPIED_THRESH_MASK)
1876 rxq->pkts_coal = MVPP2_OCCUPIED_THRESH_MASK;
3f518509 1877
1068549c
AT		 1878 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id);
1879 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_THRESH_REG,
a786841d 1880 rxq->pkts_coal);
a704bb5c
TP		 1881
1882 put_cpu();
3f518509
MW		 1883}
1884
213f428f
TP		 1885/* For some reason in the LSP this is done on each CPU. Why ? */
1886static void mvpp2_tx_pkts_coal_set(struct mvpp2_port *port,
1887 struct mvpp2_tx_queue *txq)
1888{
e531f767 1889 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
213f428f
TP		 1890 u32 val;
1891
1892 if (txq->done_pkts_coal > MVPP2_TXQ_THRESH_MASK)
1893 txq->done_pkts_coal = MVPP2_TXQ_THRESH_MASK;
1894
1895 val = (txq->done_pkts_coal << MVPP2_TXQ_THRESH_OFFSET);
1068549c
AT		 1896 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
1897 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_THRESH_REG, val);
213f428f
TP		 1898
1899 put_cpu();
1900}
1901
ab42676a
TP		 1902static u32 mvpp2_usec_to_cycles(u32 usec, unsigned long clk_hz)
1903{
1904 u64 tmp = (u64)clk_hz * usec;
1905
1906 do_div(tmp, USEC_PER_SEC);
1907
1908 return tmp > U32_MAX ? U32_MAX : tmp;
1909}
1910
1911static u32 mvpp2_cycles_to_usec(u32 cycles, unsigned long clk_hz)
1912{
1913 u64 tmp = (u64)cycles * USEC_PER_SEC;
1914
1915 do_div(tmp, clk_hz);
1916
1917 return tmp > U32_MAX ? U32_MAX : tmp;
1918}
1919
3f518509
MW		 1920/* Set the time delay in usec before Rx interrupt */
1921static void mvpp2_rx_time_coal_set(struct mvpp2_port *port,
d63f9e41 1922 struct mvpp2_rx_queue *rxq)
3f518509 1923{
ab42676a
TP		 1924 unsigned long freq = port->priv->tclk;
1925 u32 val = mvpp2_usec_to_cycles(rxq->time_coal, freq);
1926
1927 if (val > MVPP2_MAX_ISR_RX_THRESHOLD) {
1928 rxq->time_coal =
1929 mvpp2_cycles_to_usec(MVPP2_MAX_ISR_RX_THRESHOLD, freq);
1930
1931 /* re-evaluate to get actual register value */
1932 val = mvpp2_usec_to_cycles(rxq->time_coal, freq);
1933 }
3f518509 1934
3f518509 1935 mvpp2_write(port->priv, MVPP2_ISR_RX_THRESHOLD_REG(rxq->id), val);
3f518509
MW		 1936}
1937
213f428f
TP		 1938static void mvpp2_tx_time_coal_set(struct mvpp2_port *port)
1939{
1940 unsigned long freq = port->priv->tclk;
1941 u32 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq);
1942
1943 if (val > MVPP2_MAX_ISR_TX_THRESHOLD) {
1944 port->tx_time_coal =
1945 mvpp2_cycles_to_usec(MVPP2_MAX_ISR_TX_THRESHOLD, freq);
1946
1947 /* re-evaluate to get actual register value */
1948 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq);
1949 }
1950
1951 mvpp2_write(port->priv, MVPP2_ISR_TX_THRESHOLD_REG(port->id), val);
1952}
1953
3f518509
MW		 1954/* Free Tx queue skbuffs */
1955static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
1956 struct mvpp2_tx_queue *txq,
1957 struct mvpp2_txq_pcpu *txq_pcpu, int num)
1958{
1959 int i;
1960
1961 for (i = 0; i < num; i++) {
8354491c
TP		 1962 struct mvpp2_txq_pcpu_buf *tx_buf =
1963 txq_pcpu->buffs + txq_pcpu->txq_get_index;
3f518509 1964
20920267
AT		 1965 if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma))
1966 dma_unmap_single(port->dev->dev.parent, tx_buf->dma,
1967 tx_buf->size, DMA_TO_DEVICE);
36fb7435
TP		 1968 if (tx_buf->skb)
1969 dev_kfree_skb_any(tx_buf->skb);
1970
1971 mvpp2_txq_inc_get(txq_pcpu);
3f518509
MW		 1972 }
1973}
1974
1975static inline struct mvpp2_rx_queue *mvpp2_get_rx_queue(struct mvpp2_port *port,
1976 u32 cause)
1977{
1978 int queue = fls(cause) - 1;
1979
1980 return port->rxqs[queue];
1981}
1982
1983static inline struct mvpp2_tx_queue *mvpp2_get_tx_queue(struct mvpp2_port *port,
1984 u32 cause)
1985{
edc660fa 1986 int queue = fls(cause) - 1;
3f518509
MW		 1987
1988 return port->txqs[queue];
1989}
1990
1991/* Handle end of transmission */
1992static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
1993 struct mvpp2_txq_pcpu *txq_pcpu)
1994{
1995 struct netdev_queue *nq = netdev_get_tx_queue(port->dev, txq->log_id);
1996 int tx_done;
1997
e531f767 1998 if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id()))
3f518509
MW		 1999 netdev_err(port->dev, "wrong cpu on the end of Tx processing\n");
2000
2001 tx_done = mvpp2_txq_sent_desc_proc(port, txq);
2002 if (!tx_done)
2003 return;
2004 mvpp2_txq_bufs_free(port, txq, txq_pcpu, tx_done);
2005
2006 txq_pcpu->count -= tx_done;
2007
2008 if (netif_tx_queue_stopped(nq))
1d17db08 2009 if (txq_pcpu->count <= txq_pcpu->wake_threshold)
3f518509
MW		 2010 netif_tx_wake_queue(nq);
2011}
2012
213f428f 2013static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause,
543ec376 2014 unsigned int thread)
edc660fa
MW		 2015{
2016 struct mvpp2_tx_queue *txq;
2017 struct mvpp2_txq_pcpu *txq_pcpu;
2018 unsigned int tx_todo = 0;
2019
2020 while (cause) {
2021 txq = mvpp2_get_tx_queue(port, cause);
2022 if (!txq)
2023 break;
2024
543ec376 2025 txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
edc660fa
MW		 2026
2027 if (txq_pcpu->count) {
2028 mvpp2_txq_done(port, txq, txq_pcpu);
2029 tx_todo += txq_pcpu->count;
2030 }
2031
2032 cause &= ~(1 << txq->log_id);
2033 }
2034 return tx_todo;
2035}
2036
3f518509
MW		 2037/* Rx/Tx queue initialization/cleanup methods */
2038
2039/* Allocate and initialize descriptors for aggr TXQ */
2040static int mvpp2_aggr_txq_init(struct platform_device *pdev,
850623b3 2041 struct mvpp2_tx_queue *aggr_txq,
543ec376 2042 unsigned int thread, struct mvpp2 *priv)
3f518509 2043{
b02f31fb
TP		 2044 u32 txq_dma;
2045
3f518509 2046 /* Allocate memory for TX descriptors */
750afb08
LC		 2047 aggr_txq->descs = dma_alloc_coherent(&pdev->dev,
2048 MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
2049 &aggr_txq->descs_dma, GFP_KERNEL);
3f518509
MW		 2050 if (!aggr_txq->descs)
2051 return -ENOMEM;
2052
02856a3b 2053 aggr_txq->last_desc = MVPP2_AGGR_TXQ_SIZE - 1;
3f518509
MW		 2054
2055 /* Aggr TXQ no reset WA */
2056 aggr_txq->next_desc_to_proc = mvpp2_read(priv,
543ec376 2057 MVPP2_AGGR_TXQ_INDEX_REG(thread));
3f518509 2058
b02f31fb
TP		 2059 /* Set Tx descriptors queue starting address indirect
2060 * access
2061 */
2062 if (priv->hw_version == MVPP21)
2063 txq_dma = aggr_txq->descs_dma;
2064 else
2065 txq_dma = aggr_txq->descs_dma >>
2066 MVPP22_AGGR_TXQ_DESC_ADDR_OFFS;
2067
543ec376
AT		 2068 mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_ADDR_REG(thread), txq_dma);
2069 mvpp2_write(priv, MVPP2_AGGR_TXQ_DESC_SIZE_REG(thread),
85affd7e 2070 MVPP2_AGGR_TXQ_SIZE);
3f518509
MW		 2071
2072 return 0;
2073}
2074
2075/* Create a specified Rx queue */
2076static int mvpp2_rxq_init(struct mvpp2_port *port,
2077 struct mvpp2_rx_queue *rxq)
2078
2079{
543ec376 2080 unsigned int thread;
b02f31fb
TP		 2081 u32 rxq_dma;
2082
3f518509
MW		 2083 rxq->size = port->rx_ring_size;
2084
2085 /* Allocate memory for RX descriptors */
2086 rxq->descs = dma_alloc_coherent(port->dev->dev.parent,
2087 rxq->size * MVPP2_DESC_ALIGNED_SIZE,
20396136 2088 &rxq->descs_dma, GFP_KERNEL);
3f518509
MW		 2089 if (!rxq->descs)
2090 return -ENOMEM;
2091
3f518509
MW		 2092 rxq->last_desc = rxq->size - 1;
2093
2094 /* Zero occupied and non-occupied counters - direct access */
2095 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
2096
2097 /* Set Rx descriptors queue starting address - indirect access */
e531f767 2098 thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
1068549c 2099 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id);
b02f31fb
TP		 2100 if (port->priv->hw_version == MVPP21)
2101 rxq_dma = rxq->descs_dma;
2102 else
2103 rxq_dma = rxq->descs_dma >> MVPP22_DESC_ADDR_OFFS;
1068549c
AT		 2104 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma);
2105 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, rxq->size);
2106 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_INDEX_REG, 0);
a704bb5c 2107 put_cpu();
3f518509
MW		 2108
2109 /* Set Offset */
2110 mvpp2_rxq_offset_set(port, rxq->id, NET_SKB_PAD);
2111
2112 /* Set coalescing pkts and time */
d63f9e41
TP		 2113 mvpp2_rx_pkts_coal_set(port, rxq);
2114 mvpp2_rx_time_coal_set(port, rxq);
3f518509
MW		 2115
2116 /* Add number of descriptors ready for receiving packets */
2117 mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size);
2118
2119 return 0;
2120}
2121
2122/* Push packets received by the RXQ to BM pool */
2123static void mvpp2_rxq_drop_pkts(struct mvpp2_port *port,
2124 struct mvpp2_rx_queue *rxq)
2125{
2126 int rx_received, i;
2127
2128 rx_received = mvpp2_rxq_received(port, rxq->id);
2129 if (!rx_received)
2130 return;
2131
2132 for (i = 0; i < rx_received; i++) {
2133 struct mvpp2_rx_desc *rx_desc = mvpp2_rxq_next_desc_get(rxq);
56b8aae9
TP		 2134 u32 status = mvpp2_rxdesc_status_get(port, rx_desc);
2135 int pool;
2136
2137 pool = (status & MVPP2_RXD_BM_POOL_ID_MASK) >>
2138 MVPP2_RXD_BM_POOL_ID_OFFS;
3f518509 2139
7d7627ba 2140 mvpp2_bm_pool_put(port, pool,
ac3dd277
TP		 2141 mvpp2_rxdesc_dma_addr_get(port, rx_desc),
2142 mvpp2_rxdesc_cookie_get(port, rx_desc));
3f518509
MW		 2143 }
2144 mvpp2_rxq_status_update(port, rxq->id, rx_received, rx_received);
2145}
2146
2147/* Cleanup Rx queue */
2148static void mvpp2_rxq_deinit(struct mvpp2_port *port,
2149 struct mvpp2_rx_queue *rxq)
2150{
543ec376 2151 unsigned int thread;
a786841d 2152
3f518509
MW		 2153 mvpp2_rxq_drop_pkts(port, rxq);
2154
2155 if (rxq->descs)
2156 dma_free_coherent(port->dev->dev.parent,
2157 rxq->size * MVPP2_DESC_ALIGNED_SIZE,
2158 rxq->descs,
20396136 2159 rxq->descs_dma);
3f518509
MW		 2160
2161 rxq->descs = NULL;
2162 rxq->last_desc = 0;
2163 rxq->next_desc_to_proc = 0;
20396136 2164 rxq->descs_dma = 0;
3f518509
MW		 2165
2166 /* Clear Rx descriptors queue starting address and size;
2167 * free descriptor number
2168 */
2169 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0);
e531f767 2170 thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
1068549c
AT		 2171 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id);
2172 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, 0);
2173 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, 0);
a704bb5c 2174 put_cpu();
3f518509
MW		 2175}
2176
2177/* Create and initialize a Tx queue */
2178static int mvpp2_txq_init(struct mvpp2_port *port,
2179 struct mvpp2_tx_queue *txq)
2180{
2181 u32 val;
074c74df 2182 unsigned int thread;
850623b3 2183 int desc, desc_per_txq, tx_port_num;
3f518509
MW		 2184 struct mvpp2_txq_pcpu *txq_pcpu;
2185
2186 txq->size = port->tx_ring_size;
2187
2188 /* Allocate memory for Tx descriptors */
2189 txq->descs = dma_alloc_coherent(port->dev->dev.parent,
2190 txq->size * MVPP2_DESC_ALIGNED_SIZE,
20396136 2191 &txq->descs_dma, GFP_KERNEL);
3f518509
MW		 2192 if (!txq->descs)
2193 return -ENOMEM;
2194
3f518509
MW		 2195 txq->last_desc = txq->size - 1;
2196
2197 /* Set Tx descriptors queue starting address - indirect access */
e531f767 2198 thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
1068549c
AT		 2199 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
2200 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG,
a786841d 2201 txq->descs_dma);
1068549c 2202 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG,
a786841d 2203 txq->size & MVPP2_TXQ_DESC_SIZE_MASK);
1068549c
AT		 2204 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_INDEX_REG, 0);
2205 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_RSVD_CLR_REG,
a786841d 2206 txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET);
1068549c 2207 val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PENDING_REG);
3f518509 2208 val &= ~MVPP2_TXQ_PENDING_MASK;
1068549c 2209 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PENDING_REG, val);
3f518509
MW		 2210
2211 /* Calculate base address in prefetch buffer. We reserve 16 descriptors
2212 * for each existing TXQ.
2213 * TCONTS for PON port must be continuous from 0 to MVPP2_MAX_TCONT
a3302baa 2214 * GBE ports assumed to be continuous from 0 to MVPP2_MAX_PORTS
3f518509
MW		 2215 */
2216 desc_per_txq = 16;
2217 desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) +
2218 (txq->log_id * desc_per_txq);
2219
1068549c 2220 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG,
a786841d
TP		 2221 MVPP2_PREF_BUF_PTR(desc) | MVPP2_PREF_BUF_SIZE_16 |
2222 MVPP2_PREF_BUF_THRESH(desc_per_txq / 2));
a704bb5c 2223 put_cpu();
3f518509
MW		 2224
2225 /* WRR / EJP configuration - indirect access */
2226 tx_port_num = mvpp2_egress_port(port);
2227 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num);
2228
2229 val = mvpp2_read(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id));
2230 val &= ~MVPP2_TXQ_REFILL_PERIOD_ALL_MASK;
2231 val |= MVPP2_TXQ_REFILL_PERIOD_MASK(1);
2232 val |= MVPP2_TXQ_REFILL_TOKENS_ALL_MASK;
2233 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id), val);
2234
2235 val = MVPP2_TXQ_TOKEN_SIZE_MAX;
2236 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq->log_id),
2237 val);
2238
e531f767 2239 for (thread = 0; thread < port->priv->nthreads; thread++) {
074c74df 2240 txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
3f518509 2241 txq_pcpu->size = txq->size;
02c91ece
ME		 2242 txq_pcpu->buffs = kmalloc_array(txq_pcpu->size,
2243 sizeof(*txq_pcpu->buffs),
2244 GFP_KERNEL);
8354491c 2245 if (!txq_pcpu->buffs)
ba2d8d88 2246 return -ENOMEM;
3f518509
MW		 2247
2248 txq_pcpu->count = 0;
2249 txq_pcpu->reserved_num = 0;
2250 txq_pcpu->txq_put_index = 0;
2251 txq_pcpu->txq_get_index = 0;
b70d4a51 2252 txq_pcpu->tso_headers = NULL;
186cd4d4 2253
1d17db08
AT		 2254 txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS;
2255 txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2;
2256
186cd4d4
AT		 2257 txq_pcpu->tso_headers =
2258 dma_alloc_coherent(port->dev->dev.parent,
822eaf7c 2259 txq_pcpu->size * TSO_HEADER_SIZE,
186cd4d4
AT		 2260 &txq_pcpu->tso_headers_dma,
2261 GFP_KERNEL);
2262 if (!txq_pcpu->tso_headers)
ba2d8d88 2263 return -ENOMEM;
3f518509
MW		 2264 }
2265
2266 return 0;
2267}
2268
2269/* Free allocated TXQ resources */
2270static void mvpp2_txq_deinit(struct mvpp2_port *port,
2271 struct mvpp2_tx_queue *txq)
2272{
2273 struct mvpp2_txq_pcpu *txq_pcpu;
074c74df 2274 unsigned int thread;
3f518509 2275
e531f767 2276 for (thread = 0; thread < port->priv->nthreads; thread++) {
074c74df 2277 txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
8354491c 2278 kfree(txq_pcpu->buffs);
186cd4d4 2279
b70d4a51
AT		 2280 if (txq_pcpu->tso_headers)
2281 dma_free_coherent(port->dev->dev.parent,
2282 txq_pcpu->size * TSO_HEADER_SIZE,
2283 txq_pcpu->tso_headers,
2284 txq_pcpu->tso_headers_dma);
2285
2286 txq_pcpu->tso_headers = NULL;
3f518509
MW		 2287 }
2288
2289 if (txq->descs)
2290 dma_free_coherent(port->dev->dev.parent,
2291 txq->size * MVPP2_DESC_ALIGNED_SIZE,
20396136 2292 txq->descs, txq->descs_dma);
3f518509
MW		 2293
2294 txq->descs = NULL;
2295 txq->last_desc = 0;
2296 txq->next_desc_to_proc = 0;
20396136 2297 txq->descs_dma = 0;
3f518509
MW		 2298
2299 /* Set minimum bandwidth for disabled TXQs */
2300 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq->id), 0);
2301
2302 /* Set Tx descriptors queue starting address and size */
e531f767 2303 thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
1068549c
AT		 2304 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
2305 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG, 0);
2306 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG, 0);
a704bb5c 2307 put_cpu();
3f518509
MW		 2308}
2309
2310/* Cleanup Tx ports */
2311static void mvpp2_txq_clean(struct mvpp2_port *port, struct mvpp2_tx_queue *txq)
2312{
2313 struct mvpp2_txq_pcpu *txq_pcpu;
850623b3 2314 int delay, pending;
e531f767 2315 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu());
3f518509
MW		 2316 u32 val;
2317
1068549c
AT		 2318 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id);
2319 val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG);
3f518509 2320 val |= MVPP2_TXQ_DRAIN_EN_MASK;
1068549c 2321 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val);
3f518509
MW		 2322
2323 /* The napi queue has been stopped so wait for all packets
2324 * to be transmitted.
2325 */
2326 delay = 0;
2327 do {
2328 if (delay >= MVPP2_TX_PENDING_TIMEOUT_MSEC) {
2329 netdev_warn(port->dev,
2330 "port %d: cleaning queue %d timed out\n",
2331 port->id, txq->log_id);
2332 break;
2333 }
2334 mdelay(1);
2335 delay++;
2336
1068549c 2337 pending = mvpp2_thread_read(port->priv, thread,
a786841d
TP		 2338 MVPP2_TXQ_PENDING_REG);
2339 pending &= MVPP2_TXQ_PENDING_MASK;
3f518509
MW		 2340 } while (pending);
2341
2342 val &= ~MVPP2_TXQ_DRAIN_EN_MASK;
1068549c 2343 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val);
a704bb5c 2344 put_cpu();
3f518509 2345
e531f767 2346 for (thread = 0; thread < port->priv->nthreads; thread++) {
074c74df 2347 txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
3f518509
MW		 2348
2349 /* Release all packets */
2350 mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count);
2351
2352 /* Reset queue */
2353 txq_pcpu->count = 0;
2354 txq_pcpu->txq_put_index = 0;
2355 txq_pcpu->txq_get_index = 0;
2356 }
2357}
2358
2359/* Cleanup all Tx queues */
2360static void mvpp2_cleanup_txqs(struct mvpp2_port *port)
2361{
2362 struct mvpp2_tx_queue *txq;
2363 int queue;
2364 u32 val;
2365
2366 val = mvpp2_read(port->priv, MVPP2_TX_PORT_FLUSH_REG);
2367
2368 /* Reset Tx ports and delete Tx queues */
2369 val |= MVPP2_TX_PORT_FLUSH_MASK(port->id);
2370 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val);
2371
09f83975 2372 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 2373 txq = port->txqs[queue];
2374 mvpp2_txq_clean(port, txq);
2375 mvpp2_txq_deinit(port, txq);
2376 }
2377
2378 on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1);
2379
2380 val &= ~MVPP2_TX_PORT_FLUSH_MASK(port->id);
2381 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val);
2382}
2383
2384/* Cleanup all Rx queues */
2385static void mvpp2_cleanup_rxqs(struct mvpp2_port *port)
2386{
2387 int queue;
2388
09f83975 2389 for (queue = 0; queue < port->nrxqs; queue++)
3f518509
MW		 2390 mvpp2_rxq_deinit(port, port->rxqs[queue]);
2391}
2392
2393/* Init all Rx queues for port */
2394static int mvpp2_setup_rxqs(struct mvpp2_port *port)
2395{
2396 int queue, err;
2397
09f83975 2398 for (queue = 0; queue < port->nrxqs; queue++) {
3f518509
MW		 2399 err = mvpp2_rxq_init(port, port->rxqs[queue]);
2400 if (err)
2401 goto err_cleanup;
2402 }
2403 return 0;
2404
2405err_cleanup:
2406 mvpp2_cleanup_rxqs(port);
2407 return err;
2408}
2409
2410/* Init all tx queues for port */
2411static int mvpp2_setup_txqs(struct mvpp2_port *port)
2412{
2413 struct mvpp2_tx_queue *txq;
0d283ab5 2414 int queue, err, cpu;
3f518509 2415
09f83975 2416 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 2417 txq = port->txqs[queue];
2418 err = mvpp2_txq_init(port, txq);
2419 if (err)
2420 goto err_cleanup;
0d283ab5
MC		 2421
2422 /* Assign this queue to a CPU */
2423 cpu = queue % num_present_cpus();
2424 netif_set_xps_queue(port->dev, cpumask_of(cpu), queue);
3f518509
MW		 2425 }
2426
213f428f
TP		 2427 if (port->has_tx_irqs) {
2428 mvpp2_tx_time_coal_set(port);
2429 for (queue = 0; queue < port->ntxqs; queue++) {
2430 txq = port->txqs[queue];
2431 mvpp2_tx_pkts_coal_set(port, txq);
2432 }
2433 }
2434
3f518509
MW		 2435 on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1);
2436 return 0;
2437
2438err_cleanup:
2439 mvpp2_cleanup_txqs(port);
2440 return err;
2441}
2442
2443/* The callback for per-port interrupt */
2444static irqreturn_t mvpp2_isr(int irq, void *dev_id)
2445{
591f4cfa 2446 struct mvpp2_queue_vector *qv = dev_id;
3f518509 2447
591f4cfa 2448 mvpp2_qvec_interrupt_disable(qv);
3f518509 2449
591f4cfa 2450 napi_schedule(&qv->napi);
3f518509
MW		 2451
2452 return IRQ_HANDLED;
2453}
2454
fd3651b2
AT		 2455/* Per-port interrupt for link status changes */
2456static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
2457{
2458 struct mvpp2_port *port = (struct mvpp2_port *)dev_id;
2459 struct net_device *dev = port->dev;
2460 bool event = false, link = false;
2461 u32 val;
2462
2463 mvpp22_gop_mask_irq(port);
2464
2465 if (port->gop_id == 0 &&
2466 port->phy_interface == PHY_INTERFACE_MODE_10GKR) {
2467 val = readl(port->base + MVPP22_XLG_INT_STAT);
2468 if (val & MVPP22_XLG_INT_STAT_LINK) {
2469 event = true;
2470 val = readl(port->base + MVPP22_XLG_STATUS);
2471 if (val & MVPP22_XLG_STATUS_LINK_UP)
2472 link = true;
2473 }
2474 } else if (phy_interface_mode_is_rgmii(port->phy_interface) ||
d97c9f4a 2475 port->phy_interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 2476 port->phy_interface == PHY_INTERFACE_MODE_1000BASEX ||
2477 port->phy_interface == PHY_INTERFACE_MODE_2500BASEX) {
fd3651b2
AT		 2478 val = readl(port->base + MVPP22_GMAC_INT_STAT);
2479 if (val & MVPP22_GMAC_INT_STAT_LINK) {
2480 event = true;
2481 val = readl(port->base + MVPP2_GMAC_STATUS0);
2482 if (val & MVPP2_GMAC_STATUS0_LINK_UP)
2483 link = true;
2484 }
2485 }
2486
4bb04326
AT		 2487 if (port->phylink) {
2488 phylink_mac_change(port->phylink, link);
2489 goto handled;
2490 }
2491
fd3651b2
AT		 2492 if (!netif_running(dev) || !event)
2493 goto handled;
2494
2495 if (link) {
2496 mvpp2_interrupts_enable(port);
2497
2498 mvpp2_egress_enable(port);
2499 mvpp2_ingress_enable(port);
2500 netif_carrier_on(dev);
2501 netif_tx_wake_all_queues(dev);
2502 } else {
2503 netif_tx_stop_all_queues(dev);
2504 netif_carrier_off(dev);
2505 mvpp2_ingress_disable(port);
2506 mvpp2_egress_disable(port);
2507
2508 mvpp2_interrupts_disable(port);
2509 }
2510
2511handled:
2512 mvpp22_gop_unmask_irq(port);
2513 return IRQ_HANDLED;
2514}
2515
edc660fa
MW		 2516static void mvpp2_timer_set(struct mvpp2_port_pcpu *port_pcpu)
2517{
2518 ktime_t interval;
2519
2520 if (!port_pcpu->timer_scheduled) {
2521 port_pcpu->timer_scheduled = true;
8b0e1953 2522 interval = MVPP2_TXDONE_HRTIMER_PERIOD_NS;
edc660fa
MW		 2523 hrtimer_start(&port_pcpu->tx_done_timer, interval,
2524 HRTIMER_MODE_REL_PINNED);
2525 }
2526}
2527
2528static void mvpp2_tx_proc_cb(unsigned long data)
2529{
2530 struct net_device *dev = (struct net_device *)data;
2531 struct mvpp2_port *port = netdev_priv(dev);
074c74df 2532 struct mvpp2_port_pcpu *port_pcpu;
edc660fa
MW		 2533 unsigned int tx_todo, cause;
2534
074c74df 2535 port_pcpu = per_cpu_ptr(port->pcpu,
e531f767 2536 mvpp2_cpu_to_thread(port->priv, smp_processor_id()));
074c74df 2537
edc660fa
MW		 2538 if (!netif_running(dev))
2539 return;
2540 port_pcpu->timer_scheduled = false;
2541
2542 /* Process all the Tx queues */
09f83975 2543 cause = (1 << port->ntxqs) - 1;
074c74df 2544 tx_todo = mvpp2_tx_done(port, cause,
e531f767 2545 mvpp2_cpu_to_thread(port->priv, smp_processor_id()));
edc660fa
MW		 2546
2547 /* Set the timer in case not all the packets were processed */
2548 if (tx_todo)
2549 mvpp2_timer_set(port_pcpu);
2550}
2551
2552static enum hrtimer_restart mvpp2_hr_timer_cb(struct hrtimer *timer)
2553{
2554 struct mvpp2_port_pcpu *port_pcpu = container_of(timer,
2555 struct mvpp2_port_pcpu,
2556 tx_done_timer);
2557
2558 tasklet_schedule(&port_pcpu->tx_done_tasklet);
2559
2560 return HRTIMER_NORESTART;
2561}
2562
3f518509
MW		 2563/* Main RX/TX processing routines */
2564
2565/* Display more error info */
2566static void mvpp2_rx_error(struct mvpp2_port *port,
2567 struct mvpp2_rx_desc *rx_desc)
2568{
ac3dd277
TP		 2569 u32 status = mvpp2_rxdesc_status_get(port, rx_desc);
2570 size_t sz = mvpp2_rxdesc_size_get(port, rx_desc);
934e0f83 2571 char *err_str = NULL;
3f518509
MW		 2572
2573 switch (status & MVPP2_RXD_ERR_CODE_MASK) {
2574 case MVPP2_RXD_ERR_CRC:
934e0f83 2575 err_str = "crc";
3f518509
MW		 2576 break;
2577 case MVPP2_RXD_ERR_OVERRUN:
934e0f83 2578 err_str = "overrun";
3f518509
MW		 2579 break;
2580 case MVPP2_RXD_ERR_RESOURCE:
934e0f83 2581 err_str = "resource";
3f518509
MW		 2582 break;
2583 }
934e0f83
YM		 2584 if (err_str && net_ratelimit())
2585 netdev_err(port->dev,
2586 "bad rx status %08x (%s error), size=%zu\n",
2587 status, err_str, sz);
3f518509
MW		 2588}
2589
2590/* Handle RX checksum offload */
2591static void mvpp2_rx_csum(struct mvpp2_port *port, u32 status,
2592 struct sk_buff *skb)
2593{
2594 if (((status & MVPP2_RXD_L3_IP4) &&
2595 !(status & MVPP2_RXD_IP4_HEADER_ERR)) ||
2596 (status & MVPP2_RXD_L3_IP6))
2597 if (((status & MVPP2_RXD_L4_UDP) ||
2598 (status & MVPP2_RXD_L4_TCP)) &&
2599 (status & MVPP2_RXD_L4_CSUM_OK)) {
2600 skb->csum = 0;
2601 skb->ip_summed = CHECKSUM_UNNECESSARY;
2602 return;
2603 }
2604
2605 skb->ip_summed = CHECKSUM_NONE;
2606}
2607
2608/* Reuse skb if possible, or allocate a new skb and add it to BM pool */
2609static int mvpp2_rx_refill(struct mvpp2_port *port,
56b8aae9 2610 struct mvpp2_bm_pool *bm_pool, int pool)
3f518509 2611{
20396136 2612 dma_addr_t dma_addr;
4e4a105f 2613 phys_addr_t phys_addr;
0e037281 2614 void *buf;
3f518509 2615
3f518509 2616 /* No recycle or too many buffers are in use, so allocate a new skb */
4e4a105f
TP		 2617 buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr, &phys_addr,
2618 GFP_ATOMIC);
0e037281 2619 if (!buf)
3f518509
MW		 2620 return -ENOMEM;
2621
7d7627ba 2622 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
7ef7e1d9 2623
3f518509
MW		 2624 return 0;
2625}
2626
2627/* Handle tx checksum */
2628static u32 mvpp2_skb_tx_csum(struct mvpp2_port *port, struct sk_buff *skb)
2629{
2630 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2631 int ip_hdr_len = 0;
2632 u8 l4_proto;
35f3625c 2633 __be16 l3_proto = vlan_get_protocol(skb);
3f518509 2634
35f3625c 2635 if (l3_proto == htons(ETH_P_IP)) {
3f518509
MW		 2636 struct iphdr *ip4h = ip_hdr(skb);
2637
2638 /* Calculate IPv4 checksum and L4 checksum */
2639 ip_hdr_len = ip4h->ihl;
2640 l4_proto = ip4h->protocol;
35f3625c 2641 } else if (l3_proto == htons(ETH_P_IPV6)) {
3f518509
MW		 2642 struct ipv6hdr *ip6h = ipv6_hdr(skb);
2643
2644 /* Read l4_protocol from one of IPv6 extra headers */
2645 if (skb_network_header_len(skb) > 0)
2646 ip_hdr_len = (skb_network_header_len(skb) >> 2);
2647 l4_proto = ip6h->nexthdr;
2648 } else {
2649 return MVPP2_TXD_L4_CSUM_NOT;
2650 }
2651
2652 return mvpp2_txq_desc_csum(skb_network_offset(skb),
35f3625c 2653 l3_proto, ip_hdr_len, l4_proto);
3f518509
MW		 2654 }
2655
2656 return MVPP2_TXD_L4_CSUM_NOT | MVPP2_TXD_IP_CSUM_DISABLE;
2657}
2658
3f518509 2659/* Main rx processing */
591f4cfa
TP		 2660static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
2661 int rx_todo, struct mvpp2_rx_queue *rxq)
3f518509
MW		 2662{
2663 struct net_device *dev = port->dev;
b5015854
MW		 2664 int rx_received;
2665 int rx_done = 0;
3f518509
MW		 2666 u32 rcvd_pkts = 0;
2667 u32 rcvd_bytes = 0;
2668
2669 /* Get number of received packets and clamp the to-do */
2670 rx_received = mvpp2_rxq_received(port, rxq->id);
2671 if (rx_todo > rx_received)
2672 rx_todo = rx_received;
2673
b5015854 2674 while (rx_done < rx_todo) {
3f518509
MW		 2675 struct mvpp2_rx_desc *rx_desc = mvpp2_rxq_next_desc_get(rxq);
2676 struct mvpp2_bm_pool *bm_pool;
2677 struct sk_buff *skb;
0e037281 2678 unsigned int frag_size;
20396136 2679 dma_addr_t dma_addr;
ac3dd277 2680 phys_addr_t phys_addr;
56b8aae9 2681 u32 rx_status;
3f518509 2682 int pool, rx_bytes, err;
0e037281 2683 void *data;
3f518509 2684
b5015854 2685 rx_done++;
ac3dd277
TP		 2686 rx_status = mvpp2_rxdesc_status_get(port, rx_desc);
2687 rx_bytes = mvpp2_rxdesc_size_get(port, rx_desc);
2688 rx_bytes -= MVPP2_MH_SIZE;
2689 dma_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc);
2690 phys_addr = mvpp2_rxdesc_cookie_get(port, rx_desc);
2691 data = (void *)phys_to_virt(phys_addr);
2692
56b8aae9
TP		 2693 pool = (rx_status & MVPP2_RXD_BM_POOL_ID_MASK) >>
2694 MVPP2_RXD_BM_POOL_ID_OFFS;
3f518509 2695 bm_pool = &port->priv->bm_pools[pool];
3f518509
MW		 2696
2697 /* In case of an error, release the requested buffer pointer
2698 * to the Buffer Manager. This request process is controlled
2699 * by the hardware, and the information about the buffer is
2700 * comprised by the RX descriptor.
2701 */
2702 if (rx_status & MVPP2_RXD_ERR_SUMMARY) {
8a52488b 2703err_drop_frame:
3f518509
MW		 2704 dev->stats.rx_errors++;
2705 mvpp2_rx_error(port, rx_desc);
b5015854 2706 /* Return the buffer to the pool */
7d7627ba 2707 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
3f518509
MW		 2708 continue;
2709 }
2710
0e037281
TP		 2711 if (bm_pool->frag_size > PAGE_SIZE)
2712 frag_size = 0;
2713 else
2714 frag_size = bm_pool->frag_size;
2715
2716 skb = build_skb(data, frag_size);
2717 if (!skb) {
2718 netdev_warn(port->dev, "skb build failed\n");
2719 goto err_drop_frame;
2720 }
3f518509 2721
56b8aae9 2722 err = mvpp2_rx_refill(port, bm_pool, pool);
b5015854
MW		 2723 if (err) {
2724 netdev_err(port->dev, "failed to refill BM pools\n");
2725 goto err_drop_frame;
2726 }
2727
20396136 2728 dma_unmap_single(dev->dev.parent, dma_addr,
4229d502
MW		 2729 bm_pool->buf_size, DMA_FROM_DEVICE);
2730
3f518509
MW		 2731 rcvd_pkts++;
2732 rcvd_bytes += rx_bytes;
3f518509 2733
0e037281 2734 skb_reserve(skb, MVPP2_MH_SIZE + NET_SKB_PAD);
3f518509
MW		 2735 skb_put(skb, rx_bytes);
2736 skb->protocol = eth_type_trans(skb, dev);
2737 mvpp2_rx_csum(port, rx_status, skb);
2738
591f4cfa 2739 napi_gro_receive(napi, skb);
3f518509
MW		 2740 }
2741
2742 if (rcvd_pkts) {
2743 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
2744
2745 u64_stats_update_begin(&stats->syncp);
2746 stats->rx_packets += rcvd_pkts;
2747 stats->rx_bytes += rcvd_bytes;
2748 u64_stats_update_end(&stats->syncp);
2749 }
2750
2751 /* Update Rx queue management counters */
2752 wmb();
b5015854 2753 mvpp2_rxq_status_update(port, rxq->id, rx_done, rx_done);
3f518509
MW		 2754
2755 return rx_todo;
2756}
2757
2758static inline void
ac3dd277 2759tx_desc_unmap_put(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
3f518509
MW		 2760 struct mvpp2_tx_desc *desc)
2761{
e531f767 2762 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
074c74df 2763 struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
20920267 2764
ac3dd277
TP		 2765 dma_addr_t buf_dma_addr =
2766 mvpp2_txdesc_dma_addr_get(port, desc);
2767 size_t buf_sz =
2768 mvpp2_txdesc_size_get(port, desc);
20920267
AT		 2769 if (!IS_TSO_HEADER(txq_pcpu, buf_dma_addr))
2770 dma_unmap_single(port->dev->dev.parent, buf_dma_addr,
2771 buf_sz, DMA_TO_DEVICE);
3f518509
MW		 2772 mvpp2_txq_desc_put(txq);
2773}
2774
2775/* Handle tx fragmentation processing */
2776static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
2777 struct mvpp2_tx_queue *aggr_txq,
2778 struct mvpp2_tx_queue *txq)
2779{
e531f767 2780 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
074c74df 2781 struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
3f518509
MW		 2782 struct mvpp2_tx_desc *tx_desc;
2783 int i;
20396136 2784 dma_addr_t buf_dma_addr;
3f518509
MW		 2785
2786 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2787 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2788 void *addr = page_address(frag->page.p) + frag->page_offset;
2789
2790 tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
ac3dd277
TP		 2791 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
2792 mvpp2_txdesc_size_set(port, tx_desc, frag->size);
3f518509 2793
20396136 2794 buf_dma_addr = dma_map_single(port->dev->dev.parent, addr,
a3302baa 2795 frag->size, DMA_TO_DEVICE);
20396136 2796 if (dma_mapping_error(port->dev->dev.parent, buf_dma_addr)) {
3f518509 2797 mvpp2_txq_desc_put(txq);
32bae631 2798 goto cleanup;
3f518509
MW		 2799 }
2800
6eb5d375 2801 mvpp2_txdesc_dma_addr_set(port, tx_desc, buf_dma_addr);
3f518509
MW		 2802
2803 if (i == (skb_shinfo(skb)->nr_frags - 1)) {
2804 /* Last descriptor */
ac3dd277
TP		 2805 mvpp2_txdesc_cmd_set(port, tx_desc,
2806 MVPP2_TXD_L_DESC);
2807 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
3f518509
MW		 2808 } else {
2809 /* Descriptor in the middle: Not First, Not Last */
ac3dd277
TP		 2810 mvpp2_txdesc_cmd_set(port, tx_desc, 0);
2811 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
3f518509
MW		 2812 }
2813 }
2814
2815 return 0;
32bae631 2816cleanup:
3f518509
MW		 2817 /* Release all descriptors that were used to map fragments of
2818 * this packet, as well as the corresponding DMA mappings
2819 */
2820 for (i = i - 1; i >= 0; i--) {
2821 tx_desc = txq->descs + i;
ac3dd277 2822 tx_desc_unmap_put(port, txq, tx_desc);
3f518509
MW		 2823 }
2824
2825 return -ENOMEM;
2826}
2827
186cd4d4
AT		 2828static inline void mvpp2_tso_put_hdr(struct sk_buff *skb,
2829 struct net_device *dev,
2830 struct mvpp2_tx_queue *txq,
2831 struct mvpp2_tx_queue *aggr_txq,
2832 struct mvpp2_txq_pcpu *txq_pcpu,
2833 int hdr_sz)
2834{
2835 struct mvpp2_port *port = netdev_priv(dev);
2836 struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
2837 dma_addr_t addr;
2838
2839 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
2840 mvpp2_txdesc_size_set(port, tx_desc, hdr_sz);
2841
2842 addr = txq_pcpu->tso_headers_dma +
2843 txq_pcpu->txq_put_index * TSO_HEADER_SIZE;
6eb5d375 2844 mvpp2_txdesc_dma_addr_set(port, tx_desc, addr);
186cd4d4
AT		 2845
2846 mvpp2_txdesc_cmd_set(port, tx_desc, mvpp2_skb_tx_csum(port, skb) |
2847 MVPP2_TXD_F_DESC |
2848 MVPP2_TXD_PADDING_DISABLE);
2849 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
2850}
2851
2852static inline int mvpp2_tso_put_data(struct sk_buff *skb,
2853 struct net_device *dev, struct tso_t *tso,
2854 struct mvpp2_tx_queue *txq,
2855 struct mvpp2_tx_queue *aggr_txq,
2856 struct mvpp2_txq_pcpu *txq_pcpu,
2857 int sz, bool left, bool last)
2858{
2859 struct mvpp2_port *port = netdev_priv(dev);
2860 struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
2861 dma_addr_t buf_dma_addr;
2862
2863 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
2864 mvpp2_txdesc_size_set(port, tx_desc, sz);
2865
2866 buf_dma_addr = dma_map_single(dev->dev.parent, tso->data, sz,
2867 DMA_TO_DEVICE);
2868 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) {
2869 mvpp2_txq_desc_put(txq);
2870 return -ENOMEM;
2871 }
2872
6eb5d375 2873 mvpp2_txdesc_dma_addr_set(port, tx_desc, buf_dma_addr);
186cd4d4
AT		 2874
2875 if (!left) {
2876 mvpp2_txdesc_cmd_set(port, tx_desc, MVPP2_TXD_L_DESC);
2877 if (last) {
2878 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
2879 return 0;
2880 }
2881 } else {
2882 mvpp2_txdesc_cmd_set(port, tx_desc, 0);
2883 }
2884
2885 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
2886 return 0;
2887}
2888
2889static int mvpp2_tx_tso(struct sk_buff *skb, struct net_device *dev,
2890 struct mvpp2_tx_queue *txq,
2891 struct mvpp2_tx_queue *aggr_txq,
2892 struct mvpp2_txq_pcpu *txq_pcpu)
2893{
2894 struct mvpp2_port *port = netdev_priv(dev);
2895 struct tso_t tso;
2896 int hdr_sz = skb_transport_offset(skb) + tcp_hdrlen(skb);
2897 int i, len, descs = 0;
2898
2899 /* Check number of available descriptors */
e531f767 2900 if (mvpp2_aggr_desc_num_check(port, aggr_txq, tso_count_descs(skb)) ||
074c74df 2901 mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu,
186cd4d4
AT		 2902 tso_count_descs(skb)))
2903 return 0;
2904
2905 tso_start(skb, &tso);
2906 len = skb->len - hdr_sz;
2907 while (len > 0) {
2908 int left = min_t(int, skb_shinfo(skb)->gso_size, len);
2909 char *hdr = txq_pcpu->tso_headers +
2910 txq_pcpu->txq_put_index * TSO_HEADER_SIZE;
2911
2912 len -= left;
2913 descs++;
2914
2915 tso_build_hdr(skb, hdr, &tso, left, len == 0);
2916 mvpp2_tso_put_hdr(skb, dev, txq, aggr_txq, txq_pcpu, hdr_sz);
2917
2918 while (left > 0) {
2919 int sz = min_t(int, tso.size, left);
2920 left -= sz;
2921 descs++;
2922
2923 if (mvpp2_tso_put_data(skb, dev, &tso, txq, aggr_txq,
2924 txq_pcpu, sz, left, len == 0))
2925 goto release;
2926 tso_build_data(skb, &tso, sz);
2927 }
2928 }
2929
2930 return descs;
2931
2932release:
2933 for (i = descs - 1; i >= 0; i--) {
2934 struct mvpp2_tx_desc *tx_desc = txq->descs + i;
2935 tx_desc_unmap_put(port, txq, tx_desc);
2936 }
2937 return 0;
2938}
2939
3f518509 2940/* Main tx processing */
f03508ce 2941static netdev_tx_t mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
3f518509
MW		 2942{
2943 struct mvpp2_port *port = netdev_priv(dev);
2944 struct mvpp2_tx_queue *txq, *aggr_txq;
2945 struct mvpp2_txq_pcpu *txq_pcpu;
2946 struct mvpp2_tx_desc *tx_desc;
20396136 2947 dma_addr_t buf_dma_addr;
e531f767 2948 unsigned long flags = 0;
074c74df 2949 unsigned int thread;
3f518509
MW		 2950 int frags = 0;
2951 u16 txq_id;
2952 u32 tx_cmd;
2953
e531f767 2954 thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
074c74df 2955
3f518509
MW		 2956 txq_id = skb_get_queue_mapping(skb);
2957 txq = port->txqs[txq_id];
074c74df
AT		 2958 txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
2959 aggr_txq = &port->priv->aggr_txqs[thread];
3f518509 2960
e531f767
AT		 2961 if (test_bit(thread, &port->priv->lock_map))
2962 spin_lock_irqsave(&port->tx_lock[thread], flags);
2963
186cd4d4
AT		 2964 if (skb_is_gso(skb)) {
2965 frags = mvpp2_tx_tso(skb, dev, txq, aggr_txq, txq_pcpu);
2966 goto out;
2967 }
3f518509
MW		 2968 frags = skb_shinfo(skb)->nr_frags + 1;
2969
2970 /* Check number of available descriptors */
e531f767 2971 if (mvpp2_aggr_desc_num_check(port, aggr_txq, frags) ||
074c74df 2972 mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, frags)) {
3f518509
MW		 2973 frags = 0;
2974 goto out;
2975 }
2976
2977 /* Get a descriptor for the first part of the packet */
2978 tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
ac3dd277
TP		 2979 mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
2980 mvpp2_txdesc_size_set(port, tx_desc, skb_headlen(skb));
3f518509 2981
20396136 2982 buf_dma_addr = dma_map_single(dev->dev.parent, skb->data,
ac3dd277 2983 skb_headlen(skb), DMA_TO_DEVICE);
20396136 2984 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) {
3f518509
MW		 2985 mvpp2_txq_desc_put(txq);
2986 frags = 0;
2987 goto out;
2988 }
ac3dd277 2989
6eb5d375 2990 mvpp2_txdesc_dma_addr_set(port, tx_desc, buf_dma_addr);
3f518509
MW		 2991
2992 tx_cmd = mvpp2_skb_tx_csum(port, skb);
2993
2994 if (frags == 1) {
2995 /* First and Last descriptor */
2996 tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_L_DESC;
ac3dd277
TP		 2997 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
2998 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
3f518509
MW		 2999 } else {
3000 /* First but not Last */
3001 tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_PADDING_DISABLE;
ac3dd277
TP		 3002 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
3003 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
3f518509
MW		 3004
3005 /* Continue with other skb fragments */
3006 if (mvpp2_tx_frag_process(port, skb, aggr_txq, txq)) {
ac3dd277 3007 tx_desc_unmap_put(port, txq, tx_desc);
3f518509 3008 frags = 0;
3f518509
MW		 3009 }
3010 }
3011
3f518509
MW		 3012out:
3013 if (frags > 0) {
074c74df 3014 struct mvpp2_pcpu_stats *stats = per_cpu_ptr(port->stats, thread);
186cd4d4
AT		 3015 struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
3016
3017 txq_pcpu->reserved_num -= frags;
3018 txq_pcpu->count += frags;
3019 aggr_txq->count += frags;
3020
3021 /* Enable transmit */
3022 wmb();
3023 mvpp2_aggr_txq_pend_desc_add(port, frags);
3024
1d17db08 3025 if (txq_pcpu->count >= txq_pcpu->stop_threshold)
186cd4d4 3026 netif_tx_stop_queue(nq);
3f518509
MW		 3027
3028 u64_stats_update_begin(&stats->syncp);
3029 stats->tx_packets++;
3030 stats->tx_bytes += skb->len;
3031 u64_stats_update_end(&stats->syncp);
3032 } else {
3033 dev->stats.tx_dropped++;
3034 dev_kfree_skb_any(skb);
3035 }
3036
edc660fa 3037 /* Finalize TX processing */
082297e6 3038 if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)
edc660fa
MW		 3039 mvpp2_txq_done(port, txq, txq_pcpu);
3040
3041 /* Set the timer in case not all frags were processed */
213f428f
TP		 3042 if (!port->has_tx_irqs && txq_pcpu->count <= frags &&
3043 txq_pcpu->count > 0) {
074c74df 3044 struct mvpp2_port_pcpu *port_pcpu = per_cpu_ptr(port->pcpu, thread);
edc660fa
MW		 3045
3046 mvpp2_timer_set(port_pcpu);
3047 }
3048
e531f767
AT		 3049 if (test_bit(thread, &port->priv->lock_map))
3050 spin_unlock_irqrestore(&port->tx_lock[thread], flags);
3051
3f518509
MW		 3052 return NETDEV_TX_OK;
3053}
3054
3055static inline void mvpp2_cause_error(struct net_device *dev, int cause)
3056{
3057 if (cause & MVPP2_CAUSE_FCS_ERR_MASK)
3058 netdev_err(dev, "FCS error\n");
3059 if (cause & MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK)
3060 netdev_err(dev, "rx fifo overrun error\n");
3061 if (cause & MVPP2_CAUSE_TX_FIFO_UNDERRUN_MASK)
3062 netdev_err(dev, "tx fifo underrun error\n");
3063}
3064
edc660fa 3065static int mvpp2_poll(struct napi_struct *napi, int budget)
3f518509 3066{
213f428f 3067 u32 cause_rx_tx, cause_rx, cause_tx, cause_misc;
edc660fa
MW		 3068 int rx_done = 0;
3069 struct mvpp2_port *port = netdev_priv(napi->dev);
591f4cfa 3070 struct mvpp2_queue_vector *qv;
e531f767 3071 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
3f518509 3072
591f4cfa
TP		 3073 qv = container_of(napi, struct mvpp2_queue_vector, napi);
3074
3f518509
MW		 3075 /* Rx/Tx cause register
3076 *
3077 * Bits 0-15: each bit indicates received packets on the Rx queue
3078 * (bit 0 is for Rx queue 0).
3079 *
3080 * Bits 16-23: each bit indicates transmitted packets on the Tx queue
3081 * (bit 16 is for Tx queue 0).
3082 *
3083 * Each CPU has its own Rx/Tx cause register
3084 */
1068549c 3085 cause_rx_tx = mvpp2_thread_read_relaxed(port->priv, qv->sw_thread_id,
cdcfeb0f 3086 MVPP2_ISR_RX_TX_CAUSE_REG(port->id));
3f518509 3087
213f428f 3088 cause_misc = cause_rx_tx & MVPP2_CAUSE_MISC_SUM_MASK;
3f518509
MW		 3089 if (cause_misc) {
3090 mvpp2_cause_error(port->dev, cause_misc);
3091
3092 /* Clear the cause register */
3093 mvpp2_write(port->priv, MVPP2_ISR_MISC_CAUSE_REG, 0);
1068549c 3094 mvpp2_thread_write(port->priv, thread,
a786841d
TP		 3095 MVPP2_ISR_RX_TX_CAUSE_REG(port->id),
3096 cause_rx_tx & ~MVPP2_CAUSE_MISC_SUM_MASK);
3f518509
MW		 3097 }
3098
774268f3
AT		 3099 if (port->has_tx_irqs) {
3100 cause_tx = cause_rx_tx & MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK;
3101 if (cause_tx) {
3102 cause_tx >>= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET;
3103 mvpp2_tx_done(port, cause_tx, qv->sw_thread_id);
3104 }
213f428f 3105 }
3f518509
MW		 3106
3107 /* Process RX packets */
70afb58e
AT		 3108 cause_rx = cause_rx_tx &
3109 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version);
213f428f 3110 cause_rx <<= qv->first_rxq;
591f4cfa 3111 cause_rx |= qv->pending_cause_rx;
3f518509
MW		 3112 while (cause_rx && budget > 0) {
3113 int count;
3114 struct mvpp2_rx_queue *rxq;
3115
3116 rxq = mvpp2_get_rx_queue(port, cause_rx);
3117 if (!rxq)
3118 break;
3119
591f4cfa 3120 count = mvpp2_rx(port, napi, budget, rxq);
3f518509
MW		 3121 rx_done += count;
3122 budget -= count;
3123 if (budget > 0) {
3124 /* Clear the bit associated to this Rx queue
3125 * so that next iteration will continue from
3126 * the next Rx queue.
3127 */
3128 cause_rx &= ~(1 << rxq->logic_rxq);
3129 }
3130 }
3131
3132 if (budget > 0) {
3133 cause_rx = 0;
6ad20165 3134 napi_complete_done(napi, rx_done);
3f518509 3135
591f4cfa 3136 mvpp2_qvec_interrupt_enable(qv);
3f518509 3137 }
591f4cfa 3138 qv->pending_cause_rx = cause_rx;
3f518509
MW		 3139 return rx_done;
3140}
3141
4bb04326 3142static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
3f518509 3143{
4bb04326
AT		 3144 u32 ctrl3;
3145
3146 /* comphy reconfiguration */
3147 mvpp22_comphy_init(port);
3148
3149 /* gop reconfiguration */
3150 mvpp22_gop_init(port);
3151
3152 /* Only GOP port 0 has an XLG MAC */
3153 if (port->gop_id == 0) {
3154 ctrl3 = readl(port->base + MVPP22_XLG_CTRL3_REG);
3155 ctrl3 &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
3156
3157 if (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
3158 port->phy_interface == PHY_INTERFACE_MODE_10GKR)
3159 ctrl3 |= MVPP22_XLG_CTRL3_MACMODESELECT_10G;
3160 else
3161 ctrl3 |= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC;
3162
3163 writel(ctrl3, port->base + MVPP22_XLG_CTRL3_REG);
3164 }
8e07269d 3165
76eb1b1d
SC		 3166 if (port->gop_id == 0 &&
3167 (port->phy_interface == PHY_INTERFACE_MODE_XAUI ||
3168 port->phy_interface == PHY_INTERFACE_MODE_10GKR))
3169 mvpp2_xlg_max_rx_size_set(port);
3170 else
3171 mvpp2_gmac_max_rx_size_set(port);
4bb04326
AT		 3172}
3173
3174/* Set hw internals when starting port */
3175static void mvpp2_start_dev(struct mvpp2_port *port)
3176{
3177 int i;
76eb1b1d 3178
3f518509
MW		 3179 mvpp2_txp_max_tx_size_set(port);
3180
591f4cfa
TP		 3181 for (i = 0; i < port->nqvecs; i++)
3182 napi_enable(&port->qvecs[i].napi);
3f518509 3183
543ec376 3184 /* Enable interrupts on all threads */
3f518509
MW		 3185 mvpp2_interrupts_enable(port);
3186
4bb04326
AT		 3187 if (port->priv->hw_version == MVPP22)
3188 mvpp22_mode_reconfigure(port);
3189
3190 if (port->phylink) {
3191 phylink_start(port->phylink);
3192 } else {
3193 /* Phylink isn't used as of now for ACPI, so the MAC has to be
3194 * configured manually when the interface is started. This will
3195 * be removed as soon as the phylink ACPI support lands in.
3196 */
3197 struct phylink_link_state state = {
3198 .interface = port->phy_interface,
4bb04326
AT		 3199 };
3200 mvpp2_mac_config(port->dev, MLO_AN_INBAND, &state);
41948ccb
AT		 3201 mvpp2_mac_link_up(port->dev, MLO_AN_INBAND, port->phy_interface,
3202 NULL);
542897d9 3203 }
f84bf386 3204
3f518509
MW		 3205 netif_tx_start_all_queues(port->dev);
3206}
3207
3208/* Set hw internals when stopping port */
3209static void mvpp2_stop_dev(struct mvpp2_port *port)
3210{
591f4cfa 3211 int i;
8e07269d 3212
543ec376 3213 /* Disable interrupts on all threads */
3f518509
MW		 3214 mvpp2_interrupts_disable(port);
3215
591f4cfa
TP		 3216 for (i = 0; i < port->nqvecs; i++)
3217 napi_disable(&port->qvecs[i].napi);
3f518509 3218
4bb04326
AT		 3219 if (port->phylink)
3220 phylink_stop(port->phylink);
542897d9 3221 phy_power_off(port->comphy);
3f518509
MW		 3222}
3223
3f518509
MW		 3224static int mvpp2_check_ringparam_valid(struct net_device *dev,
3225 struct ethtool_ringparam *ring)
3226{
3227 u16 new_rx_pending = ring->rx_pending;
3228 u16 new_tx_pending = ring->tx_pending;
3229
3230 if (ring->rx_pending == 0 || ring->tx_pending == 0)
3231 return -EINVAL;
3232
7cf87e4a
YM		 3233 if (ring->rx_pending > MVPP2_MAX_RXD_MAX)
3234 new_rx_pending = MVPP2_MAX_RXD_MAX;
3f518509
MW		 3235 else if (!IS_ALIGNED(ring->rx_pending, 16))
3236 new_rx_pending = ALIGN(ring->rx_pending, 16);
3237
7cf87e4a
YM		 3238 if (ring->tx_pending > MVPP2_MAX_TXD_MAX)
3239 new_tx_pending = MVPP2_MAX_TXD_MAX;
3f518509
MW		 3240 else if (!IS_ALIGNED(ring->tx_pending, 32))
3241 new_tx_pending = ALIGN(ring->tx_pending, 32);
3242
76e583c5
AT		 3243 /* The Tx ring size cannot be smaller than the minimum number of
3244 * descriptors needed for TSO.
3245 */
3246 if (new_tx_pending < MVPP2_MAX_SKB_DESCS)
3247 new_tx_pending = ALIGN(MVPP2_MAX_SKB_DESCS, 32);
3248
3f518509
MW		 3249 if (ring->rx_pending != new_rx_pending) {
3250 netdev_info(dev, "illegal Rx ring size value %d, round to %d\n",
3251 ring->rx_pending, new_rx_pending);
3252 ring->rx_pending = new_rx_pending;
3253 }
3254
3255 if (ring->tx_pending != new_tx_pending) {
3256 netdev_info(dev, "illegal Tx ring size value %d, round to %d\n",
3257 ring->tx_pending, new_tx_pending);
3258 ring->tx_pending = new_tx_pending;
3259 }
3260
3261 return 0;
3262}
3263
26975821 3264static void mvpp21_get_mac_address(struct mvpp2_port *port, unsigned char *addr)
3f518509
MW		 3265{
3266 u32 mac_addr_l, mac_addr_m, mac_addr_h;
3267
3268 mac_addr_l = readl(port->base + MVPP2_GMAC_CTRL_1_REG);
3269 mac_addr_m = readl(port->priv->lms_base + MVPP2_SRC_ADDR_MIDDLE);
3270 mac_addr_h = readl(port->priv->lms_base + MVPP2_SRC_ADDR_HIGH);
3271 addr[0] = (mac_addr_h >> 24) & 0xFF;
3272 addr[1] = (mac_addr_h >> 16) & 0xFF;
3273 addr[2] = (mac_addr_h >> 8) & 0xFF;
3274 addr[3] = mac_addr_h & 0xFF;
3275 addr[4] = mac_addr_m & 0xFF;
3276 addr[5] = (mac_addr_l >> MVPP2_GMAC_SA_LOW_OFFS) & 0xFF;
3277}
3278
591f4cfa
TP		 3279static int mvpp2_irqs_init(struct mvpp2_port *port)
3280{
3281 int err, i;
3282
3283 for (i = 0; i < port->nqvecs; i++) {
3284 struct mvpp2_queue_vector *qv = port->qvecs + i;
3285
a6b3a3fa
MZ		 3286 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) {
3287 qv->mask = kzalloc(cpumask_size(), GFP_KERNEL);
3288 if (!qv->mask) {
3289 err = -ENOMEM;
3290 goto err;
3291 }
3292
13c249a9 3293 irq_set_status_flags(qv->irq, IRQ_NO_BALANCING);
a6b3a3fa 3294 }
13c249a9 3295
591f4cfa
TP		 3296 err = request_irq(qv->irq, mvpp2_isr, 0, port->dev->name, qv);
3297 if (err)
3298 goto err;
213f428f 3299
e531f767 3300 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) {
e531f767
AT		 3301 unsigned int cpu;
3302
3303 for_each_present_cpu(cpu) {
3304 if (mvpp2_cpu_to_thread(port->priv, cpu) ==
3305 qv->sw_thread_id)
a6b3a3fa 3306 cpumask_set_cpu(cpu, qv->mask);
e531f767
AT		 3307 }
3308
a6b3a3fa 3309 irq_set_affinity_hint(qv->irq, qv->mask);
e531f767 3310 }
591f4cfa
TP		 3311 }
3312
3313 return 0;
3314err:
3315 for (i = 0; i < port->nqvecs; i++) {
3316 struct mvpp2_queue_vector *qv = port->qvecs + i;
3317
213f428f 3318 irq_set_affinity_hint(qv->irq, NULL);
a6b3a3fa
MZ		 3319 kfree(qv->mask);
3320 qv->mask = NULL;
591f4cfa
TP		 3321 free_irq(qv->irq, qv);
3322 }
3323
3324 return err;
3325}
3326
3327static void mvpp2_irqs_deinit(struct mvpp2_port *port)
3328{
3329 int i;
3330
3331 for (i = 0; i < port->nqvecs; i++) {
3332 struct mvpp2_queue_vector *qv = port->qvecs + i;
3333
213f428f 3334 irq_set_affinity_hint(qv->irq, NULL);
a6b3a3fa
MZ		 3335 kfree(qv->mask);
3336 qv->mask = NULL;
13c249a9 3337 irq_clear_status_flags(qv->irq, IRQ_NO_BALANCING);
591f4cfa
TP		 3338 free_irq(qv->irq, qv);
3339 }
3340}
3341
4c4a5686
YM		 3342static bool mvpp22_rss_is_supported(void)
3343{
3344 return queue_mode == MVPP2_QDIST_MULTI_MODE;
3345}
3346
3f518509
MW		 3347static int mvpp2_open(struct net_device *dev)
3348{
3349 struct mvpp2_port *port = netdev_priv(dev);
fd3651b2 3350 struct mvpp2 *priv = port->priv;
3f518509
MW		 3351 unsigned char mac_bcast[ETH_ALEN] = {
3352 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
4bb04326 3353 bool valid = false;
3f518509
MW		 3354 int err;
3355
ce2a27c7 3356 err = mvpp2_prs_mac_da_accept(port, mac_bcast, true);
3f518509
MW		 3357 if (err) {
3358 netdev_err(dev, "mvpp2_prs_mac_da_accept BC failed\n");
3359 return err;
3360 }
ce2a27c7 3361 err = mvpp2_prs_mac_da_accept(port, dev->dev_addr, true);
3f518509 3362 if (err) {
ce2a27c7 3363 netdev_err(dev, "mvpp2_prs_mac_da_accept own addr failed\n");
3f518509
MW		 3364 return err;
3365 }
3366 err = mvpp2_prs_tag_mode_set(port->priv, port->id, MVPP2_TAG_TYPE_MH);
3367 if (err) {
3368 netdev_err(dev, "mvpp2_prs_tag_mode_set failed\n");
3369 return err;
3370 }
3371 err = mvpp2_prs_def_flow(port);
3372 if (err) {
3373 netdev_err(dev, "mvpp2_prs_def_flow failed\n");
3374 return err;
3375 }
3376
3377 /* Allocate the Rx/Tx queues */
3378 err = mvpp2_setup_rxqs(port);
3379 if (err) {
3380 netdev_err(port->dev, "cannot allocate Rx queues\n");
3381 return err;
3382 }
3383
3384 err = mvpp2_setup_txqs(port);
3385 if (err) {
3386 netdev_err(port->dev, "cannot allocate Tx queues\n");
3387 goto err_cleanup_rxqs;
3388 }
3389
591f4cfa 3390 err = mvpp2_irqs_init(port);
3f518509 3391 if (err) {
591f4cfa 3392 netdev_err(port->dev, "cannot init IRQs\n");
3f518509
MW		 3393 goto err_cleanup_txqs;
3394 }
3395
4bb04326
AT		 3396 /* Phylink isn't supported yet in ACPI mode */
3397 if (port->of_node) {
3398 err = phylink_of_phy_connect(port->phylink, port->of_node, 0);
3399 if (err) {
3400 netdev_err(port->dev, "could not attach PHY (%d)\n",
3401 err);
3402 goto err_free_irq;
3403 }
3404
3405 valid = true;
3406 }
3407
3408 if (priv->hw_version == MVPP22 && port->link_irq && !port->phylink) {
fd3651b2
AT		 3409 err = request_irq(port->link_irq, mvpp2_link_status_isr, 0,
3410 dev->name, port);
3411 if (err) {
3412 netdev_err(port->dev, "cannot request link IRQ %d\n",
3413 port->link_irq);
3414 goto err_free_irq;
3415 }
3416
3417 mvpp22_gop_setup_irq(port);
fd3651b2 3418
4bb04326
AT		 3419 /* In default link is down */
3420 netif_carrier_off(port->dev);
3f518509 3421
4bb04326
AT		 3422 valid = true;
3423 } else {
3424 port->link_irq = 0;
3425 }
3426
3427 if (!valid) {
3428 netdev_err(port->dev,
3429 "invalid configuration: no dt or link IRQ");
3430 goto err_free_irq;
3431 }
3f518509
MW		 3432
3433 /* Unmask interrupts on all CPUs */
3434 on_each_cpu(mvpp2_interrupts_unmask, port, 1);
213f428f 3435 mvpp2_shared_interrupt_mask_unmask(port, false);
3f518509
MW		 3436
3437 mvpp2_start_dev(port);
3438
118d6298 3439 /* Start hardware statistics gathering */
e5c500eb 3440 queue_delayed_work(priv->stats_queue, &port->stats_work,
118d6298
MR		 3441 MVPP2_MIB_COUNTERS_STATS_DELAY);
3442
3f518509
MW		 3443 return 0;
3444
3445err_free_irq:
591f4cfa 3446 mvpp2_irqs_deinit(port);
3f518509
MW		 3447err_cleanup_txqs:
3448 mvpp2_cleanup_txqs(port);
3449err_cleanup_rxqs:
3450 mvpp2_cleanup_rxqs(port);
3451 return err;
3452}
3453
3454static int mvpp2_stop(struct net_device *dev)
3455{
3456 struct mvpp2_port *port = netdev_priv(dev);
edc660fa 3457 struct mvpp2_port_pcpu *port_pcpu;
074c74df 3458 unsigned int thread;
3f518509
MW		 3459
3460 mvpp2_stop_dev(port);
3f518509 3461
e531f767 3462 /* Mask interrupts on all threads */
3f518509 3463 on_each_cpu(mvpp2_interrupts_mask, port, 1);
213f428f 3464 mvpp2_shared_interrupt_mask_unmask(port, true);
3f518509 3465
4bb04326
AT		 3466 if (port->phylink)
3467 phylink_disconnect_phy(port->phylink);
3468 if (port->link_irq)
fd3651b2
AT		 3469 free_irq(port->link_irq, port);
3470
591f4cfa 3471 mvpp2_irqs_deinit(port);
213f428f 3472 if (!port->has_tx_irqs) {
e531f767 3473 for (thread = 0; thread < port->priv->nthreads; thread++) {
074c74df 3474 port_pcpu = per_cpu_ptr(port->pcpu, thread);
edc660fa 3475
213f428f
TP		 3476 hrtimer_cancel(&port_pcpu->tx_done_timer);
3477 port_pcpu->timer_scheduled = false;
3478 tasklet_kill(&port_pcpu->tx_done_tasklet);
3479 }
edc660fa 3480 }
3f518509
MW		 3481 mvpp2_cleanup_rxqs(port);
3482 mvpp2_cleanup_txqs(port);
3483
e5c500eb 3484 cancel_delayed_work_sync(&port->stats_work);
118d6298 3485
3f518509
MW		 3486 return 0;
3487}
3488
10fea26c
MC		 3489static int mvpp2_prs_mac_da_accept_list(struct mvpp2_port *port,
3490 struct netdev_hw_addr_list *list)
3f518509 3491{
3f518509 3492 struct netdev_hw_addr *ha;
10fea26c
MC		 3493 int ret;
3494
3495 netdev_hw_addr_list_for_each(ha, list) {
3496 ret = mvpp2_prs_mac_da_accept(port, ha->addr, true);
3497 if (ret)
3498 return ret;
3f518509 3499 }
56beda3d 3500
10fea26c
MC		 3501 return 0;
3502}
3503
3504static void mvpp2_set_rx_promisc(struct mvpp2_port *port, bool enable)
3505{
3506 if (!enable && (port->dev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
56beda3d 3507 mvpp2_prs_vid_enable_filtering(port);
10fea26c
MC		 3508 else
3509 mvpp2_prs_vid_disable_filtering(port);
3510
3511 mvpp2_prs_mac_promisc_set(port->priv, port->id,
3512 MVPP2_PRS_L2_UNI_CAST, enable);
3513
3514 mvpp2_prs_mac_promisc_set(port->priv, port->id,
3515 MVPP2_PRS_L2_MULTI_CAST, enable);
3516}
3517
3518static void mvpp2_set_rx_mode(struct net_device *dev)
3519{
3520 struct mvpp2_port *port = netdev_priv(dev);
3521
3522 /* Clear the whole UC and MC list */
3523 mvpp2_prs_mac_del_all(port);
3524
3525 if (dev->flags & IFF_PROMISC) {
3526 mvpp2_set_rx_promisc(port, true);
3527 return;
3528 }
3529
3530 mvpp2_set_rx_promisc(port, false);
3531
3532 if (netdev_uc_count(dev) > MVPP2_PRS_MAC_UC_FILT_MAX ||
3533 mvpp2_prs_mac_da_accept_list(port, &dev->uc))
3534 mvpp2_prs_mac_promisc_set(port->priv, port->id,
3535 MVPP2_PRS_L2_UNI_CAST, true);
3536
3537 if (dev->flags & IFF_ALLMULTI) {
3538 mvpp2_prs_mac_promisc_set(port->priv, port->id,
3539 MVPP2_PRS_L2_MULTI_CAST, true);
3540 return;
3541 }
3542
3543 if (netdev_mc_count(dev) > MVPP2_PRS_MAC_MC_FILT_MAX ||
3544 mvpp2_prs_mac_da_accept_list(port, &dev->mc))
3545 mvpp2_prs_mac_promisc_set(port->priv, port->id,
3546 MVPP2_PRS_L2_MULTI_CAST, true);
3f518509
MW		 3547}
3548
3549static int mvpp2_set_mac_address(struct net_device *dev, void *p)
3550{
3f518509
MW		 3551 const struct sockaddr *addr = p;
3552 int err;
3553
5b0ab2f4
YM		 3554 if (!is_valid_ether_addr(addr->sa_data))
3555 return -EADDRNOTAVAIL;
3f518509
MW		 3556
3557 err = mvpp2_prs_update_mac_da(dev, addr->sa_data);
5b0ab2f4
YM		 3558 if (err) {
3559 /* Reconfigure parser accept the original MAC address */
3560 mvpp2_prs_update_mac_da(dev, dev->dev_addr);
3561 netdev_err(dev, "failed to change MAC address\n");
3562 }
3f518509
MW		 3563 return err;
3564}
3565
3566static int mvpp2_change_mtu(struct net_device *dev, int mtu)
3567{
3568 struct mvpp2_port *port = netdev_priv(dev);
3569 int err;
3570
5777987e
JW		 3571 if (!IS_ALIGNED(MVPP2_RX_PKT_SIZE(mtu), 8)) {
3572 netdev_info(dev, "illegal MTU value %d, round to %d\n", mtu,
3573 ALIGN(MVPP2_RX_PKT_SIZE(mtu), 8));
3574 mtu = ALIGN(MVPP2_RX_PKT_SIZE(mtu), 8);
3f518509
MW		 3575 }
3576
3577 if (!netif_running(dev)) {
3578 err = mvpp2_bm_update_mtu(dev, mtu);
3579 if (!err) {
3580 port->pkt_size = MVPP2_RX_PKT_SIZE(mtu);
3581 return 0;
3582 }
3583
3584 /* Reconfigure BM to the original MTU */
3585 err = mvpp2_bm_update_mtu(dev, dev->mtu);
3586 if (err)
c1175547 3587 goto log_error;
3f518509
MW		 3588 }
3589
3590 mvpp2_stop_dev(port);
3591
3592 err = mvpp2_bm_update_mtu(dev, mtu);
3593 if (!err) {
3594 port->pkt_size = MVPP2_RX_PKT_SIZE(mtu);
3595 goto out_start;
3596 }
3597
3598 /* Reconfigure BM to the original MTU */
3599 err = mvpp2_bm_update_mtu(dev, dev->mtu);
3600 if (err)
c1175547 3601 goto log_error;
3f518509
MW		 3602
3603out_start:
3604 mvpp2_start_dev(port);
3605 mvpp2_egress_enable(port);
3606 mvpp2_ingress_enable(port);
3607
3608 return 0;
c1175547 3609log_error:
dfd4240a 3610 netdev_err(dev, "failed to change MTU\n");
3f518509
MW		 3611 return err;
3612}
3613
bc1f4470 3614static void
3f518509
MW		 3615mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
3616{
3617 struct mvpp2_port *port = netdev_priv(dev);
3618 unsigned int start;
850623b3 3619 unsigned int cpu;
3f518509
MW		 3620
3621 for_each_possible_cpu(cpu) {
3622 struct mvpp2_pcpu_stats *cpu_stats;
3623 u64 rx_packets;
3624 u64 rx_bytes;
3625 u64 tx_packets;
3626 u64 tx_bytes;
3627
3628 cpu_stats = per_cpu_ptr(port->stats, cpu);
3629 do {
3630 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
3631 rx_packets = cpu_stats->rx_packets;
3632 rx_bytes = cpu_stats->rx_bytes;
3633 tx_packets = cpu_stats->tx_packets;
3634 tx_bytes = cpu_stats->tx_bytes;
3635 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
3636
3637 stats->rx_packets += rx_packets;
3638 stats->rx_bytes += rx_bytes;
3639 stats->tx_packets += tx_packets;
3640 stats->tx_bytes += tx_bytes;
3641 }
3642
3643 stats->rx_errors = dev->stats.rx_errors;
3644 stats->rx_dropped = dev->stats.rx_dropped;
3645 stats->tx_dropped = dev->stats.tx_dropped;
3f518509
MW		 3646}
3647
bd695a5f
TP		 3648static int mvpp2_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
3649{
4bb04326 3650 struct mvpp2_port *port = netdev_priv(dev);
bd695a5f 3651
4bb04326 3652 if (!port->phylink)
bd695a5f
TP		 3653 return -ENOTSUPP;
3654
4bb04326 3655 return phylink_mii_ioctl(port->phylink, ifr, cmd);
bd695a5f
TP		 3656}
3657
56beda3d
MC		 3658static int mvpp2_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
3659{
3660 struct mvpp2_port *port = netdev_priv(dev);
3661 int ret;
3662
3663 ret = mvpp2_prs_vid_entry_add(port, vid);
3664 if (ret)
3665 netdev_err(dev, "rx-vlan-filter offloading cannot accept more than %d VIDs per port\n",
3666 MVPP2_PRS_VLAN_FILT_MAX - 1);
3667 return ret;
3668}
3669
3670static int mvpp2_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
3671{
3672 struct mvpp2_port *port = netdev_priv(dev);
3673
3674 mvpp2_prs_vid_entry_remove(port, vid);
3675 return 0;
3676}
3677
3678static int mvpp2_set_features(struct net_device *dev,
3679 netdev_features_t features)
3680{
3681 netdev_features_t changed = dev->features ^ features;
3682 struct mvpp2_port *port = netdev_priv(dev);
3683
3684 if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) {
3685 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
3686 mvpp2_prs_vid_enable_filtering(port);
3687 } else {
3688 /* Invalidate all registered VID filters for this
3689 * port
3690 */
3691 mvpp2_prs_vid_remove_all(port);
3692
3693 mvpp2_prs_vid_disable_filtering(port);
3694 }
3695 }
3696
d33ec452
MC		 3697 if (changed & NETIF_F_RXHASH) {
3698 if (features & NETIF_F_RXHASH)
3699 mvpp22_rss_enable(port);
3700 else
3701 mvpp22_rss_disable(port);
3702 }
3703
56beda3d
MC		 3704 return 0;
3705}
3706
3f518509
MW		 3707/* Ethtool methods */
3708
4bb04326
AT		 3709static int mvpp2_ethtool_nway_reset(struct net_device *dev)
3710{
3711 struct mvpp2_port *port = netdev_priv(dev);
3712
3713 if (!port->phylink)
3714 return -ENOTSUPP;
3715
3716 return phylink_ethtool_nway_reset(port->phylink);
3717}
3718
3f518509
MW		 3719/* Set interrupt coalescing for ethtools */
3720static int mvpp2_ethtool_set_coalesce(struct net_device *dev,
3721 struct ethtool_coalesce *c)
3722{
3723 struct mvpp2_port *port = netdev_priv(dev);
3724 int queue;
3725
09f83975 3726 for (queue = 0; queue < port->nrxqs; queue++) {
3f518509
MW		 3727 struct mvpp2_rx_queue *rxq = port->rxqs[queue];
3728
3729 rxq->time_coal = c->rx_coalesce_usecs;
3730 rxq->pkts_coal = c->rx_max_coalesced_frames;
d63f9e41
TP		 3731 mvpp2_rx_pkts_coal_set(port, rxq);
3732 mvpp2_rx_time_coal_set(port, rxq);
3f518509
MW		 3733 }
3734
213f428f
TP		 3735 if (port->has_tx_irqs) {
3736 port->tx_time_coal = c->tx_coalesce_usecs;
3737 mvpp2_tx_time_coal_set(port);
3738 }
3739
09f83975 3740 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 3741 struct mvpp2_tx_queue *txq = port->txqs[queue];
3742
3743 txq->done_pkts_coal = c->tx_max_coalesced_frames;
213f428f
TP		 3744
3745 if (port->has_tx_irqs)
3746 mvpp2_tx_pkts_coal_set(port, txq);
3f518509
MW		 3747 }
3748
3f518509
MW		 3749 return 0;
3750}
3751
3752/* get coalescing for ethtools */
3753static int mvpp2_ethtool_get_coalesce(struct net_device *dev,
3754 struct ethtool_coalesce *c)
3755{
3756 struct mvpp2_port *port = netdev_priv(dev);
3757
385c284f
AT		 3758 c->rx_coalesce_usecs = port->rxqs[0]->time_coal;
3759 c->rx_max_coalesced_frames = port->rxqs[0]->pkts_coal;
3760 c->tx_max_coalesced_frames = port->txqs[0]->done_pkts_coal;
24b28ccb 3761 c->tx_coalesce_usecs = port->tx_time_coal;
3f518509
MW		 3762 return 0;
3763}
3764
3765static void mvpp2_ethtool_get_drvinfo(struct net_device *dev,
3766 struct ethtool_drvinfo *drvinfo)
3767{
3768 strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME,
3769 sizeof(drvinfo->driver));
3770 strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION,
3771 sizeof(drvinfo->version));
3772 strlcpy(drvinfo->bus_info, dev_name(&dev->dev),
3773 sizeof(drvinfo->bus_info));
3774}
3775
3776static void mvpp2_ethtool_get_ringparam(struct net_device *dev,
3777 struct ethtool_ringparam *ring)
3778{
3779 struct mvpp2_port *port = netdev_priv(dev);
3780
7cf87e4a
YM		 3781 ring->rx_max_pending = MVPP2_MAX_RXD_MAX;
3782 ring->tx_max_pending = MVPP2_MAX_TXD_MAX;
3f518509
MW		 3783 ring->rx_pending = port->rx_ring_size;
3784 ring->tx_pending = port->tx_ring_size;
3785}
3786
3787static int mvpp2_ethtool_set_ringparam(struct net_device *dev,
3788 struct ethtool_ringparam *ring)
3789{
3790 struct mvpp2_port *port = netdev_priv(dev);
3791 u16 prev_rx_ring_size = port->rx_ring_size;
3792 u16 prev_tx_ring_size = port->tx_ring_size;
3793 int err;
3794
3795 err = mvpp2_check_ringparam_valid(dev, ring);
3796 if (err)
3797 return err;
3798
3799 if (!netif_running(dev)) {
3800 port->rx_ring_size = ring->rx_pending;
3801 port->tx_ring_size = ring->tx_pending;
3802 return 0;
3803 }
3804
3805 /* The interface is running, so we have to force a
3806 * reallocation of the queues
3807 */
3808 mvpp2_stop_dev(port);
3809 mvpp2_cleanup_rxqs(port);
3810 mvpp2_cleanup_txqs(port);
3811
3812 port->rx_ring_size = ring->rx_pending;
3813 port->tx_ring_size = ring->tx_pending;
3814
3815 err = mvpp2_setup_rxqs(port);
3816 if (err) {
3817 /* Reallocate Rx queues with the original ring size */
3818 port->rx_ring_size = prev_rx_ring_size;
3819 ring->rx_pending = prev_rx_ring_size;
3820 err = mvpp2_setup_rxqs(port);
3821 if (err)
3822 goto err_out;
3823 }
3824 err = mvpp2_setup_txqs(port);
3825 if (err) {
3826 /* Reallocate Tx queues with the original ring size */
3827 port->tx_ring_size = prev_tx_ring_size;
3828 ring->tx_pending = prev_tx_ring_size;
3829 err = mvpp2_setup_txqs(port);
3830 if (err)
3831 goto err_clean_rxqs;
3832 }
3833
3834 mvpp2_start_dev(port);
3835 mvpp2_egress_enable(port);
3836 mvpp2_ingress_enable(port);
3837
3838 return 0;
3839
3840err_clean_rxqs:
3841 mvpp2_cleanup_rxqs(port);
3842err_out:
dfd4240a 3843 netdev_err(dev, "failed to change ring parameters");
3f518509
MW		 3844 return err;
3845}
3846
4bb04326
AT		 3847static void mvpp2_ethtool_get_pause_param(struct net_device *dev,
3848 struct ethtool_pauseparam *pause)
3849{
3850 struct mvpp2_port *port = netdev_priv(dev);
3851
3852 if (!port->phylink)
3853 return;
3854
3855 phylink_ethtool_get_pauseparam(port->phylink, pause);
3856}
3857
3858static int mvpp2_ethtool_set_pause_param(struct net_device *dev,
3859 struct ethtool_pauseparam *pause)
3860{
3861 struct mvpp2_port *port = netdev_priv(dev);
3862
3863 if (!port->phylink)
3864 return -ENOTSUPP;
3865
3866 return phylink_ethtool_set_pauseparam(port->phylink, pause);
3867}
3868
3869static int mvpp2_ethtool_get_link_ksettings(struct net_device *dev,
3870 struct ethtool_link_ksettings *cmd)
3871{
3872 struct mvpp2_port *port = netdev_priv(dev);
3873
3874 if (!port->phylink)
3875 return -ENOTSUPP;
3876
3877 return phylink_ethtool_ksettings_get(port->phylink, cmd);
3878}
3879
3880static int mvpp2_ethtool_set_link_ksettings(struct net_device *dev,
3881 const struct ethtool_link_ksettings *cmd)
3882{
3883 struct mvpp2_port *port = netdev_priv(dev);
3884
3885 if (!port->phylink)
3886 return -ENOTSUPP;
3887
3888 return phylink_ethtool_ksettings_set(port->phylink, cmd);
3889}
3890
8179642b
AT		 3891static int mvpp2_ethtool_get_rxnfc(struct net_device *dev,
3892 struct ethtool_rxnfc *info, u32 *rules)
3893{
3894 struct mvpp2_port *port = netdev_priv(dev);
436d4fdb 3895 int ret = 0;
8179642b
AT		 3896
3897 if (!mvpp22_rss_is_supported())
3898 return -EOPNOTSUPP;
3899
3900 switch (info->cmd) {
436d4fdb
MC		 3901 case ETHTOOL_GRXFH:
3902 ret = mvpp2_ethtool_rxfh_get(port, info);
3903 break;
8179642b
AT		 3904 case ETHTOOL_GRXRINGS:
3905 info->data = port->nrxqs;
3906 break;
3907 default:
3908 return -ENOTSUPP;
3909 }
3910
436d4fdb
MC		 3911 return ret;
3912}
3913
3914static int mvpp2_ethtool_set_rxnfc(struct net_device *dev,
3915 struct ethtool_rxnfc *info)
3916{
3917 struct mvpp2_port *port = netdev_priv(dev);
3918 int ret = 0;
3919
3920 if (!mvpp22_rss_is_supported())
3921 return -EOPNOTSUPP;
3922
3923 switch (info->cmd) {
3924 case ETHTOOL_SRXFH:
3925 ret = mvpp2_ethtool_rxfh_set(port, info);
3926 break;
3927 default:
3928 return -EOPNOTSUPP;
3929 }
3930 return ret;
8179642b
AT		 3931}
3932
3933static u32 mvpp2_ethtool_get_rxfh_indir_size(struct net_device *dev)
3934{
3935 return mvpp22_rss_is_supported() ? MVPP22_RSS_TABLE_ENTRIES : 0;
3936}
3937
3938static int mvpp2_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
3939 u8 *hfunc)
3940{
3941 struct mvpp2_port *port = netdev_priv(dev);
3942
3943 if (!mvpp22_rss_is_supported())
3944 return -EOPNOTSUPP;
3945
3946 if (indir)
3947 memcpy(indir, port->indir,
3948 ARRAY_SIZE(port->indir) * sizeof(port->indir[0]));
3949
3950 if (hfunc)
3951 *hfunc = ETH_RSS_HASH_CRC32;
3952
3953 return 0;
3954}
3955
3956static int mvpp2_ethtool_set_rxfh(struct net_device *dev, const u32 *indir,
3957 const u8 *key, const u8 hfunc)
3958{
3959 struct mvpp2_port *port = netdev_priv(dev);
3960
3961 if (!mvpp22_rss_is_supported())
3962 return -EOPNOTSUPP;
3963
3964 if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_CRC32)
3965 return -EOPNOTSUPP;
3966
3967 if (key)
3968 return -EOPNOTSUPP;
3969
3970 if (indir) {
3971 memcpy(port->indir, indir,
3972 ARRAY_SIZE(port->indir) * sizeof(port->indir[0]));
3973 mvpp22_rss_fill_table(port, port->id);
3974 }
3975
3976 return 0;
3977}
3978
3f518509
MW		 3979/* Device ops */
3980
3981static const struct net_device_ops mvpp2_netdev_ops = {
3982 .ndo_open = mvpp2_open,
3983 .ndo_stop = mvpp2_stop,
3984 .ndo_start_xmit = mvpp2_tx,
3985 .ndo_set_rx_mode = mvpp2_set_rx_mode,
3986 .ndo_set_mac_address = mvpp2_set_mac_address,
3987 .ndo_change_mtu = mvpp2_change_mtu,
3988 .ndo_get_stats64 = mvpp2_get_stats64,
bd695a5f 3989 .ndo_do_ioctl = mvpp2_ioctl,
56beda3d
MC		 3990 .ndo_vlan_rx_add_vid = mvpp2_vlan_rx_add_vid,
3991 .ndo_vlan_rx_kill_vid = mvpp2_vlan_rx_kill_vid,
3992 .ndo_set_features = mvpp2_set_features,
3f518509
MW		 3993};
3994
3995static const struct ethtool_ops mvpp2_eth_tool_ops = {
4bb04326 3996 .nway_reset = mvpp2_ethtool_nway_reset,
dcd3e73a
AT		 3997 .get_link = ethtool_op_get_link,
3998 .set_coalesce = mvpp2_ethtool_set_coalesce,
3999 .get_coalesce = mvpp2_ethtool_get_coalesce,
4000 .get_drvinfo = mvpp2_ethtool_get_drvinfo,
4001 .get_ringparam = mvpp2_ethtool_get_ringparam,
4002 .set_ringparam = mvpp2_ethtool_set_ringparam,
4003 .get_strings = mvpp2_ethtool_get_strings,
4004 .get_ethtool_stats = mvpp2_ethtool_get_stats,
4005 .get_sset_count = mvpp2_ethtool_get_sset_count,
4bb04326
AT		 4006 .get_pauseparam = mvpp2_ethtool_get_pause_param,
4007 .set_pauseparam = mvpp2_ethtool_set_pause_param,
4008 .get_link_ksettings = mvpp2_ethtool_get_link_ksettings,
4009 .set_link_ksettings = mvpp2_ethtool_set_link_ksettings,
8179642b 4010 .get_rxnfc = mvpp2_ethtool_get_rxnfc,
436d4fdb 4011 .set_rxnfc = mvpp2_ethtool_set_rxnfc,
8179642b
AT		 4012 .get_rxfh_indir_size = mvpp2_ethtool_get_rxfh_indir_size,
4013 .get_rxfh = mvpp2_ethtool_get_rxfh,
4014 .set_rxfh = mvpp2_ethtool_set_rxfh,
4015
3f518509
MW		 4016};
4017
213f428f
TP		 4018/* Used for PPv2.1, or PPv2.2 with the old Device Tree binding that
4019 * had a single IRQ defined per-port.
4020 */
4021static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port,
4022 struct device_node *port_node)
591f4cfa
TP		 4023{
4024 struct mvpp2_queue_vector *v = &port->qvecs[0];
4025
4026 v->first_rxq = 0;
4027 v->nrxqs = port->nrxqs;
4028 v->type = MVPP2_QUEUE_VECTOR_SHARED;
4029 v->sw_thread_id = 0;
4030 v->sw_thread_mask = *cpumask_bits(cpu_online_mask);
4031 v->port = port;
4032 v->irq = irq_of_parse_and_map(port_node, 0);
4033 if (v->irq <= 0)
4034 return -EINVAL;
4035 netif_napi_add(port->dev, &v->napi, mvpp2_poll,
4036 NAPI_POLL_WEIGHT);
4037
4038 port->nqvecs = 1;
4039
4040 return 0;
4041}
4042
213f428f
TP		 4043static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
4044 struct device_node *port_node)
4045{
e531f767 4046 struct mvpp2 *priv = port->priv;
213f428f
TP		 4047 struct mvpp2_queue_vector *v;
4048 int i, ret;
4049
e531f767
AT		 4050 switch (queue_mode) {
4051 case MVPP2_QDIST_SINGLE_MODE:
4052 port->nqvecs = priv->nthreads + 1;
4053 break;
4054 case MVPP2_QDIST_MULTI_MODE:
4055 port->nqvecs = priv->nthreads;
4056 break;
4057 }
213f428f
TP		 4058
4059 for (i = 0; i < port->nqvecs; i++) {
4060 char irqname[16];
4061
4062 v = port->qvecs + i;
4063
4064 v->port = port;
4065 v->type = MVPP2_QUEUE_VECTOR_PRIVATE;
4066 v->sw_thread_id = i;
4067 v->sw_thread_mask = BIT(i);
4068
a9aac385
AT		 4069 if (port->flags & MVPP2_F_DT_COMPAT)
4070 snprintf(irqname, sizeof(irqname), "tx-cpu%d", i);
4071 else
4072 snprintf(irqname, sizeof(irqname), "hif%d", i);
213f428f
TP		 4073
4074 if (queue_mode == MVPP2_QDIST_MULTI_MODE) {
4075 v->first_rxq = i * MVPP2_DEFAULT_RXQ;
4076 v->nrxqs = MVPP2_DEFAULT_RXQ;
4077 } else if (queue_mode == MVPP2_QDIST_SINGLE_MODE &&
4078 i == (port->nqvecs - 1)) {
4079 v->first_rxq = 0;
4080 v->nrxqs = port->nrxqs;
4081 v->type = MVPP2_QUEUE_VECTOR_SHARED;
a9aac385
AT		 4082
4083 if (port->flags & MVPP2_F_DT_COMPAT)
4084 strncpy(irqname, "rx-shared", sizeof(irqname));
213f428f
TP		 4085 }
4086
a75edc7c
MW		 4087 if (port_node)
4088 v->irq = of_irq_get_byname(port_node, irqname);
4089 else
4090 v->irq = fwnode_irq_get(port->fwnode, i);
213f428f
TP		 4091 if (v->irq <= 0) {
4092 ret = -EINVAL;
4093 goto err;
4094 }
4095
4096 netif_napi_add(port->dev, &v->napi, mvpp2_poll,
4097 NAPI_POLL_WEIGHT);
4098 }
4099
4100 return 0;
4101
4102err:
4103 for (i = 0; i < port->nqvecs; i++)
4104 irq_dispose_mapping(port->qvecs[i].irq);
4105 return ret;
4106}
4107
4108static int mvpp2_queue_vectors_init(struct mvpp2_port *port,
4109 struct device_node *port_node)
4110{
4111 if (port->has_tx_irqs)
4112 return mvpp2_multi_queue_vectors_init(port, port_node);
4113 else
4114 return mvpp2_simple_queue_vectors_init(port, port_node);
4115}
4116
591f4cfa
TP		 4117static void mvpp2_queue_vectors_deinit(struct mvpp2_port *port)
4118{
4119 int i;
4120
4121 for (i = 0; i < port->nqvecs; i++)
4122 irq_dispose_mapping(port->qvecs[i].irq);
4123}
4124
4125/* Configure Rx queue group interrupt for this port */
4126static void mvpp2_rx_irqs_setup(struct mvpp2_port *port)
4127{
4128 struct mvpp2 *priv = port->priv;
4129 u32 val;
4130 int i;
4131
4132 if (priv->hw_version == MVPP21) {
4133 mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id),
4134 port->nrxqs);
4135 return;
4136 }
4137
4138 /* Handle the more complicated PPv2.2 case */
4139 for (i = 0; i < port->nqvecs; i++) {
4140 struct mvpp2_queue_vector *qv = port->qvecs + i;
4141
4142 if (!qv->nrxqs)
4143 continue;
4144
4145 val = qv->sw_thread_id;
4146 val |= port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET;
4147 mvpp2_write(priv, MVPP22_ISR_RXQ_GROUP_INDEX_REG, val);
4148
4149 val = qv->first_rxq;
4150 val |= qv->nrxqs << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET;
4151 mvpp2_write(priv, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG, val);
4152 }
4153}
4154
3f518509
MW		 4155/* Initialize port HW */
4156static int mvpp2_port_init(struct mvpp2_port *port)
4157{
4158 struct device *dev = port->dev->dev.parent;
4159 struct mvpp2 *priv = port->priv;
4160 struct mvpp2_txq_pcpu *txq_pcpu;
074c74df 4161 unsigned int thread;
850623b3 4162 int queue, err;
3f518509 4163
09f83975
TP		 4164 /* Checks for hardware constraints */
4165 if (port->first_rxq + port->nrxqs >
59b9a31e 4166 MVPP2_MAX_PORTS * priv->max_port_rxqs)
3f518509
MW		 4167 return -EINVAL;
4168
790d32c6
MC		 4169 if (port->nrxqs % MVPP2_DEFAULT_RXQ ||
4170 port->nrxqs > priv->max_port_rxqs || port->ntxqs > MVPP2_MAX_TXQ)
09f83975
TP		 4171 return -EINVAL;
4172
3f518509
MW		 4173 /* Disable port */
4174 mvpp2_egress_disable(port);
4175 mvpp2_port_disable(port);
4176
213f428f
TP		 4177 port->tx_time_coal = MVPP2_TXDONE_COAL_USEC;
4178
09f83975 4179 port->txqs = devm_kcalloc(dev, port->ntxqs, sizeof(*port->txqs),
3f518509
MW		 4180 GFP_KERNEL);
4181 if (!port->txqs)
4182 return -ENOMEM;
4183
4184 /* Associate physical Tx queues to this port and initialize.
4185 * The mapping is predefined.
4186 */
09f83975 4187 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 4188 int queue_phy_id = mvpp2_txq_phys(port->id, queue);
4189 struct mvpp2_tx_queue *txq;
4190
4191 txq = devm_kzalloc(dev, sizeof(*txq), GFP_KERNEL);
177c8d1c
CJ		 4192 if (!txq) {
4193 err = -ENOMEM;
4194 goto err_free_percpu;
4195 }
3f518509
MW		 4196
4197 txq->pcpu = alloc_percpu(struct mvpp2_txq_pcpu);
4198 if (!txq->pcpu) {
4199 err = -ENOMEM;
4200 goto err_free_percpu;
4201 }
4202
4203 txq->id = queue_phy_id;
4204 txq->log_id = queue;
4205 txq->done_pkts_coal = MVPP2_TXDONE_COAL_PKTS_THRESH;
e531f767 4206 for (thread = 0; thread < priv->nthreads; thread++) {
074c74df
AT		 4207 txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
4208 txq_pcpu->thread = thread;
3f518509
MW		 4209 }
4210
4211 port->txqs[queue] = txq;
4212 }
4213
09f83975 4214 port->rxqs = devm_kcalloc(dev, port->nrxqs, sizeof(*port->rxqs),
3f518509
MW		 4215 GFP_KERNEL);
4216 if (!port->rxqs) {
4217 err = -ENOMEM;
4218 goto err_free_percpu;
4219 }
4220
4221 /* Allocate and initialize Rx queue for this port */
09f83975 4222 for (queue = 0; queue < port->nrxqs; queue++) {
3f518509
MW		 4223 struct mvpp2_rx_queue *rxq;
4224
4225 /* Map physical Rx queue to port's logical Rx queue */
4226 rxq = devm_kzalloc(dev, sizeof(*rxq), GFP_KERNEL);
d82b0c21
JZ		 4227 if (!rxq) {
4228 err = -ENOMEM;
3f518509 4229 goto err_free_percpu;
d82b0c21 4230 }
3f518509
MW		 4231 /* Map this Rx queue to a physical queue */
4232 rxq->id = port->first_rxq + queue;
4233 rxq->port = port->id;
4234 rxq->logic_rxq = queue;
4235
4236 port->rxqs[queue] = rxq;
4237 }
4238
591f4cfa 4239 mvpp2_rx_irqs_setup(port);
3f518509
MW		 4240
4241 /* Create Rx descriptor rings */
09f83975 4242 for (queue = 0; queue < port->nrxqs; queue++) {
3f518509
MW		 4243 struct mvpp2_rx_queue *rxq = port->rxqs[queue];
4244
4245 rxq->size = port->rx_ring_size;
4246 rxq->pkts_coal = MVPP2_RX_COAL_PKTS;
4247 rxq->time_coal = MVPP2_RX_COAL_USEC;
4248 }
4249
4250 mvpp2_ingress_disable(port);
4251
4252 /* Port default configuration */
4253 mvpp2_defaults_set(port);
4254
4255 /* Port's classifier configuration */
4256 mvpp2_cls_oversize_rxq_set(port);
4257 mvpp2_cls_port_config(port);
4258
e6e21c02
MC		 4259 if (mvpp22_rss_is_supported())
4260 mvpp22_rss_port_init(port);
4261
3f518509
MW		 4262 /* Provide an initial Rx packet size */
4263 port->pkt_size = MVPP2_RX_PKT_SIZE(port->dev->mtu);
4264
4265 /* Initialize pools for swf */
4266 err = mvpp2_swf_bm_pool_init(port);
4267 if (err)
4268 goto err_free_percpu;
4269
4270 return 0;
4271
4272err_free_percpu:
09f83975 4273 for (queue = 0; queue < port->ntxqs; queue++) {
3f518509
MW		 4274 if (!port->txqs[queue])
4275 continue;
4276 free_percpu(port->txqs[queue]->pcpu);
4277 }
4278 return err;
4279}
4280
a9aac385
AT		 4281static bool mvpp22_port_has_legacy_tx_irqs(struct device_node *port_node,
4282 unsigned long *flags)
4283{
4284 char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1", "tx-cpu2",
4285 "tx-cpu3" };
4286 int i;
4287
4288 for (i = 0; i < 5; i++)
4289 if (of_property_match_string(port_node, "interrupt-names",
4290 irqs[i]) < 0)
4291 return false;
4292
4293 *flags |= MVPP2_F_DT_COMPAT;
4294 return true;
4295}
4296
4297/* Checks if the port dt description has the required Tx interrupts:
4298 * - PPv2.1: there are no such interrupts.
4299 * - PPv2.2:
4300 * - The old DTs have: "rx-shared", "tx-cpuX" with X in [0...3]
4301 * - The new ones have: "hifX" with X in [0..8]
4302 *
4303 * All those variants are supported to keep the backward compatibility.
213f428f 4304 */
a9aac385
AT		 4305static bool mvpp2_port_has_irqs(struct mvpp2 *priv,
4306 struct device_node *port_node,
4307 unsigned long *flags)
213f428f 4308{
a9aac385
AT		 4309 char name[5];
4310 int i;
213f428f 4311
fd4a1056
AT		 4312 /* ACPI */
4313 if (!port_node)
4314 return true;
4315
213f428f
TP		 4316 if (priv->hw_version == MVPP21)
4317 return false;
4318
a9aac385
AT		 4319 if (mvpp22_port_has_legacy_tx_irqs(port_node, flags))
4320 return true;
4321
4322 for (i = 0; i < MVPP2_MAX_THREADS; i++) {
4323 snprintf(name, 5, "hif%d", i);
4324 if (of_property_match_string(port_node, "interrupt-names",
4325 name) < 0)
213f428f
TP		 4326 return false;
4327 }
4328
4329 return true;
4330}
4331
3ba8c81e 4332static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
24812221 4333 struct fwnode_handle *fwnode,
3ba8c81e
AT		 4334 char **mac_from)
4335{
4336 struct mvpp2_port *port = netdev_priv(dev);
4337 char hw_mac_addr[ETH_ALEN] = {0};
24812221 4338 char fw_mac_addr[ETH_ALEN];
3ba8c81e 4339
24812221
MW		 4340 if (fwnode_get_mac_address(fwnode, fw_mac_addr, ETH_ALEN)) {
4341 *mac_from = "firmware node";
4342 ether_addr_copy(dev->dev_addr, fw_mac_addr);
688cbaf2
AT		 4343 return;
4344 }
d2a6e48e 4345
688cbaf2
AT		 4346 if (priv->hw_version == MVPP21) {
4347 mvpp21_get_mac_address(port, hw_mac_addr);
4348 if (is_valid_ether_addr(hw_mac_addr)) {
4349 *mac_from = "hardware";
4350 ether_addr_copy(dev->dev_addr, hw_mac_addr);
4351 return;
4352 }
3ba8c81e 4353 }
688cbaf2
AT		 4354
4355 *mac_from = "random";
4356 eth_hw_addr_random(dev);
3ba8c81e
AT		 4357}
4358
4bb04326
AT		 4359static void mvpp2_phylink_validate(struct net_device *dev,
4360 unsigned long *supported,
4361 struct phylink_link_state *state)
4362{
0fb628f0 4363 struct mvpp2_port *port = netdev_priv(dev);
4bb04326
AT		 4364 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
4365
0fb628f0
BS		 4366 /* Invalid combinations */
4367 switch (state->interface) {
4368 case PHY_INTERFACE_MODE_10GKR:
4369 case PHY_INTERFACE_MODE_XAUI:
4370 if (port->gop_id != 0)
4371 goto empty_set;
4372 break;
4373 case PHY_INTERFACE_MODE_RGMII:
4374 case PHY_INTERFACE_MODE_RGMII_ID:
4375 case PHY_INTERFACE_MODE_RGMII_RXID:
4376 case PHY_INTERFACE_MODE_RGMII_TXID:
4377 if (port->gop_id == 0)
4378 goto empty_set;
4379 break;
4380 default:
4381 break;
4382 }
4383
4bb04326
AT		 4384 phylink_set(mask, Autoneg);
4385 phylink_set_port_modes(mask);
4386 phylink_set(mask, Pause);
4387 phylink_set(mask, Asym_Pause);
4388
d97c9f4a
AT		 4389 switch (state->interface) {
4390 case PHY_INTERFACE_MODE_10GKR:
0fb628f0 4391 case PHY_INTERFACE_MODE_XAUI:
01b3fd5a 4392 case PHY_INTERFACE_MODE_NA:
00679177 4393 if (port->gop_id == 0) {
1b451fb2 4394 phylink_set(mask, 10000baseT_Full);
00679177
AT		 4395 phylink_set(mask, 10000baseCR_Full);
4396 phylink_set(mask, 10000baseSR_Full);
4397 phylink_set(mask, 10000baseLR_Full);
4398 phylink_set(mask, 10000baseLRM_Full);
4399 phylink_set(mask, 10000baseER_Full);
4400 phylink_set(mask, 10000baseKR_Full);
4401 }
d97c9f4a 4402 /* Fall-through */
0fb628f0
BS		 4403 case PHY_INTERFACE_MODE_RGMII:
4404 case PHY_INTERFACE_MODE_RGMII_ID:
4405 case PHY_INTERFACE_MODE_RGMII_RXID:
4406 case PHY_INTERFACE_MODE_RGMII_TXID:
4407 case PHY_INTERFACE_MODE_SGMII:
d97c9f4a
AT		 4408 phylink_set(mask, 10baseT_Half);
4409 phylink_set(mask, 10baseT_Full);
4410 phylink_set(mask, 100baseT_Half);
4411 phylink_set(mask, 100baseT_Full);
d97c9f4a
AT		 4412 /* Fall-through */
4413 case PHY_INTERFACE_MODE_1000BASEX:
a6fe31de 4414 case PHY_INTERFACE_MODE_2500BASEX:
d97c9f4a
AT		 4415 phylink_set(mask, 1000baseT_Full);
4416 phylink_set(mask, 1000baseX_Full);
a6fe31de 4417 phylink_set(mask, 2500baseX_Full);
0fb628f0
BS		 4418 break;
4419 default:
4420 goto empty_set;
4bb04326
AT		 4421 }
4422
4423 bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS);
4424 bitmap_and(state->advertising, state->advertising, mask,
4425 __ETHTOOL_LINK_MODE_MASK_NBITS);
0fb628f0
BS		 4426 return;
4427
4428empty_set:
4429 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
4bb04326
AT		 4430}
4431
4432static void mvpp22_xlg_link_state(struct mvpp2_port *port,
4433 struct phylink_link_state *state)
4434{
4435 u32 val;
4436
4437 state->speed = SPEED_10000;
4438 state->duplex = 1;
4439 state->an_complete = 1;
4440
4441 val = readl(port->base + MVPP22_XLG_STATUS);
4442 state->link = !!(val & MVPP22_XLG_STATUS_LINK_UP);
4443
4444 state->pause = 0;
4445 val = readl(port->base + MVPP22_XLG_CTRL0_REG);
4446 if (val & MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN)
4447 state->pause |= MLO_PAUSE_TX;
4448 if (val & MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN)
4449 state->pause |= MLO_PAUSE_RX;
4450}
4451
4452static void mvpp2_gmac_link_state(struct mvpp2_port *port,
4453 struct phylink_link_state *state)
4454{
4455 u32 val;
4456
4457 val = readl(port->base + MVPP2_GMAC_STATUS0);
4458
4459 state->an_complete = !!(val & MVPP2_GMAC_STATUS0_AN_COMPLETE);
4460 state->link = !!(val & MVPP2_GMAC_STATUS0_LINK_UP);
4461 state->duplex = !!(val & MVPP2_GMAC_STATUS0_FULL_DUPLEX);
4462
d97c9f4a
AT		 4463 switch (port->phy_interface) {
4464 case PHY_INTERFACE_MODE_1000BASEX:
4bb04326 4465 state->speed = SPEED_1000;
d97c9f4a 4466 break;
a6fe31de
AT		 4467 case PHY_INTERFACE_MODE_2500BASEX:
4468 state->speed = SPEED_2500;
4469 break;
d97c9f4a
AT		 4470 default:
4471 if (val & MVPP2_GMAC_STATUS0_GMII_SPEED)
4472 state->speed = SPEED_1000;
4473 else if (val & MVPP2_GMAC_STATUS0_MII_SPEED)
4474 state->speed = SPEED_100;
4475 else
4476 state->speed = SPEED_10;
4477 }
4bb04326
AT		 4478
4479 state->pause = 0;
4480 if (val & MVPP2_GMAC_STATUS0_RX_PAUSE)
4481 state->pause |= MLO_PAUSE_RX;
4482 if (val & MVPP2_GMAC_STATUS0_TX_PAUSE)
4483 state->pause |= MLO_PAUSE_TX;
4484}
4485
4486static int mvpp2_phylink_mac_link_state(struct net_device *dev,
4487 struct phylink_link_state *state)
4488{
4489 struct mvpp2_port *port = netdev_priv(dev);
4490
4491 if (port->priv->hw_version == MVPP22 && port->gop_id == 0) {
4492 u32 mode = readl(port->base + MVPP22_XLG_CTRL3_REG);
4493 mode &= MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
4494
4495 if (mode == MVPP22_XLG_CTRL3_MACMODESELECT_10G) {
4496 mvpp22_xlg_link_state(port, state);
4497 return 1;
4498 }
4499 }
4500
4501 mvpp2_gmac_link_state(port, state);
4502 return 1;
4503}
4504
4505static void mvpp2_mac_an_restart(struct net_device *dev)
4506{
4507 struct mvpp2_port *port = netdev_priv(dev);
4508 u32 val;
4509
4510 if (port->phy_interface != PHY_INTERFACE_MODE_SGMII)
4511 return;
4512
4513 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4514 /* The RESTART_AN bit is cleared by the h/w after restarting the AN
4515 * process.
4516 */
4517 val |= MVPP2_GMAC_IN_BAND_RESTART_AN | MVPP2_GMAC_IN_BAND_AUTONEG;
4518 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4519}
4520
4521static void mvpp2_xlg_config(struct mvpp2_port *port, unsigned int mode,
4522 const struct phylink_link_state *state)
4523{
4524 u32 ctrl0, ctrl4;
4525
4526 ctrl0 = readl(port->base + MVPP22_XLG_CTRL0_REG);
4527 ctrl4 = readl(port->base + MVPP22_XLG_CTRL4_REG);
4528
4529 if (state->pause & MLO_PAUSE_TX)
4530 ctrl0 |= MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN;
4531 if (state->pause & MLO_PAUSE_RX)
4532 ctrl0 |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
4533
4534 ctrl4 &= ~MVPP22_XLG_CTRL4_MACMODSELECT_GMAC;
4535 ctrl4 |= MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC |
4536 MVPP22_XLG_CTRL4_EN_IDLE_CHECK;
4537
4538 writel(ctrl0, port->base + MVPP22_XLG_CTRL0_REG);
4539 writel(ctrl4, port->base + MVPP22_XLG_CTRL4_REG);
4540}
4541
4542static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
4543 const struct phylink_link_state *state)
4544{
4545 u32 an, ctrl0, ctrl2, ctrl4;
4546
4547 an = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4548 ctrl0 = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
4549 ctrl2 = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
4550 ctrl4 = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
4551
4552 /* Force link down */
4553 an &= ~MVPP2_GMAC_FORCE_LINK_PASS;
4554 an |= MVPP2_GMAC_FORCE_LINK_DOWN;
4555 writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4556
4557 /* Set the GMAC in a reset state */
4558 ctrl2 |= MVPP2_GMAC_PORT_RESET_MASK;
4559 writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG);
4560
4561 an &= ~(MVPP2_GMAC_CONFIG_MII_SPEED | MVPP2_GMAC_CONFIG_GMII_SPEED |
4562 MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FC_ADV_EN |
4563 MVPP2_GMAC_FC_ADV_ASM_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG |
4564 MVPP2_GMAC_CONFIG_FULL_DUPLEX | MVPP2_GMAC_AN_DUPLEX_EN |
4565 MVPP2_GMAC_FORCE_LINK_DOWN);
4566 ctrl0 &= ~MVPP2_GMAC_PORT_TYPE_MASK;
4567 ctrl2 &= ~(MVPP2_GMAC_PORT_RESET_MASK | MVPP2_GMAC_PCS_ENABLE_MASK);
4568
a6fe31de
AT		 4569 if (state->interface == PHY_INTERFACE_MODE_1000BASEX ||
4570 state->interface == PHY_INTERFACE_MODE_2500BASEX) {
4571 /* 1000BaseX and 2500BaseX ports cannot negotiate speed nor can
4572 * they negotiate duplex: they are always operating with a fixed
4573 * speed of 1000/2500Mbps in full duplex, so force 1000/2500
4574 * speed and full duplex here.
d97c9f4a
AT		 4575 */
4576 ctrl0 |= MVPP2_GMAC_PORT_TYPE_MASK;
4577 an |= MVPP2_GMAC_CONFIG_GMII_SPEED |
4578 MVPP2_GMAC_CONFIG_FULL_DUPLEX;
4579 } else if (!phy_interface_mode_is_rgmii(state->interface)) {
4bb04326 4580 an |= MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG;
d97c9f4a 4581 }
4bb04326
AT		 4582
4583 if (state->duplex)
4584 an |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
4585 if (phylink_test(state->advertising, Pause))
4586 an |= MVPP2_GMAC_FC_ADV_EN;
4587 if (phylink_test(state->advertising, Asym_Pause))
4588 an |= MVPP2_GMAC_FC_ADV_ASM_EN;
4589
d97c9f4a 4590 if (state->interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 4591 state->interface == PHY_INTERFACE_MODE_1000BASEX ||
4592 state->interface == PHY_INTERFACE_MODE_2500BASEX) {
4bb04326
AT		 4593 an |= MVPP2_GMAC_IN_BAND_AUTONEG;
4594 ctrl2 |= MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PCS_ENABLE_MASK;
4595
4596 ctrl4 &= ~(MVPP22_CTRL4_EXT_PIN_GMII_SEL |
4597 MVPP22_CTRL4_RX_FC_EN | MVPP22_CTRL4_TX_FC_EN);
4598 ctrl4 |= MVPP22_CTRL4_SYNC_BYPASS_DIS |
4599 MVPP22_CTRL4_DP_CLK_SEL |
4600 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
4601
4602 if (state->pause & MLO_PAUSE_TX)
4603 ctrl4 |= MVPP22_CTRL4_TX_FC_EN;
4604 if (state->pause & MLO_PAUSE_RX)
4605 ctrl4 |= MVPP22_CTRL4_RX_FC_EN;
4606 } else if (phy_interface_mode_is_rgmii(state->interface)) {
4607 an |= MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS;
4608
4609 if (state->speed == SPEED_1000)
4610 an |= MVPP2_GMAC_CONFIG_GMII_SPEED;
4611 else if (state->speed == SPEED_100)
4612 an |= MVPP2_GMAC_CONFIG_MII_SPEED;
4613
4614 ctrl4 &= ~MVPP22_CTRL4_DP_CLK_SEL;
4615 ctrl4 |= MVPP22_CTRL4_EXT_PIN_GMII_SEL |
4616 MVPP22_CTRL4_SYNC_BYPASS_DIS |
4617 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
4618 }
4619
4620 writel(ctrl0, port->base + MVPP2_GMAC_CTRL_0_REG);
4621 writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG);
4622 writel(ctrl4, port->base + MVPP22_GMAC_CTRL_4_REG);
4623 writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4624}
4625
4626static void mvpp2_mac_config(struct net_device *dev, unsigned int mode,
4627 const struct phylink_link_state *state)
4628{
4629 struct mvpp2_port *port = netdev_priv(dev);
4630
4631 /* Check for invalid configuration */
4632 if (state->interface == PHY_INTERFACE_MODE_10GKR && port->gop_id != 0) {
4633 netdev_err(dev, "Invalid mode on %s\n", dev->name);
4634 return;
4635 }
4636
4bb04326
AT		 4637 /* Make sure the port is disabled when reconfiguring the mode */
4638 mvpp2_port_disable(port);
4639
4640 if (port->priv->hw_version == MVPP22 &&
4641 port->phy_interface != state->interface) {
4642 port->phy_interface = state->interface;
4643
4644 /* Reconfigure the serdes lanes */
4645 phy_power_off(port->comphy);
4646 mvpp22_mode_reconfigure(port);
4647 }
4648
4649 /* mac (re)configuration */
4650 if (state->interface == PHY_INTERFACE_MODE_10GKR)
4651 mvpp2_xlg_config(port, mode, state);
4652 else if (phy_interface_mode_is_rgmii(state->interface) ||
d97c9f4a 4653 state->interface == PHY_INTERFACE_MODE_SGMII ||
a6fe31de
AT		 4654 state->interface == PHY_INTERFACE_MODE_1000BASEX ||
4655 state->interface == PHY_INTERFACE_MODE_2500BASEX)
4bb04326
AT		 4656 mvpp2_gmac_config(port, mode, state);
4657
4658 if (port->priv->hw_version == MVPP21 && port->flags & MVPP2_F_LOOPBACK)
4659 mvpp2_port_loopback_set(port, state);
4660
41948ccb 4661 mvpp2_port_enable(port);
4bb04326
AT		 4662}
4663
4664static void mvpp2_mac_link_up(struct net_device *dev, unsigned int mode,
4665 phy_interface_t interface, struct phy_device *phy)
4666{
4667 struct mvpp2_port *port = netdev_priv(dev);
4668 u32 val;
4669
4670 if (!phylink_autoneg_inband(mode) &&
4671 interface != PHY_INTERFACE_MODE_10GKR) {
4672 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4673 val &= ~MVPP2_GMAC_FORCE_LINK_DOWN;
4674 if (phy_interface_mode_is_rgmii(interface))
4675 val |= MVPP2_GMAC_FORCE_LINK_PASS;
4676 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4677 }
4678
4679 mvpp2_port_enable(port);
4680
4681 mvpp2_egress_enable(port);
4682 mvpp2_ingress_enable(port);
4683 netif_tx_wake_all_queues(dev);
4684}
4685
4686static void mvpp2_mac_link_down(struct net_device *dev, unsigned int mode,
4687 phy_interface_t interface)
4688{
4689 struct mvpp2_port *port = netdev_priv(dev);
4690 u32 val;
4691
4692 if (!phylink_autoneg_inband(mode) &&
4693 interface != PHY_INTERFACE_MODE_10GKR) {
4694 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4695 val &= ~MVPP2_GMAC_FORCE_LINK_PASS;
4696 val |= MVPP2_GMAC_FORCE_LINK_DOWN;
4697 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
4698 }
4699
4700 netif_tx_stop_all_queues(dev);
4701 mvpp2_egress_disable(port);
4702 mvpp2_ingress_disable(port);
4703
4704 /* When using link interrupts to notify phylink of a MAC state change,
4705 * we do not want the port to be disabled (we want to receive further
4706 * interrupts, to be notified when the port will have a link later).
4707 */
4708 if (!port->has_phy)
4709 return;
4710
4711 mvpp2_port_disable(port);
4712}
4713
4714static const struct phylink_mac_ops mvpp2_phylink_ops = {
4715 .validate = mvpp2_phylink_validate,
4716 .mac_link_state = mvpp2_phylink_mac_link_state,
4717 .mac_an_restart = mvpp2_mac_an_restart,
4718 .mac_config = mvpp2_mac_config,
4719 .mac_link_up = mvpp2_mac_link_up,
4720 .mac_link_down = mvpp2_mac_link_down,
4721};
4722
3f518509
MW		 4723/* Ports initialization */
4724static int mvpp2_port_probe(struct platform_device *pdev,
24812221 4725 struct fwnode_handle *port_fwnode,
bf147153 4726 struct mvpp2 *priv)
3f518509 4727{
a75edc7c 4728 struct phy *comphy = NULL;
3f518509 4729 struct mvpp2_port *port;
edc660fa 4730 struct mvpp2_port_pcpu *port_pcpu;
24812221 4731 struct device_node *port_node = to_of_node(port_fwnode);
3f518509
MW		 4732 struct net_device *dev;
4733 struct resource *res;
4bb04326 4734 struct phylink *phylink;
3ba8c81e 4735 char *mac_from = "";
074c74df 4736 unsigned int ntxqs, nrxqs, thread;
a9aac385 4737 unsigned long flags = 0;
213f428f 4738 bool has_tx_irqs;
3f518509
MW		 4739 u32 id;
4740 int features;
4741 int phy_mode;
850623b3 4742 int err, i;
3f518509 4743
fd4a1056
AT		 4744 has_tx_irqs = mvpp2_port_has_irqs(priv, port_node, &flags);
4745 if (!has_tx_irqs && queue_mode == MVPP2_QDIST_MULTI_MODE) {
4746 dev_err(&pdev->dev,
4747 "not enough IRQs to support multi queue mode\n");
4748 return -EINVAL;
a75edc7c 4749 }
213f428f 4750
09f83975 4751 ntxqs = MVPP2_MAX_TXQ;
213f428f
TP		 4752 if (priv->hw_version == MVPP22 && queue_mode == MVPP2_QDIST_MULTI_MODE)
4753 nrxqs = MVPP2_DEFAULT_RXQ * num_possible_cpus();
4754 else
4755 nrxqs = MVPP2_DEFAULT_RXQ;
09f83975
TP		 4756
4757 dev = alloc_etherdev_mqs(sizeof(*port), ntxqs, nrxqs);
3f518509
MW		 4758 if (!dev)
4759 return -ENOMEM;
4760
24812221 4761 phy_mode = fwnode_get_phy_mode(port_fwnode);
3f518509
MW		 4762 if (phy_mode < 0) {
4763 dev_err(&pdev->dev, "incorrect phy mode\n");
4764 err = phy_mode;
4765 goto err_free_netdev;
4766 }
4767
a75edc7c
MW		 4768 if (port_node) {
4769 comphy = devm_of_phy_get(&pdev->dev, port_node, NULL);
4770 if (IS_ERR(comphy)) {
4771 if (PTR_ERR(comphy) == -EPROBE_DEFER) {
4772 err = -EPROBE_DEFER;
4773 goto err_free_netdev;
4774 }
4775 comphy = NULL;
542897d9 4776 }
542897d9
AT		 4777 }
4778
24812221 4779 if (fwnode_property_read_u32(port_fwnode, "port-id", &id)) {
3f518509
MW		 4780 err = -EINVAL;
4781 dev_err(&pdev->dev, "missing port-id value\n");
4782 goto err_free_netdev;
4783 }
4784
7cf87e4a 4785 dev->tx_queue_len = MVPP2_MAX_TXD_MAX;
3f518509
MW		 4786 dev->watchdog_timeo = 5 * HZ;
4787 dev->netdev_ops = &mvpp2_netdev_ops;
4788 dev->ethtool_ops = &mvpp2_eth_tool_ops;
4789
4790 port = netdev_priv(dev);
591f4cfa 4791 port->dev = dev;
a75edc7c 4792 port->fwnode = port_fwnode;
4bb04326 4793 port->has_phy = !!of_find_property(port_node, "phy", NULL);
09f83975
TP		 4794 port->ntxqs = ntxqs;
4795 port->nrxqs = nrxqs;
213f428f
TP		 4796 port->priv = priv;
4797 port->has_tx_irqs = has_tx_irqs;
a9aac385 4798 port->flags = flags;
3f518509 4799
591f4cfa
TP		 4800 err = mvpp2_queue_vectors_init(port, port_node);
4801 if (err)
3f518509 4802 goto err_free_netdev;
3f518509 4803
a75edc7c
MW		 4804 if (port_node)
4805 port->link_irq = of_irq_get_byname(port_node, "link");
4806 else
4807 port->link_irq = fwnode_irq_get(port_fwnode, port->nqvecs + 1);
fd3651b2
AT		 4808 if (port->link_irq == -EPROBE_DEFER) {
4809 err = -EPROBE_DEFER;
4810 goto err_deinit_qvecs;
4811 }
4812 if (port->link_irq <= 0)
4813 /* the link irq is optional */
4814 port->link_irq = 0;
4815
24812221 4816 if (fwnode_property_read_bool(port_fwnode, "marvell,loopback"))
3f518509
MW		 4817 port->flags |= MVPP2_F_LOOPBACK;
4818
3f518509 4819 port->id = id;
59b9a31e 4820 if (priv->hw_version == MVPP21)
09f83975 4821 port->first_rxq = port->id * port->nrxqs;
59b9a31e
TP		 4822 else
4823 port->first_rxq = port->id * priv->max_port_rxqs;
4824
4bb04326 4825 port->of_node = port_node;
3f518509 4826 port->phy_interface = phy_mode;
542897d9 4827 port->comphy = comphy;
3f518509 4828
a786841d
TP		 4829 if (priv->hw_version == MVPP21) {
4830 res = platform_get_resource(pdev, IORESOURCE_MEM, 2 + id);
4831 port->base = devm_ioremap_resource(&pdev->dev, res);
4832 if (IS_ERR(port->base)) {
4833 err = PTR_ERR(port->base);
fd3651b2 4834 goto err_free_irq;
a786841d 4835 }
118d6298
MR		 4836
4837 port->stats_base = port->priv->lms_base +
4838 MVPP21_MIB_COUNTERS_OFFSET +
4839 port->gop_id * MVPP21_MIB_COUNTERS_PORT_SZ;
a786841d 4840 } else {
24812221
MW		 4841 if (fwnode_property_read_u32(port_fwnode, "gop-port-id",
4842 &port->gop_id)) {
a786841d
TP		 4843 err = -EINVAL;
4844 dev_err(&pdev->dev, "missing gop-port-id value\n");
591f4cfa 4845 goto err_deinit_qvecs;
a786841d
TP		 4846 }
4847
4848 port->base = priv->iface_base + MVPP22_GMAC_BASE(port->gop_id);
118d6298
MR		 4849 port->stats_base = port->priv->iface_base +
4850 MVPP22_MIB_COUNTERS_OFFSET +
4851 port->gop_id * MVPP22_MIB_COUNTERS_PORT_SZ;
3f518509
MW		 4852 }
4853
118d6298 4854 /* Alloc per-cpu and ethtool stats */
3f518509
MW		 4855 port->stats = netdev_alloc_pcpu_stats(struct mvpp2_pcpu_stats);
4856 if (!port->stats) {
4857 err = -ENOMEM;
fd3651b2 4858 goto err_free_irq;
3f518509
MW		 4859 }
4860
118d6298
MR		 4861 port->ethtool_stats = devm_kcalloc(&pdev->dev,
4862 ARRAY_SIZE(mvpp2_ethtool_regs),
4863 sizeof(u64), GFP_KERNEL);
4864 if (!port->ethtool_stats) {
4865 err = -ENOMEM;
4866 goto err_free_stats;
4867 }
4868
e5c500eb
MR		 4869 mutex_init(&port->gather_stats_lock);
4870 INIT_DELAYED_WORK(&port->stats_work, mvpp2_gather_hw_statistics);
4871
24812221 4872 mvpp2_port_copy_mac_addr(dev, priv, port_fwnode, &mac_from);
3f518509 4873
7cf87e4a
YM		 4874 port->tx_ring_size = MVPP2_MAX_TXD_DFLT;
4875 port->rx_ring_size = MVPP2_MAX_RXD_DFLT;
3f518509
MW		 4876 SET_NETDEV_DEV(dev, &pdev->dev);
4877
4878 err = mvpp2_port_init(port);
4879 if (err < 0) {
4880 dev_err(&pdev->dev, "failed to init port %d\n", id);
4881 goto err_free_stats;
4882 }
26975821 4883
26975821
TP		 4884 mvpp2_port_periodic_xon_disable(port);
4885
26975821 4886 mvpp2_port_reset(port);
3f518509 4887
edc660fa
MW		 4888 port->pcpu = alloc_percpu(struct mvpp2_port_pcpu);
4889 if (!port->pcpu) {
4890 err = -ENOMEM;
4891 goto err_free_txq_pcpu;
4892 }
4893
213f428f 4894 if (!port->has_tx_irqs) {
e531f767 4895 for (thread = 0; thread < priv->nthreads; thread++) {
074c74df 4896 port_pcpu = per_cpu_ptr(port->pcpu, thread);
edc660fa 4897
213f428f
TP		 4898 hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC,
4899 HRTIMER_MODE_REL_PINNED);
4900 port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb;
4901 port_pcpu->timer_scheduled = false;
edc660fa 4902
213f428f
TP		 4903 tasklet_init(&port_pcpu->tx_done_tasklet,
4904 mvpp2_tx_proc_cb,
4905 (unsigned long)dev);
4906 }
edc660fa
MW		 4907 }
4908
381c5671
AT		 4909 features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4910 NETIF_F_TSO;
3f518509 4911 dev->features = features | NETIF_F_RXCSUM;
56beda3d
MC		 4912 dev->hw_features |= features | NETIF_F_RXCSUM | NETIF_F_GRO |
4913 NETIF_F_HW_VLAN_CTAG_FILTER;
576193f2 4914
d33ec452
MC		 4915 if (mvpp22_rss_is_supported())
4916 dev->hw_features |= NETIF_F_RXHASH;
4917
576193f2
SC		 4918 if (port->pool_long->id == MVPP2_BM_JUMBO && port->id != 0) {
4919 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
4920 dev->hw_features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
4921 }
4922
3f518509 4923 dev->vlan_features |= features;
1d17db08 4924 dev->gso_max_segs = MVPP2_MAX_TSO_SEGS;
10fea26c 4925 dev->priv_flags |= IFF_UNICAST_FLT;
3f518509 4926
576193f2 4927 /* MTU range: 68 - 9704 */
5777987e 4928 dev->min_mtu = ETH_MIN_MTU;
576193f2
SC		 4929 /* 9704 == 9728 - 20 and rounding to 8 */
4930 dev->max_mtu = MVPP2_BM_JUMBO_PKT_SIZE;
c4053ef3 4931 dev->dev.of_node = port_node;
5777987e 4932
4bb04326
AT		 4933 /* Phylink isn't used w/ ACPI as of now */
4934 if (port_node) {
4935 phylink = phylink_create(dev, port_fwnode, phy_mode,
4936 &mvpp2_phylink_ops);
4937 if (IS_ERR(phylink)) {
4938 err = PTR_ERR(phylink);
4939 goto err_free_port_pcpu;
4940 }
4941 port->phylink = phylink;
4942 } else {
4943 port->phylink = NULL;
4944 }
4945
3f518509
MW		 4946 err = register_netdev(dev);
4947 if (err < 0) {
4948 dev_err(&pdev->dev, "failed to register netdev\n");
4bb04326 4949 goto err_phylink;
3f518509
MW		 4950 }
4951 netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr);
4952
bf147153
MW		 4953 priv->port_list[priv->port_count++] = port;
4954
3f518509
MW		 4955 return 0;
4956
4bb04326
AT		 4957err_phylink:
4958 if (port->phylink)
4959 phylink_destroy(port->phylink);
edc660fa
MW		 4960err_free_port_pcpu:
4961 free_percpu(port->pcpu);
3f518509 4962err_free_txq_pcpu:
09f83975 4963 for (i = 0; i < port->ntxqs; i++)
3f518509
MW		 4964 free_percpu(port->txqs[i]->pcpu);
4965err_free_stats:
4966 free_percpu(port->stats);
fd3651b2
AT		 4967err_free_irq:
4968 if (port->link_irq)
4969 irq_dispose_mapping(port->link_irq);
591f4cfa
TP		 4970err_deinit_qvecs:
4971 mvpp2_queue_vectors_deinit(port);
3f518509
MW		 4972err_free_netdev:
4973 free_netdev(dev);
4974 return err;
4975}
4976
4977/* Ports removal routine */
4978static void mvpp2_port_remove(struct mvpp2_port *port)
4979{
4980 int i;
4981
4982 unregister_netdev(port->dev);
4bb04326
AT		 4983 if (port->phylink)
4984 phylink_destroy(port->phylink);
edc660fa 4985 free_percpu(port->pcpu);
3f518509 4986 free_percpu(port->stats);
09f83975 4987 for (i = 0; i < port->ntxqs; i++)
3f518509 4988 free_percpu(port->txqs[i]->pcpu);
591f4cfa 4989 mvpp2_queue_vectors_deinit(port);
fd3651b2
AT		 4990 if (port->link_irq)
4991 irq_dispose_mapping(port->link_irq);
3f518509
MW		 4992 free_netdev(port->dev);
4993}
4994
4995/* Initialize decoding windows */
4996static void mvpp2_conf_mbus_windows(const struct mbus_dram_target_info *dram,
4997 struct mvpp2 *priv)
4998{
4999 u32 win_enable;
5000 int i;
5001
5002 for (i = 0; i < 6; i++) {
5003 mvpp2_write(priv, MVPP2_WIN_BASE(i), 0);
5004 mvpp2_write(priv, MVPP2_WIN_SIZE(i), 0);
5005
5006 if (i < 4)
5007 mvpp2_write(priv, MVPP2_WIN_REMAP(i), 0);
5008 }
5009
5010 win_enable = 0;
5011
5012 for (i = 0; i < dram->num_cs; i++) {
5013 const struct mbus_dram_window *cs = dram->cs + i;
5014
5015 mvpp2_write(priv, MVPP2_WIN_BASE(i),
5016 (cs->base & 0xffff0000) | (cs->mbus_attr << 8) |
5017 dram->mbus_dram_target_id);
5018
5019 mvpp2_write(priv, MVPP2_WIN_SIZE(i),
5020 (cs->size - 1) & 0xffff0000);
5021
5022 win_enable |= (1 << i);
5023 }
5024
5025 mvpp2_write(priv, MVPP2_BASE_ADDR_ENABLE, win_enable);
5026}
5027
5028/* Initialize Rx FIFO's */
5029static void mvpp2_rx_fifo_init(struct mvpp2 *priv)
5030{
5031 int port;
5032
5033 for (port = 0; port < MVPP2_MAX_PORTS; port++) {
5034 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(port),
2d1d7df8 5035 MVPP2_RX_FIFO_PORT_DATA_SIZE_4KB);
3f518509 5036 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(port),
2d1d7df8
AT		 5037 MVPP2_RX_FIFO_PORT_ATTR_SIZE_4KB);
5038 }
5039
5040 mvpp2_write(priv, MVPP2_RX_MIN_PKT_SIZE_REG,
5041 MVPP2_RX_FIFO_PORT_MIN_PKT);
5042 mvpp2_write(priv, MVPP2_RX_FIFO_INIT_REG, 0x1);
5043}
5044
5045static void mvpp22_rx_fifo_init(struct mvpp2 *priv)
5046{
5047 int port;
5048
5049 /* The FIFO size parameters are set depending on the maximum speed a
5050 * given port can handle:
5051 * - Port 0: 10Gbps
5052 * - Port 1: 2.5Gbps
5053 * - Ports 2 and 3: 1Gbps
5054 */
5055
5056 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(0),
5057 MVPP2_RX_FIFO_PORT_DATA_SIZE_32KB);
5058 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(0),
5059 MVPP2_RX_FIFO_PORT_ATTR_SIZE_32KB);
5060
5061 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(1),
5062 MVPP2_RX_FIFO_PORT_DATA_SIZE_8KB);
5063 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(1),
5064 MVPP2_RX_FIFO_PORT_ATTR_SIZE_8KB);
5065
5066 for (port = 2; port < MVPP2_MAX_PORTS; port++) {
5067 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(port),
5068 MVPP2_RX_FIFO_PORT_DATA_SIZE_4KB);
5069 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(port),
5070 MVPP2_RX_FIFO_PORT_ATTR_SIZE_4KB);
3f518509
MW		 5071 }
5072
5073 mvpp2_write(priv, MVPP2_RX_MIN_PKT_SIZE_REG,
5074 MVPP2_RX_FIFO_PORT_MIN_PKT);
5075 mvpp2_write(priv, MVPP2_RX_FIFO_INIT_REG, 0x1);
5076}
5077
93ff130f
YM		 5078/* Initialize Tx FIFO's: the total FIFO size is 19kB on PPv2.2 and 10G
5079 * interfaces must have a Tx FIFO size of 10kB. As only port 0 can do 10G,
5080 * configure its Tx FIFO size to 10kB and the others ports Tx FIFO size to 3kB.
5081 */
7c10f974
AT		 5082static void mvpp22_tx_fifo_init(struct mvpp2 *priv)
5083{
93ff130f 5084 int port, size, thrs;
7c10f974 5085
93ff130f
YM		 5086 for (port = 0; port < MVPP2_MAX_PORTS; port++) {
5087 if (port == 0) {
5088 size = MVPP22_TX_FIFO_DATA_SIZE_10KB;
5089 thrs = MVPP2_TX_FIFO_THRESHOLD_10KB;
5090 } else {
5091 size = MVPP22_TX_FIFO_DATA_SIZE_3KB;
5092 thrs = MVPP2_TX_FIFO_THRESHOLD_3KB;
5093 }
5094 mvpp2_write(priv, MVPP22_TX_FIFO_SIZE_REG(port), size);
5095 mvpp2_write(priv, MVPP22_TX_FIFO_THRESH_REG(port), thrs);
5096 }
7c10f974
AT		 5097}
5098
6763ce31
TP		 5099static void mvpp2_axi_init(struct mvpp2 *priv)
5100{
5101 u32 val, rdval, wrval;
5102
5103 mvpp2_write(priv, MVPP22_BM_ADDR_HIGH_RLS_REG, 0x0);
5104
5105 /* AXI Bridge Configuration */
5106
5107 rdval = MVPP22_AXI_CODE_CACHE_RD_CACHE
5108 << MVPP22_AXI_ATTR_CACHE_OFFS;
5109 rdval |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
5110 << MVPP22_AXI_ATTR_DOMAIN_OFFS;
5111
5112 wrval = MVPP22_AXI_CODE_CACHE_WR_CACHE
5113 << MVPP22_AXI_ATTR_CACHE_OFFS;
5114 wrval |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
5115 << MVPP22_AXI_ATTR_DOMAIN_OFFS;
5116
5117 /* BM */
5118 mvpp2_write(priv, MVPP22_AXI_BM_WR_ATTR_REG, wrval);
5119 mvpp2_write(priv, MVPP22_AXI_BM_RD_ATTR_REG, rdval);
5120
5121 /* Descriptors */
5122 mvpp2_write(priv, MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG, rdval);
5123 mvpp2_write(priv, MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG, wrval);
5124 mvpp2_write(priv, MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG, rdval);
5125 mvpp2_write(priv, MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG, wrval);
5126
5127 /* Buffer Data */
5128 mvpp2_write(priv, MVPP22_AXI_TX_DATA_RD_ATTR_REG, rdval);
5129 mvpp2_write(priv, MVPP22_AXI_RX_DATA_WR_ATTR_REG, wrval);
5130
5131 val = MVPP22_AXI_CODE_CACHE_NON_CACHE
5132 << MVPP22_AXI_CODE_CACHE_OFFS;
5133 val |= MVPP22_AXI_CODE_DOMAIN_SYSTEM
5134 << MVPP22_AXI_CODE_DOMAIN_OFFS;
5135 mvpp2_write(priv, MVPP22_AXI_RD_NORMAL_CODE_REG, val);
5136 mvpp2_write(priv, MVPP22_AXI_WR_NORMAL_CODE_REG, val);
5137
5138 val = MVPP22_AXI_CODE_CACHE_RD_CACHE
5139 << MVPP22_AXI_CODE_CACHE_OFFS;
5140 val |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
5141 << MVPP22_AXI_CODE_DOMAIN_OFFS;
5142
5143 mvpp2_write(priv, MVPP22_AXI_RD_SNOOP_CODE_REG, val);
5144
5145 val = MVPP22_AXI_CODE_CACHE_WR_CACHE
5146 << MVPP22_AXI_CODE_CACHE_OFFS;
5147 val |= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
5148 << MVPP22_AXI_CODE_DOMAIN_OFFS;
5149
5150 mvpp2_write(priv, MVPP22_AXI_WR_SNOOP_CODE_REG, val);
5151}
5152
3f518509
MW		 5153/* Initialize network controller common part HW */
5154static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
5155{
5156 const struct mbus_dram_target_info *dram_target_info;
5157 int err, i;
08a23755 5158 u32 val;
3f518509 5159
3f518509
MW		 5160 /* MBUS windows configuration */
5161 dram_target_info = mv_mbus_dram_info();
5162 if (dram_target_info)
5163 mvpp2_conf_mbus_windows(dram_target_info, priv);
5164
6763ce31
TP		 5165 if (priv->hw_version == MVPP22)
5166 mvpp2_axi_init(priv);
5167
08a23755 5168 /* Disable HW PHY polling */
26975821
TP		 5169 if (priv->hw_version == MVPP21) {
5170 val = readl(priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
5171 val |= MVPP2_PHY_AN_STOP_SMI0_MASK;
5172 writel(val, priv->lms_base + MVPP2_PHY_AN_CFG0_REG);
5173 } else {
5174 val = readl(priv->iface_base + MVPP22_SMI_MISC_CFG_REG);
5175 val &= ~MVPP22_SMI_POLLING_EN;
5176 writel(val, priv->iface_base + MVPP22_SMI_MISC_CFG_REG);
5177 }
08a23755 5178
3f518509 5179 /* Allocate and initialize aggregated TXQs */
074c74df 5180 priv->aggr_txqs = devm_kcalloc(&pdev->dev, MVPP2_MAX_THREADS,
d7ce3cec 5181 sizeof(*priv->aggr_txqs),
3f518509
MW		 5182 GFP_KERNEL);
5183 if (!priv->aggr_txqs)
5184 return -ENOMEM;
5185
074c74df 5186 for (i = 0; i < MVPP2_MAX_THREADS; i++) {
3f518509
MW		 5187 priv->aggr_txqs[i].id = i;
5188 priv->aggr_txqs[i].size = MVPP2_AGGR_TXQ_SIZE;
85affd7e 5189 err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i], i, priv);
3f518509
MW		 5190 if (err < 0)
5191 return err;
5192 }
5193
7c10f974
AT		 5194 /* Fifo Init */
5195 if (priv->hw_version == MVPP21) {
2d1d7df8 5196 mvpp2_rx_fifo_init(priv);
7c10f974 5197 } else {
2d1d7df8 5198 mvpp22_rx_fifo_init(priv);
7c10f974
AT		 5199 mvpp22_tx_fifo_init(priv);
5200 }
3f518509 5201
26975821
TP		 5202 if (priv->hw_version == MVPP21)
5203 writel(MVPP2_EXT_GLOBAL_CTRL_DEFAULT,
5204 priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG);
3f518509
MW		 5205
5206 /* Allow cache snoop when transmiting packets */
5207 mvpp2_write(priv, MVPP2_TX_SNOOP_REG, 0x1);
5208
5209 /* Buffer Manager initialization */
5210 err = mvpp2_bm_init(pdev, priv);
5211 if (err < 0)
5212 return err;
5213
5214 /* Parser default initialization */
5215 err = mvpp2_prs_default_init(pdev, priv);
5216 if (err < 0)
5217 return err;
5218
5219 /* Classifier default initialization */
5220 mvpp2_cls_init(priv);
5221
5222 return 0;
5223}
5224
5225static int mvpp2_probe(struct platform_device *pdev)
5226{
a75edc7c 5227 const struct acpi_device_id *acpi_id;
24812221
MW		 5228 struct fwnode_handle *fwnode = pdev->dev.fwnode;
5229 struct fwnode_handle *port_fwnode;
3f518509
MW		 5230 struct mvpp2 *priv;
5231 struct resource *res;
a786841d 5232 void __iomem *base;
e531f767 5233 int i, shared;
3f518509
MW		 5234 int err;
5235
0b92e594 5236 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
3f518509
MW		 5237 if (!priv)
5238 return -ENOMEM;
5239
a75edc7c
MW		 5240 if (has_acpi_companion(&pdev->dev)) {
5241 acpi_id = acpi_match_device(pdev->dev.driver->acpi_match_table,
5242 &pdev->dev);
92ee77d1
KL		 5243 if (!acpi_id)
5244 return -EINVAL;
a75edc7c
MW		 5245 priv->hw_version = (unsigned long)acpi_id->driver_data;
5246 } else {
5247 priv->hw_version =
5248 (unsigned long)of_device_get_match_data(&pdev->dev);
5249 }
faca9247 5250
1e27a628
MC		 5251 /* multi queue mode isn't supported on PPV2.1, fallback to single
5252 * mode
5253 */
5254 if (priv->hw_version == MVPP21)
5255 queue_mode = MVPP2_QDIST_SINGLE_MODE;
5256
3f518509 5257 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
a786841d
TP		 5258 base = devm_ioremap_resource(&pdev->dev, res);
5259 if (IS_ERR(base))
5260 return PTR_ERR(base);
5261
5262 if (priv->hw_version == MVPP21) {
5263 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
5264 priv->lms_base = devm_ioremap_resource(&pdev->dev, res);
5265 if (IS_ERR(priv->lms_base))
5266 return PTR_ERR(priv->lms_base);
5267 } else {
5268 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
a75edc7c
MW		 5269 if (has_acpi_companion(&pdev->dev)) {
5270 /* In case the MDIO memory region is declared in
5271 * the ACPI, it can already appear as 'in-use'
5272 * in the OS. Because it is overlapped by second
5273 * region of the network controller, make
5274 * sure it is released, before requesting it again.
5275 * The care is taken by mvpp2 driver to avoid
5276 * concurrent access to this memory region.
5277 */
5278 release_resource(res);
5279 }
a786841d
TP		 5280 priv->iface_base = devm_ioremap_resource(&pdev->dev, res);
5281 if (IS_ERR(priv->iface_base))
5282 return PTR_ERR(priv->iface_base);
a75edc7c 5283 }
f84bf386 5284
a75edc7c 5285 if (priv->hw_version == MVPP22 && dev_of_node(&pdev->dev)) {
f84bf386
AT		 5286 priv->sysctrl_base =
5287 syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
5288 "marvell,system-controller");
5289 if (IS_ERR(priv->sysctrl_base))
5290 /* The system controller regmap is optional for dt
5291 * compatibility reasons. When not provided, the
5292 * configuration of the GoP relies on the
5293 * firmware/bootloader.
5294 */
5295 priv->sysctrl_base = NULL;
a786841d
TP		 5296 }
5297
01d04936
SC		 5298 mvpp2_setup_bm_pool();
5299
e531f767
AT		 5300
5301 priv->nthreads = min_t(unsigned int, num_present_cpus(),
5302 MVPP2_MAX_THREADS);
5303
5304 shared = num_present_cpus() - priv->nthreads;
5305 if (shared > 0)
5306 bitmap_fill(&priv->lock_map,
5307 min_t(int, shared, MVPP2_MAX_THREADS));
5308
df089aa0 5309 for (i = 0; i < MVPP2_MAX_THREADS; i++) {
a786841d
TP		 5310 u32 addr_space_sz;
5311
5312 addr_space_sz = (priv->hw_version == MVPP21 ?
5313 MVPP21_ADDR_SPACE_SZ : MVPP22_ADDR_SPACE_SZ);
df089aa0 5314 priv->swth_base[i] = base + i * addr_space_sz;
a786841d 5315 }
3f518509 5316
59b9a31e
TP		 5317 if (priv->hw_version == MVPP21)
5318 priv->max_port_rxqs = 8;
5319 else
5320 priv->max_port_rxqs = 32;
5321
a75edc7c
MW		 5322 if (dev_of_node(&pdev->dev)) {
5323 priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk");
5324 if (IS_ERR(priv->pp_clk))
5325 return PTR_ERR(priv->pp_clk);
5326 err = clk_prepare_enable(priv->pp_clk);
5327 if (err < 0)
5328 return err;
3f518509 5329
a75edc7c
MW		 5330 priv->gop_clk = devm_clk_get(&pdev->dev, "gop_clk");
5331 if (IS_ERR(priv->gop_clk)) {
5332 err = PTR_ERR(priv->gop_clk);
5333 goto err_pp_clk;
fceb55d4 5334 }
a75edc7c 5335 err = clk_prepare_enable(priv->gop_clk);
fceb55d4 5336 if (err < 0)
a75edc7c
MW		 5337 goto err_pp_clk;
5338
5339 if (priv->hw_version == MVPP22) {
5340 priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk");
5341 if (IS_ERR(priv->mg_clk)) {
5342 err = PTR_ERR(priv->mg_clk);
5343 goto err_gop_clk;
5344 }
5345
5346 err = clk_prepare_enable(priv->mg_clk);
5347 if (err < 0)
5348 goto err_gop_clk;
9af771ce
MC		 5349
5350 priv->mg_core_clk = devm_clk_get(&pdev->dev, "mg_core_clk");
5351 if (IS_ERR(priv->mg_core_clk)) {
5352 priv->mg_core_clk = NULL;
5353 } else {
5354 err = clk_prepare_enable(priv->mg_core_clk);
5355 if (err < 0)
5356 goto err_mg_clk;
5357 }
a75edc7c 5358 }
4792ea04
GC		 5359
5360 priv->axi_clk = devm_clk_get(&pdev->dev, "axi_clk");
5361 if (IS_ERR(priv->axi_clk)) {
5362 err = PTR_ERR(priv->axi_clk);
5363 if (err == -EPROBE_DEFER)
9af771ce 5364 goto err_mg_core_clk;
4792ea04
GC		 5365 priv->axi_clk = NULL;
5366 } else {
5367 err = clk_prepare_enable(priv->axi_clk);
5368 if (err < 0)
9af771ce 5369 goto err_mg_core_clk;
4792ea04 5370 }
fceb55d4 5371
a75edc7c
MW		 5372 /* Get system's tclk rate */
5373 priv->tclk = clk_get_rate(priv->pp_clk);
5374 } else if (device_property_read_u32(&pdev->dev, "clock-frequency",
5375 &priv->tclk)) {
5376 dev_err(&pdev->dev, "missing clock-frequency value\n");
5377 return -EINVAL;
5378 }
3f518509 5379
2067e0a1 5380 if (priv->hw_version == MVPP22) {
da42bb27 5381 err = dma_set_mask(&pdev->dev, MVPP2_DESC_DMA_MASK);
2067e0a1 5382 if (err)
45f972ad 5383 goto err_axi_clk;
2067e0a1
TP		 5384 /* Sadly, the BM pools all share the same register to
5385 * store the high 32 bits of their address. So they
5386 * must all have the same high 32 bits, which forces
5387 * us to restrict coherent memory to DMA_BIT_MASK(32).
5388 */
5389 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
5390 if (err)
45f972ad 5391 goto err_axi_clk;
2067e0a1
TP		 5392 }
5393
3f518509
MW		 5394 /* Initialize network controller */
5395 err = mvpp2_init(pdev, priv);
5396 if (err < 0) {
5397 dev_err(&pdev->dev, "failed to initialize controller\n");
45f972ad 5398 goto err_axi_clk;
3f518509
MW		 5399 }
5400
3f518509 5401 /* Initialize ports */
24812221
MW		 5402 fwnode_for_each_available_child_node(fwnode, port_fwnode) {
5403 err = mvpp2_port_probe(pdev, port_fwnode, priv);
3f518509 5404 if (err < 0)
26146b0e 5405 goto err_port_probe;
bf147153
MW		 5406 }
5407
5408 if (priv->port_count == 0) {
5409 dev_err(&pdev->dev, "no ports enabled\n");
5410 err = -ENODEV;
45f972ad 5411 goto err_axi_clk;
3f518509
MW		 5412 }
5413
118d6298
MR		 5414 /* Statistics must be gathered regularly because some of them (like
5415 * packets counters) are 32-bit registers and could overflow quite
5416 * quickly. For instance, a 10Gb link used at full bandwidth with the
5417 * smallest packets (64B) will overflow a 32-bit counter in less than
5418 * 30 seconds. Then, use a workqueue to fill 64-bit counters.
5419 */
118d6298
MR		 5420 snprintf(priv->queue_name, sizeof(priv->queue_name),
5421 "stats-wq-%s%s", netdev_name(priv->port_list[0]->dev),
5422 priv->port_count > 1 ? "+" : "");
5423 priv->stats_queue = create_singlethread_workqueue(priv->queue_name);
5424 if (!priv->stats_queue) {
5425 err = -ENOMEM;
26146b0e 5426 goto err_port_probe;
118d6298
MR		 5427 }
5428
21da57a2
MC		 5429 mvpp2_dbgfs_init(priv, pdev->name);
5430
3f518509
MW		 5431 platform_set_drvdata(pdev, priv);
5432 return 0;
5433
26146b0e
AT		 5434err_port_probe:
5435 i = 0;
24812221 5436 fwnode_for_each_available_child_node(fwnode, port_fwnode) {
26146b0e
AT		 5437 if (priv->port_list[i])
5438 mvpp2_port_remove(priv->port_list[i]);
5439 i++;
5440 }
45f972ad 5441err_axi_clk:
4792ea04 5442 clk_disable_unprepare(priv->axi_clk);
9af771ce
MC		 5443
5444err_mg_core_clk:
5445 if (priv->hw_version == MVPP22)
5446 clk_disable_unprepare(priv->mg_core_clk);
45f972ad 5447err_mg_clk:
fceb55d4
TP		 5448 if (priv->hw_version == MVPP22)
5449 clk_disable_unprepare(priv->mg_clk);
3f518509
MW		 5450err_gop_clk:
5451 clk_disable_unprepare(priv->gop_clk);
5452err_pp_clk:
5453 clk_disable_unprepare(priv->pp_clk);
5454 return err;
5455}
5456
5457static int mvpp2_remove(struct platform_device *pdev)
5458{
5459 struct mvpp2 *priv = platform_get_drvdata(pdev);
24812221
MW		 5460 struct fwnode_handle *fwnode = pdev->dev.fwnode;
5461 struct fwnode_handle *port_fwnode;
3f518509
MW		 5462 int i = 0;
5463
21da57a2
MC		 5464 mvpp2_dbgfs_cleanup(priv);
5465
e5c500eb 5466 flush_workqueue(priv->stats_queue);
118d6298 5467 destroy_workqueue(priv->stats_queue);
118d6298 5468
24812221 5469 fwnode_for_each_available_child_node(fwnode, port_fwnode) {
e5c500eb
MR		 5470 if (priv->port_list[i]) {
5471 mutex_destroy(&priv->port_list[i]->gather_stats_lock);
3f518509 5472 mvpp2_port_remove(priv->port_list[i]);
e5c500eb 5473 }
3f518509
MW		 5474 i++;
5475 }
5476
5477 for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
5478 struct mvpp2_bm_pool *bm_pool = &priv->bm_pools[i];
5479
5480 mvpp2_bm_pool_destroy(pdev, priv, bm_pool);
5481 }
5482
074c74df 5483 for (i = 0; i < MVPP2_MAX_THREADS; i++) {
3f518509
MW		 5484 struct mvpp2_tx_queue *aggr_txq = &priv->aggr_txqs[i];
5485
5486 dma_free_coherent(&pdev->dev,
5487 MVPP2_AGGR_TXQ_SIZE * MVPP2_DESC_ALIGNED_SIZE,
5488 aggr_txq->descs,
20396136 5489 aggr_txq->descs_dma);
3f518509
MW		 5490 }
5491
a75edc7c
MW		 5492 if (is_acpi_node(port_fwnode))
5493 return 0;
5494
4792ea04 5495 clk_disable_unprepare(priv->axi_clk);
9af771ce 5496 clk_disable_unprepare(priv->mg_core_clk);
fceb55d4 5497 clk_disable_unprepare(priv->mg_clk);
3f518509
MW		 5498 clk_disable_unprepare(priv->pp_clk);
5499 clk_disable_unprepare(priv->gop_clk);
5500
5501 return 0;
5502}
5503
5504static const struct of_device_id mvpp2_match[] = {
faca9247
TP		 5505 {
5506 .compatible = "marvell,armada-375-pp2",
5507 .data = (void *)MVPP21,
5508 },
fc5e1550
TP		 5509 {
5510 .compatible = "marvell,armada-7k-pp22",
5511 .data = (void *)MVPP22,
5512 },
3f518509
MW		 5513 { }
5514};
5515MODULE_DEVICE_TABLE(of, mvpp2_match);
5516
a75edc7c
MW		 5517static const struct acpi_device_id mvpp2_acpi_match[] = {
5518 { "MRVL0110", MVPP22 },
5519 { },
5520};
5521MODULE_DEVICE_TABLE(acpi, mvpp2_acpi_match);
5522
3f518509
MW		 5523static struct platform_driver mvpp2_driver = {
5524 .probe = mvpp2_probe,
5525 .remove = mvpp2_remove,
5526 .driver = {
5527 .name = MVPP2_DRIVER_NAME,
5528 .of_match_table = mvpp2_match,
a75edc7c 5529 .acpi_match_table = ACPI_PTR(mvpp2_acpi_match),
3f518509
MW		 5530 },
5531};
5532
5533module_platform_driver(mvpp2_driver);
5534
5535MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com");
5536MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>");
c634099d 5537MODULE_LICENSE("GPL v2");
Ubuntu Focal Linux kernel mirror