1 /* Copyright 2008 - 2016 Freescale Semiconductor Inc.
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 #include <linux/init.h>
34 #include <linux/module.h>
35 #include <linux/of_platform.h>
36 #include <linux/of_mdio.h>
37 #include <linux/of_net.h>
39 #include <linux/if_arp.h>
40 #include <linux/if_vlan.h>
41 #include <linux/icmp.h>
43 #include <linux/ipv6.h>
44 #include <linux/udp.h>
45 #include <linux/tcp.h>
46 #include <linux/net.h>
47 #include <linux/skbuff.h>
48 #include <linux/etherdevice.h>
49 #include <linux/if_ether.h>
50 #include <linux/highmem.h>
51 #include <linux/percpu.h>
52 #include <linux/dma-mapping.h>
53 #include <linux/sort.h>
54 #include <soc/fsl/bman.h>
55 #include <soc/fsl/qman.h>
58 #include "fman_port.h"
62 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpaa files
63 * using trace events only need to #include <trace/events/sched.h>
65 #define CREATE_TRACE_POINTS
66 #include "dpaa_eth_trace.h"
68 static int debug
= -1;
69 module_param(debug
, int, 0444);
70 MODULE_PARM_DESC(debug
, "Module/Driver verbosity level (0=none,...,16=all)");
72 static u16 tx_timeout
= 1000;
73 module_param(tx_timeout
, ushort
, 0444);
74 MODULE_PARM_DESC(tx_timeout
, "The Tx timeout in ms");
76 #define FM_FD_STAT_RX_ERRORS \
77 (FM_FD_ERR_DMA | FM_FD_ERR_PHYSICAL | \
78 FM_FD_ERR_SIZE | FM_FD_ERR_CLS_DISCARD | \
79 FM_FD_ERR_EXTRACTION | FM_FD_ERR_NO_SCHEME | \
80 FM_FD_ERR_PRS_TIMEOUT | FM_FD_ERR_PRS_ILL_INSTRUCT | \
81 FM_FD_ERR_PRS_HDR_ERR)
83 #define FM_FD_STAT_TX_ERRORS \
84 (FM_FD_ERR_UNSUPPORTED_FORMAT | \
85 FM_FD_ERR_LENGTH | FM_FD_ERR_DMA)
87 #define DPAA_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
88 NETIF_MSG_LINK | NETIF_MSG_IFUP | \
91 #define DPAA_INGRESS_CS_THRESHOLD 0x10000000
92 /* Ingress congestion threshold on FMan ports
93 * The size in bytes of the ingress tail-drop threshold on FMan ports.
94 * Traffic piling up above this value will be rejected by QMan and discarded
98 /* Size in bytes of the FQ taildrop threshold */
99 #define DPAA_FQ_TD 0x200000
101 #define DPAA_CS_THRESHOLD_1G 0x06000000
102 /* Egress congestion threshold on 1G ports, range 0x1000 .. 0x10000000
103 * The size in bytes of the egress Congestion State notification threshold on
104 * 1G ports. The 1G dTSECs can quite easily be flooded by cores doing Tx in a
105 * tight loop (e.g. by sending UDP datagrams at "while(1) speed"),
106 * and the larger the frame size, the more acute the problem.
107 * So we have to find a balance between these factors:
108 * - avoiding the device staying congested for a prolonged time (risking
109 * the netdev watchdog to fire - see also the tx_timeout module param);
110 * - affecting performance of protocols such as TCP, which otherwise
111 * behave well under the congestion notification mechanism;
112 * - preventing the Tx cores from tightly-looping (as if the congestion
113 * threshold was too low to be effective);
114 * - running out of memory if the CS threshold is set too high.
117 #define DPAA_CS_THRESHOLD_10G 0x10000000
118 /* The size in bytes of the egress Congestion State notification threshold on
119 * 10G ports, range 0x1000 .. 0x10000000
122 /* Largest value that the FQD's OAL field can hold */
123 #define FSL_QMAN_MAX_OAL 127
125 /* Default alignment for start of data in an Rx FD */
126 #define DPAA_FD_DATA_ALIGNMENT 16
128 /* Values for the L3R field of the FM Parse Results
130 /* L3 Type field: First IP Present IPv4 */
131 #define FM_L3_PARSE_RESULT_IPV4 0x8000
132 /* L3 Type field: First IP Present IPv6 */
133 #define FM_L3_PARSE_RESULT_IPV6 0x4000
134 /* Values for the L4R field of the FM Parse Results */
135 /* L4 Type field: UDP */
136 #define FM_L4_PARSE_RESULT_UDP 0x40
137 /* L4 Type field: TCP */
138 #define FM_L4_PARSE_RESULT_TCP 0x20
140 #define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
141 #define DPAA_BUFF_RELEASE_MAX 8 /* maximum number of buffers released at once */
143 #define FSL_DPAA_BPID_INV 0xff
144 #define FSL_DPAA_ETH_MAX_BUF_COUNT 128
145 #define FSL_DPAA_ETH_REFILL_THRESHOLD 80
147 #define DPAA_TX_PRIV_DATA_SIZE 16
148 #define DPAA_PARSE_RESULTS_SIZE sizeof(struct fman_prs_result)
149 #define DPAA_TIME_STAMP_SIZE 8
150 #define DPAA_HASH_RESULTS_SIZE 8
151 #define DPAA_RX_PRIV_DATA_SIZE (u16)(DPAA_TX_PRIV_DATA_SIZE + \
152 dpaa_rx_extra_headroom)
154 #define DPAA_ETH_RX_QUEUES 128
156 #define DPAA_ENQUEUE_RETRIES 100000
158 enum port_type
{RX
, TX
};
161 struct dpaa_fq
*tx_defq
;
162 struct dpaa_fq
*tx_errq
;
163 struct dpaa_fq
*rx_defq
;
164 struct dpaa_fq
*rx_errq
;
167 /* All the dpa bps in use at any moment */
168 static struct dpaa_bp
*dpaa_bp_array
[BM_MAX_NUM_OF_POOLS
];
170 /* The raw buffer size must be cacheline aligned */
171 #define DPAA_BP_RAW_SIZE 4096
172 /* When using more than one buffer pool, the raw sizes are as follows:
175 * 3 bp: 1KB, 2KB, 4KB
176 * 4 bp: 1KB, 2KB, 4KB, 8KB
178 static inline size_t bpool_buffer_raw_size(u8 index
, u8 cnt
)
180 size_t res
= DPAA_BP_RAW_SIZE
/ 4;
183 for (i
= (cnt
< 3) ? cnt
: 3; i
< 3 + index
; i
++)
188 /* FMan-DMA requires 16-byte alignment for Rx buffers, but SKB_DATA_ALIGN is
189 * even stronger (SMP_CACHE_BYTES-aligned), so we just get away with that,
190 * via SKB_WITH_OVERHEAD(). We can't rely on netdev_alloc_frag() giving us
191 * half-page-aligned buffers, so we reserve some more space for start-of-buffer
194 #define dpaa_bp_size(raw_size) SKB_WITH_OVERHEAD((raw_size) - SMP_CACHE_BYTES)
196 static int dpaa_max_frm
;
198 static int dpaa_rx_extra_headroom
;
200 #define dpaa_get_max_mtu() \
201 (dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
203 static int dpaa_netdev_init(struct net_device
*net_dev
,
204 const struct net_device_ops
*dpaa_ops
,
207 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
208 struct device
*dev
= net_dev
->dev
.parent
;
209 struct dpaa_percpu_priv
*percpu_priv
;
213 /* Although we access another CPU's private data here
214 * we do it at initialization so it is safe
216 for_each_possible_cpu(i
) {
217 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
218 percpu_priv
->net_dev
= net_dev
;
221 net_dev
->netdev_ops
= dpaa_ops
;
222 mac_addr
= priv
->mac_dev
->addr
;
224 net_dev
->mem_start
= priv
->mac_dev
->res
->start
;
225 net_dev
->mem_end
= priv
->mac_dev
->res
->end
;
227 net_dev
->min_mtu
= ETH_MIN_MTU
;
228 net_dev
->max_mtu
= dpaa_get_max_mtu();
230 net_dev
->hw_features
|= (NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
233 net_dev
->hw_features
|= NETIF_F_SG
| NETIF_F_HIGHDMA
;
234 /* The kernels enables GSO automatically, if we declare NETIF_F_SG.
235 * For conformity, we'll still declare GSO explicitly.
237 net_dev
->features
|= NETIF_F_GSO
;
239 net_dev
->priv_flags
|= IFF_LIVE_ADDR_CHANGE
;
240 /* we do not want shared skbs on TX */
241 net_dev
->priv_flags
&= ~IFF_TX_SKB_SHARING
;
243 net_dev
->features
|= net_dev
->hw_features
;
244 net_dev
->vlan_features
= net_dev
->features
;
246 memcpy(net_dev
->perm_addr
, mac_addr
, net_dev
->addr_len
);
247 memcpy(net_dev
->dev_addr
, mac_addr
, net_dev
->addr_len
);
249 net_dev
->ethtool_ops
= &dpaa_ethtool_ops
;
251 net_dev
->needed_headroom
= priv
->tx_headroom
;
252 net_dev
->watchdog_timeo
= msecs_to_jiffies(tx_timeout
);
254 /* start without the RUNNING flag, phylib controls it later */
255 netif_carrier_off(net_dev
);
257 err
= register_netdev(net_dev
);
259 dev_err(dev
, "register_netdev() = %d\n", err
);
266 static int dpaa_stop(struct net_device
*net_dev
)
268 struct mac_device
*mac_dev
;
269 struct dpaa_priv
*priv
;
272 priv
= netdev_priv(net_dev
);
273 mac_dev
= priv
->mac_dev
;
275 netif_tx_stop_all_queues(net_dev
);
276 /* Allow the Fman (Tx) port to process in-flight frames before we
277 * try switching it off.
279 usleep_range(5000, 10000);
281 err
= mac_dev
->stop(mac_dev
);
283 netif_err(priv
, ifdown
, net_dev
, "mac_dev->stop() = %d\n",
286 for (i
= 0; i
< ARRAY_SIZE(mac_dev
->port
); i
++) {
287 error
= fman_port_disable(mac_dev
->port
[i
]);
293 phy_disconnect(net_dev
->phydev
);
294 net_dev
->phydev
= NULL
;
299 static void dpaa_tx_timeout(struct net_device
*net_dev
)
301 struct dpaa_percpu_priv
*percpu_priv
;
302 const struct dpaa_priv
*priv
;
304 priv
= netdev_priv(net_dev
);
305 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
307 netif_crit(priv
, timer
, net_dev
, "Transmit timeout latency: %u ms\n",
308 jiffies_to_msecs(jiffies
- dev_trans_start(net_dev
)));
310 percpu_priv
->stats
.tx_errors
++;
313 /* Calculates the statistics for the given device by adding the statistics
314 * collected by each CPU.
316 static void dpaa_get_stats64(struct net_device
*net_dev
,
317 struct rtnl_link_stats64
*s
)
319 int numstats
= sizeof(struct rtnl_link_stats64
) / sizeof(u64
);
320 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
321 struct dpaa_percpu_priv
*percpu_priv
;
322 u64
*netstats
= (u64
*)s
;
326 for_each_possible_cpu(i
) {
327 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
329 cpustats
= (u64
*)&percpu_priv
->stats
;
331 /* add stats from all CPUs */
332 for (j
= 0; j
< numstats
; j
++)
333 netstats
[j
] += cpustats
[j
];
337 static struct mac_device
*dpaa_mac_dev_get(struct platform_device
*pdev
)
339 struct platform_device
*of_dev
;
340 struct dpaa_eth_data
*eth_data
;
341 struct device
*dpaa_dev
, *dev
;
342 struct device_node
*mac_node
;
343 struct mac_device
*mac_dev
;
345 dpaa_dev
= &pdev
->dev
;
346 eth_data
= dpaa_dev
->platform_data
;
348 return ERR_PTR(-ENODEV
);
350 mac_node
= eth_data
->mac_node
;
352 of_dev
= of_find_device_by_node(mac_node
);
354 dev_err(dpaa_dev
, "of_find_device_by_node(%s) failed\n",
355 mac_node
->full_name
);
356 of_node_put(mac_node
);
357 return ERR_PTR(-EINVAL
);
359 of_node_put(mac_node
);
363 mac_dev
= dev_get_drvdata(dev
);
365 dev_err(dpaa_dev
, "dev_get_drvdata(%s) failed\n",
367 return ERR_PTR(-EINVAL
);
373 static int dpaa_set_mac_address(struct net_device
*net_dev
, void *addr
)
375 const struct dpaa_priv
*priv
;
376 struct mac_device
*mac_dev
;
377 struct sockaddr old_addr
;
380 priv
= netdev_priv(net_dev
);
382 memcpy(old_addr
.sa_data
, net_dev
->dev_addr
, ETH_ALEN
);
384 err
= eth_mac_addr(net_dev
, addr
);
386 netif_err(priv
, drv
, net_dev
, "eth_mac_addr() = %d\n", err
);
390 mac_dev
= priv
->mac_dev
;
392 err
= mac_dev
->change_addr(mac_dev
->fman_mac
,
393 (enet_addr_t
*)net_dev
->dev_addr
);
395 netif_err(priv
, drv
, net_dev
, "mac_dev->change_addr() = %d\n",
397 /* reverting to previous address */
398 eth_mac_addr(net_dev
, &old_addr
);
406 static void dpaa_set_rx_mode(struct net_device
*net_dev
)
408 const struct dpaa_priv
*priv
;
411 priv
= netdev_priv(net_dev
);
413 if (!!(net_dev
->flags
& IFF_PROMISC
) != priv
->mac_dev
->promisc
) {
414 priv
->mac_dev
->promisc
= !priv
->mac_dev
->promisc
;
415 err
= priv
->mac_dev
->set_promisc(priv
->mac_dev
->fman_mac
,
416 priv
->mac_dev
->promisc
);
418 netif_err(priv
, drv
, net_dev
,
419 "mac_dev->set_promisc() = %d\n",
423 err
= priv
->mac_dev
->set_multi(net_dev
, priv
->mac_dev
);
425 netif_err(priv
, drv
, net_dev
, "mac_dev->set_multi() = %d\n",
429 static struct dpaa_bp
*dpaa_bpid2pool(int bpid
)
431 if (WARN_ON(bpid
< 0 || bpid
>= BM_MAX_NUM_OF_POOLS
))
434 return dpaa_bp_array
[bpid
];
437 /* checks if this bpool is already allocated */
438 static bool dpaa_bpid2pool_use(int bpid
)
440 if (dpaa_bpid2pool(bpid
)) {
441 atomic_inc(&dpaa_bp_array
[bpid
]->refs
);
448 /* called only once per bpid by dpaa_bp_alloc_pool() */
449 static void dpaa_bpid2pool_map(int bpid
, struct dpaa_bp
*dpaa_bp
)
451 dpaa_bp_array
[bpid
] = dpaa_bp
;
452 atomic_set(&dpaa_bp
->refs
, 1);
455 static int dpaa_bp_alloc_pool(struct dpaa_bp
*dpaa_bp
)
459 if (dpaa_bp
->size
== 0 || dpaa_bp
->config_count
== 0) {
460 pr_err("%s: Buffer pool is not properly initialized! Missing size or initial number of buffers\n",
465 /* If the pool is already specified, we only create one per bpid */
466 if (dpaa_bp
->bpid
!= FSL_DPAA_BPID_INV
&&
467 dpaa_bpid2pool_use(dpaa_bp
->bpid
))
470 if (dpaa_bp
->bpid
== FSL_DPAA_BPID_INV
) {
471 dpaa_bp
->pool
= bman_new_pool();
472 if (!dpaa_bp
->pool
) {
473 pr_err("%s: bman_new_pool() failed\n",
478 dpaa_bp
->bpid
= (u8
)bman_get_bpid(dpaa_bp
->pool
);
481 if (dpaa_bp
->seed_cb
) {
482 err
= dpaa_bp
->seed_cb(dpaa_bp
);
484 goto pool_seed_failed
;
487 dpaa_bpid2pool_map(dpaa_bp
->bpid
, dpaa_bp
);
492 pr_err("%s: pool seeding failed\n", __func__
);
493 bman_free_pool(dpaa_bp
->pool
);
498 /* remove and free all the buffers from the given buffer pool */
499 static void dpaa_bp_drain(struct dpaa_bp
*bp
)
505 struct bm_buffer bmb
[8];
508 ret
= bman_acquire(bp
->pool
, bmb
, num
);
511 /* we have less than 8 buffers left;
512 * drain them one by one
518 /* Pool is fully drained */
524 for (i
= 0; i
< num
; i
++)
525 bp
->free_buf_cb(bp
, &bmb
[i
]);
529 static void dpaa_bp_free(struct dpaa_bp
*dpaa_bp
)
531 struct dpaa_bp
*bp
= dpaa_bpid2pool(dpaa_bp
->bpid
);
533 /* the mapping between bpid and dpaa_bp is done very late in the
534 * allocation procedure; if something failed before the mapping, the bp
535 * was not configured, therefore we don't need the below instructions
540 if (!atomic_dec_and_test(&bp
->refs
))
546 dpaa_bp_array
[bp
->bpid
] = NULL
;
547 bman_free_pool(bp
->pool
);
550 static void dpaa_bps_free(struct dpaa_priv
*priv
)
554 for (i
= 0; i
< DPAA_BPS_NUM
; i
++)
555 dpaa_bp_free(priv
->dpaa_bps
[i
]);
558 /* Use multiple WQs for FQ assignment:
559 * - Tx Confirmation queues go to WQ1.
560 * - Rx Error and Tx Error queues go to WQ2 (giving them a better chance
561 * to be scheduled, in case there are many more FQs in WQ3).
562 * - Rx Default and Tx queues go to WQ3 (no differentiation between
563 * Rx and Tx traffic).
564 * This ensures that Tx-confirmed buffers are timely released. In particular,
565 * it avoids congestion on the Tx Confirm FQs, which can pile up PFDRs if they
566 * are greatly outnumbered by other FQs in the system, while
567 * dequeue scheduling is round-robin.
569 static inline void dpaa_assign_wq(struct dpaa_fq
*fq
)
571 switch (fq
->fq_type
) {
572 case FQ_TYPE_TX_CONFIRM
:
573 case FQ_TYPE_TX_CONF_MQ
:
576 case FQ_TYPE_RX_ERROR
:
577 case FQ_TYPE_TX_ERROR
:
580 case FQ_TYPE_RX_DEFAULT
:
585 WARN(1, "Invalid FQ type %d for FQID %d!\n",
586 fq
->fq_type
, fq
->fqid
);
590 static struct dpaa_fq
*dpaa_fq_alloc(struct device
*dev
,
591 u32 start
, u32 count
,
592 struct list_head
*list
,
593 enum dpaa_fq_type fq_type
)
595 struct dpaa_fq
*dpaa_fq
;
598 dpaa_fq
= devm_kzalloc(dev
, sizeof(*dpaa_fq
) * count
,
603 for (i
= 0; i
< count
; i
++) {
604 dpaa_fq
[i
].fq_type
= fq_type
;
605 dpaa_fq
[i
].fqid
= start
? start
+ i
: 0;
606 list_add_tail(&dpaa_fq
[i
].list
, list
);
609 for (i
= 0; i
< count
; i
++)
610 dpaa_assign_wq(dpaa_fq
+ i
);
615 static int dpaa_alloc_all_fqs(struct device
*dev
, struct list_head
*list
,
616 struct fm_port_fqs
*port_fqs
)
618 struct dpaa_fq
*dpaa_fq
;
620 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_RX_ERROR
);
622 goto fq_alloc_failed
;
624 port_fqs
->rx_errq
= &dpaa_fq
[0];
626 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_RX_DEFAULT
);
628 goto fq_alloc_failed
;
630 port_fqs
->rx_defq
= &dpaa_fq
[0];
632 if (!dpaa_fq_alloc(dev
, 0, DPAA_ETH_TXQ_NUM
, list
, FQ_TYPE_TX_CONF_MQ
))
633 goto fq_alloc_failed
;
635 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_TX_ERROR
);
637 goto fq_alloc_failed
;
639 port_fqs
->tx_errq
= &dpaa_fq
[0];
641 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_TX_CONFIRM
);
643 goto fq_alloc_failed
;
645 port_fqs
->tx_defq
= &dpaa_fq
[0];
647 if (!dpaa_fq_alloc(dev
, 0, DPAA_ETH_TXQ_NUM
, list
, FQ_TYPE_TX
))
648 goto fq_alloc_failed
;
653 dev_err(dev
, "dpaa_fq_alloc() failed\n");
657 static u32 rx_pool_channel
;
658 static DEFINE_SPINLOCK(rx_pool_channel_init
);
660 static int dpaa_get_channel(void)
662 spin_lock(&rx_pool_channel_init
);
663 if (!rx_pool_channel
) {
667 ret
= qman_alloc_pool(&pool
);
670 rx_pool_channel
= pool
;
672 spin_unlock(&rx_pool_channel_init
);
673 if (!rx_pool_channel
)
675 return rx_pool_channel
;
678 static void dpaa_release_channel(void)
680 qman_release_pool(rx_pool_channel
);
683 static void dpaa_eth_add_channel(u16 channel
)
685 u32 pool
= QM_SDQCR_CHANNELS_POOL_CONV(channel
);
686 const cpumask_t
*cpus
= qman_affine_cpus();
687 struct qman_portal
*portal
;
690 for_each_cpu(cpu
, cpus
) {
691 portal
= qman_get_affine_portal(cpu
);
692 qman_p_static_dequeue_add(portal
, pool
);
696 /* Congestion group state change notification callback.
697 * Stops the device's egress queues while they are congested and
698 * wakes them upon exiting congested state.
699 * Also updates some CGR-related stats.
701 static void dpaa_eth_cgscn(struct qman_portal
*qm
, struct qman_cgr
*cgr
,
704 struct dpaa_priv
*priv
= (struct dpaa_priv
*)container_of(cgr
,
705 struct dpaa_priv
, cgr_data
.cgr
);
708 priv
->cgr_data
.congestion_start_jiffies
= jiffies
;
709 netif_tx_stop_all_queues(priv
->net_dev
);
710 priv
->cgr_data
.cgr_congested_count
++;
712 priv
->cgr_data
.congested_jiffies
+=
713 (jiffies
- priv
->cgr_data
.congestion_start_jiffies
);
714 netif_tx_wake_all_queues(priv
->net_dev
);
718 static int dpaa_eth_cgr_init(struct dpaa_priv
*priv
)
720 struct qm_mcc_initcgr initcgr
;
724 err
= qman_alloc_cgrid(&priv
->cgr_data
.cgr
.cgrid
);
726 if (netif_msg_drv(priv
))
727 pr_err("%s: Error %d allocating CGR ID\n",
731 priv
->cgr_data
.cgr
.cb
= dpaa_eth_cgscn
;
733 /* Enable Congestion State Change Notifications and CS taildrop */
734 memset(&initcgr
, 0, sizeof(initcgr
));
735 initcgr
.we_mask
= cpu_to_be16(QM_CGR_WE_CSCN_EN
| QM_CGR_WE_CS_THRES
);
736 initcgr
.cgr
.cscn_en
= QM_CGR_EN
;
738 /* Set different thresholds based on the MAC speed.
739 * This may turn suboptimal if the MAC is reconfigured at a speed
740 * lower than its max, e.g. if a dTSEC later negotiates a 100Mbps link.
741 * In such cases, we ought to reconfigure the threshold, too.
743 if (priv
->mac_dev
->if_support
& SUPPORTED_10000baseT_Full
)
744 cs_th
= DPAA_CS_THRESHOLD_10G
;
746 cs_th
= DPAA_CS_THRESHOLD_1G
;
747 qm_cgr_cs_thres_set64(&initcgr
.cgr
.cs_thres
, cs_th
, 1);
749 initcgr
.we_mask
|= cpu_to_be16(QM_CGR_WE_CSTD_EN
);
750 initcgr
.cgr
.cstd_en
= QM_CGR_EN
;
752 err
= qman_create_cgr(&priv
->cgr_data
.cgr
, QMAN_CGR_FLAG_USE_INIT
,
755 if (netif_msg_drv(priv
))
756 pr_err("%s: Error %d creating CGR with ID %d\n",
757 __func__
, err
, priv
->cgr_data
.cgr
.cgrid
);
758 qman_release_cgrid(priv
->cgr_data
.cgr
.cgrid
);
761 if (netif_msg_drv(priv
))
762 pr_debug("Created CGR %d for netdev with hwaddr %pM on QMan channel %d\n",
763 priv
->cgr_data
.cgr
.cgrid
, priv
->mac_dev
->addr
,
764 priv
->cgr_data
.cgr
.chan
);
770 static inline void dpaa_setup_ingress(const struct dpaa_priv
*priv
,
772 const struct qman_fq
*template)
774 fq
->fq_base
= *template;
775 fq
->net_dev
= priv
->net_dev
;
777 fq
->flags
= QMAN_FQ_FLAG_NO_ENQUEUE
;
778 fq
->channel
= priv
->channel
;
781 static inline void dpaa_setup_egress(const struct dpaa_priv
*priv
,
783 struct fman_port
*port
,
784 const struct qman_fq
*template)
786 fq
->fq_base
= *template;
787 fq
->net_dev
= priv
->net_dev
;
790 fq
->flags
= QMAN_FQ_FLAG_TO_DCPORTAL
;
791 fq
->channel
= (u16
)fman_port_get_qman_channel_id(port
);
793 fq
->flags
= QMAN_FQ_FLAG_NO_MODIFY
;
797 static void dpaa_fq_setup(struct dpaa_priv
*priv
,
798 const struct dpaa_fq_cbs
*fq_cbs
,
799 struct fman_port
*tx_port
)
801 int egress_cnt
= 0, conf_cnt
= 0, num_portals
= 0, cpu
;
802 const cpumask_t
*affine_cpus
= qman_affine_cpus();
803 u16 portals
[NR_CPUS
];
806 for_each_cpu(cpu
, affine_cpus
)
807 portals
[num_portals
++] = qman_affine_channel(cpu
);
808 if (num_portals
== 0)
809 dev_err(priv
->net_dev
->dev
.parent
,
810 "No Qman software (affine) channels found");
812 /* Initialize each FQ in the list */
813 list_for_each_entry(fq
, &priv
->dpaa_fq_list
, list
) {
814 switch (fq
->fq_type
) {
815 case FQ_TYPE_RX_DEFAULT
:
816 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->rx_defq
);
818 case FQ_TYPE_RX_ERROR
:
819 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->rx_errq
);
822 dpaa_setup_egress(priv
, fq
, tx_port
,
823 &fq_cbs
->egress_ern
);
824 /* If we have more Tx queues than the number of cores,
825 * just ignore the extra ones.
827 if (egress_cnt
< DPAA_ETH_TXQ_NUM
)
828 priv
->egress_fqs
[egress_cnt
++] = &fq
->fq_base
;
830 case FQ_TYPE_TX_CONF_MQ
:
831 priv
->conf_fqs
[conf_cnt
++] = &fq
->fq_base
;
833 case FQ_TYPE_TX_CONFIRM
:
834 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->tx_defq
);
836 case FQ_TYPE_TX_ERROR
:
837 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->tx_errq
);
840 dev_warn(priv
->net_dev
->dev
.parent
,
841 "Unknown FQ type detected!\n");
846 /* Make sure all CPUs receive a corresponding Tx queue. */
847 while (egress_cnt
< DPAA_ETH_TXQ_NUM
) {
848 list_for_each_entry(fq
, &priv
->dpaa_fq_list
, list
) {
849 if (fq
->fq_type
!= FQ_TYPE_TX
)
851 priv
->egress_fqs
[egress_cnt
++] = &fq
->fq_base
;
852 if (egress_cnt
== DPAA_ETH_TXQ_NUM
)
858 static inline int dpaa_tx_fq_to_id(const struct dpaa_priv
*priv
,
859 struct qman_fq
*tx_fq
)
863 for (i
= 0; i
< DPAA_ETH_TXQ_NUM
; i
++)
864 if (priv
->egress_fqs
[i
] == tx_fq
)
870 static int dpaa_fq_init(struct dpaa_fq
*dpaa_fq
, bool td_enable
)
872 const struct dpaa_priv
*priv
;
873 struct qman_fq
*confq
= NULL
;
874 struct qm_mcc_initfq initfq
;
880 priv
= netdev_priv(dpaa_fq
->net_dev
);
881 dev
= dpaa_fq
->net_dev
->dev
.parent
;
883 if (dpaa_fq
->fqid
== 0)
884 dpaa_fq
->flags
|= QMAN_FQ_FLAG_DYNAMIC_FQID
;
886 dpaa_fq
->init
= !(dpaa_fq
->flags
& QMAN_FQ_FLAG_NO_MODIFY
);
888 err
= qman_create_fq(dpaa_fq
->fqid
, dpaa_fq
->flags
, &dpaa_fq
->fq_base
);
890 dev_err(dev
, "qman_create_fq() failed\n");
893 fq
= &dpaa_fq
->fq_base
;
896 memset(&initfq
, 0, sizeof(initfq
));
898 initfq
.we_mask
= cpu_to_be16(QM_INITFQ_WE_FQCTRL
);
899 /* Note: we may get to keep an empty FQ in cache */
900 initfq
.fqd
.fq_ctrl
= cpu_to_be16(QM_FQCTRL_PREFERINCACHE
);
902 /* Try to reduce the number of portal interrupts for
903 * Tx Confirmation FQs.
905 if (dpaa_fq
->fq_type
== FQ_TYPE_TX_CONFIRM
)
906 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_HOLDACTIVE
);
909 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_DESTWQ
);
911 qm_fqd_set_destwq(&initfq
.fqd
, dpaa_fq
->channel
, dpaa_fq
->wq
);
913 /* Put all egress queues in a congestion group of their own.
914 * Sensu stricto, the Tx confirmation queues are Rx FQs,
915 * rather than Tx - but they nonetheless account for the
916 * memory footprint on behalf of egress traffic. We therefore
917 * place them in the netdev's CGR, along with the Tx FQs.
919 if (dpaa_fq
->fq_type
== FQ_TYPE_TX
||
920 dpaa_fq
->fq_type
== FQ_TYPE_TX_CONFIRM
||
921 dpaa_fq
->fq_type
== FQ_TYPE_TX_CONF_MQ
) {
922 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CGID
);
923 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_CGE
);
924 initfq
.fqd
.cgid
= (u8
)priv
->cgr_data
.cgr
.cgrid
;
925 /* Set a fixed overhead accounting, in an attempt to
926 * reduce the impact of fixed-size skb shells and the
927 * driver's needed headroom on system memory. This is
928 * especially the case when the egress traffic is
929 * composed of small datagrams.
930 * Unfortunately, QMan's OAL value is capped to an
931 * insufficient value, but even that is better than
932 * no overhead accounting at all.
934 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_OAC
);
935 qm_fqd_set_oac(&initfq
.fqd
, QM_OAC_CG
);
936 qm_fqd_set_oal(&initfq
.fqd
,
937 min(sizeof(struct sk_buff
) +
939 (size_t)FSL_QMAN_MAX_OAL
));
943 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_TDTHRESH
);
944 qm_fqd_set_taildrop(&initfq
.fqd
, DPAA_FQ_TD
, 1);
945 initfq
.fqd
.fq_ctrl
= cpu_to_be16(QM_FQCTRL_TDE
);
948 if (dpaa_fq
->fq_type
== FQ_TYPE_TX
) {
949 queue_id
= dpaa_tx_fq_to_id(priv
, &dpaa_fq
->fq_base
);
951 confq
= priv
->conf_fqs
[queue_id
];
954 cpu_to_be16(QM_INITFQ_WE_CONTEXTA
);
955 /* ContextA: OVOM=1(use contextA2 bits instead of ICAD)
956 * A2V=1 (contextA A2 field is valid)
957 * A0V=1 (contextA A0 field is valid)
958 * B0V=1 (contextB field is valid)
959 * ContextA A2: EBD=1 (deallocate buffers inside FMan)
960 * ContextB B0(ASPID): 0 (absolute Virtual Storage ID)
962 qm_fqd_context_a_set64(&initfq
.fqd
,
963 0x1e00000080000000ULL
);
967 /* Put all the ingress queues in our "ingress CGR". */
968 if (priv
->use_ingress_cgr
&&
969 (dpaa_fq
->fq_type
== FQ_TYPE_RX_DEFAULT
||
970 dpaa_fq
->fq_type
== FQ_TYPE_RX_ERROR
)) {
971 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CGID
);
972 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_CGE
);
973 initfq
.fqd
.cgid
= (u8
)priv
->ingress_cgr
.cgrid
;
974 /* Set a fixed overhead accounting, just like for the
977 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_OAC
);
978 qm_fqd_set_oac(&initfq
.fqd
, QM_OAC_CG
);
979 qm_fqd_set_oal(&initfq
.fqd
,
980 min(sizeof(struct sk_buff
) +
982 (size_t)FSL_QMAN_MAX_OAL
));
985 /* Initialization common to all ingress queues */
986 if (dpaa_fq
->flags
& QMAN_FQ_FLAG_NO_ENQUEUE
) {
987 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CONTEXTA
);
988 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_HOLDACTIVE
);
989 initfq
.fqd
.context_a
.stashing
.exclusive
=
990 QM_STASHING_EXCL_DATA
| QM_STASHING_EXCL_CTX
|
991 QM_STASHING_EXCL_ANNOTATION
;
992 qm_fqd_set_stashing(&initfq
.fqd
, 1, 2,
993 DIV_ROUND_UP(sizeof(struct qman_fq
),
997 err
= qman_init_fq(fq
, QMAN_INITFQ_FLAG_SCHED
, &initfq
);
999 dev_err(dev
, "qman_init_fq(%u) = %d\n",
1000 qman_fq_fqid(fq
), err
);
1001 qman_destroy_fq(fq
);
1006 dpaa_fq
->fqid
= qman_fq_fqid(fq
);
1011 static int dpaa_fq_free_entry(struct device
*dev
, struct qman_fq
*fq
)
1013 const struct dpaa_priv
*priv
;
1014 struct dpaa_fq
*dpaa_fq
;
1019 dpaa_fq
= container_of(fq
, struct dpaa_fq
, fq_base
);
1020 priv
= netdev_priv(dpaa_fq
->net_dev
);
1022 if (dpaa_fq
->init
) {
1023 err
= qman_retire_fq(fq
, NULL
);
1024 if (err
< 0 && netif_msg_drv(priv
))
1025 dev_err(dev
, "qman_retire_fq(%u) = %d\n",
1026 qman_fq_fqid(fq
), err
);
1028 error
= qman_oos_fq(fq
);
1029 if (error
< 0 && netif_msg_drv(priv
)) {
1030 dev_err(dev
, "qman_oos_fq(%u) = %d\n",
1031 qman_fq_fqid(fq
), error
);
1037 qman_destroy_fq(fq
);
1038 list_del(&dpaa_fq
->list
);
1043 static int dpaa_fq_free(struct device
*dev
, struct list_head
*list
)
1045 struct dpaa_fq
*dpaa_fq
, *tmp
;
1049 list_for_each_entry_safe(dpaa_fq
, tmp
, list
, list
) {
1050 error
= dpaa_fq_free_entry(dev
, (struct qman_fq
*)dpaa_fq
);
1051 if (error
< 0 && err
>= 0)
1058 static void dpaa_eth_init_tx_port(struct fman_port
*port
, struct dpaa_fq
*errq
,
1059 struct dpaa_fq
*defq
,
1060 struct dpaa_buffer_layout
*buf_layout
)
1062 struct fman_buffer_prefix_content buf_prefix_content
;
1063 struct fman_port_params params
;
1066 memset(¶ms
, 0, sizeof(params
));
1067 memset(&buf_prefix_content
, 0, sizeof(buf_prefix_content
));
1069 buf_prefix_content
.priv_data_size
= buf_layout
->priv_data_size
;
1070 buf_prefix_content
.pass_prs_result
= true;
1071 buf_prefix_content
.pass_hash_result
= true;
1072 buf_prefix_content
.pass_time_stamp
= false;
1073 buf_prefix_content
.data_align
= DPAA_FD_DATA_ALIGNMENT
;
1075 params
.specific_params
.non_rx_params
.err_fqid
= errq
->fqid
;
1076 params
.specific_params
.non_rx_params
.dflt_fqid
= defq
->fqid
;
1078 err
= fman_port_config(port
, ¶ms
);
1080 pr_err("%s: fman_port_config failed\n", __func__
);
1082 err
= fman_port_cfg_buf_prefix_content(port
, &buf_prefix_content
);
1084 pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1087 err
= fman_port_init(port
);
1089 pr_err("%s: fm_port_init failed\n", __func__
);
1092 static void dpaa_eth_init_rx_port(struct fman_port
*port
, struct dpaa_bp
**bps
,
1093 size_t count
, struct dpaa_fq
*errq
,
1094 struct dpaa_fq
*defq
,
1095 struct dpaa_buffer_layout
*buf_layout
)
1097 struct fman_buffer_prefix_content buf_prefix_content
;
1098 struct fman_port_rx_params
*rx_p
;
1099 struct fman_port_params params
;
1102 memset(¶ms
, 0, sizeof(params
));
1103 memset(&buf_prefix_content
, 0, sizeof(buf_prefix_content
));
1105 buf_prefix_content
.priv_data_size
= buf_layout
->priv_data_size
;
1106 buf_prefix_content
.pass_prs_result
= true;
1107 buf_prefix_content
.pass_hash_result
= true;
1108 buf_prefix_content
.pass_time_stamp
= false;
1109 buf_prefix_content
.data_align
= DPAA_FD_DATA_ALIGNMENT
;
1111 rx_p
= ¶ms
.specific_params
.rx_params
;
1112 rx_p
->err_fqid
= errq
->fqid
;
1113 rx_p
->dflt_fqid
= defq
->fqid
;
1115 count
= min(ARRAY_SIZE(rx_p
->ext_buf_pools
.ext_buf_pool
), count
);
1116 rx_p
->ext_buf_pools
.num_of_pools_used
= (u8
)count
;
1117 for (i
= 0; i
< count
; i
++) {
1118 rx_p
->ext_buf_pools
.ext_buf_pool
[i
].id
= bps
[i
]->bpid
;
1119 rx_p
->ext_buf_pools
.ext_buf_pool
[i
].size
= (u16
)bps
[i
]->size
;
1122 err
= fman_port_config(port
, ¶ms
);
1124 pr_err("%s: fman_port_config failed\n", __func__
);
1126 err
= fman_port_cfg_buf_prefix_content(port
, &buf_prefix_content
);
1128 pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1131 err
= fman_port_init(port
);
1133 pr_err("%s: fm_port_init failed\n", __func__
);
1136 static void dpaa_eth_init_ports(struct mac_device
*mac_dev
,
1137 struct dpaa_bp
**bps
, size_t count
,
1138 struct fm_port_fqs
*port_fqs
,
1139 struct dpaa_buffer_layout
*buf_layout
,
1142 struct fman_port
*rxport
= mac_dev
->port
[RX
];
1143 struct fman_port
*txport
= mac_dev
->port
[TX
];
1145 dpaa_eth_init_tx_port(txport
, port_fqs
->tx_errq
,
1146 port_fqs
->tx_defq
, &buf_layout
[TX
]);
1147 dpaa_eth_init_rx_port(rxport
, bps
, count
, port_fqs
->rx_errq
,
1148 port_fqs
->rx_defq
, &buf_layout
[RX
]);
1151 static int dpaa_bman_release(const struct dpaa_bp
*dpaa_bp
,
1152 struct bm_buffer
*bmb
, int cnt
)
1156 err
= bman_release(dpaa_bp
->pool
, bmb
, cnt
);
1157 /* Should never occur, address anyway to avoid leaking the buffers */
1158 if (unlikely(WARN_ON(err
)) && dpaa_bp
->free_buf_cb
)
1160 dpaa_bp
->free_buf_cb(dpaa_bp
, &bmb
[cnt
]);
1165 static void dpaa_release_sgt_members(struct qm_sg_entry
*sgt
)
1167 struct bm_buffer bmb
[DPAA_BUFF_RELEASE_MAX
];
1168 struct dpaa_bp
*dpaa_bp
;
1171 memset(bmb
, 0, sizeof(bmb
));
1174 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1180 WARN_ON(qm_sg_entry_is_ext(&sgt
[i
]));
1182 bm_buffer_set64(&bmb
[j
], qm_sg_entry_get64(&sgt
[i
]));
1185 } while (j
< ARRAY_SIZE(bmb
) &&
1186 !qm_sg_entry_is_final(&sgt
[i
- 1]) &&
1187 sgt
[i
- 1].bpid
== sgt
[i
].bpid
);
1189 dpaa_bman_release(dpaa_bp
, bmb
, j
);
1190 } while (!qm_sg_entry_is_final(&sgt
[i
- 1]));
1193 static void dpaa_fd_release(const struct net_device
*net_dev
,
1194 const struct qm_fd
*fd
)
1196 struct qm_sg_entry
*sgt
;
1197 struct dpaa_bp
*dpaa_bp
;
1198 struct bm_buffer bmb
;
1203 bm_buffer_set64(&bmb
, qm_fd_addr(fd
));
1205 dpaa_bp
= dpaa_bpid2pool(fd
->bpid
);
1209 if (qm_fd_get_format(fd
) == qm_fd_sg
) {
1210 vaddr
= phys_to_virt(qm_fd_addr(fd
));
1211 sgt
= vaddr
+ qm_fd_get_offset(fd
);
1213 dma_unmap_single(dpaa_bp
->dev
, qm_fd_addr(fd
), dpaa_bp
->size
,
1216 dpaa_release_sgt_members(sgt
);
1218 addr
= dma_map_single(dpaa_bp
->dev
, vaddr
, dpaa_bp
->size
,
1220 if (dma_mapping_error(dpaa_bp
->dev
, addr
)) {
1221 dev_err(dpaa_bp
->dev
, "DMA mapping failed");
1224 bm_buffer_set64(&bmb
, addr
);
1227 dpaa_bman_release(dpaa_bp
, &bmb
, 1);
1230 static void count_ern(struct dpaa_percpu_priv
*percpu_priv
,
1231 const union qm_mr_entry
*msg
)
1233 switch (msg
->ern
.rc
& QM_MR_RC_MASK
) {
1234 case QM_MR_RC_CGR_TAILDROP
:
1235 percpu_priv
->ern_cnt
.cg_tdrop
++;
1238 percpu_priv
->ern_cnt
.wred
++;
1240 case QM_MR_RC_ERROR
:
1241 percpu_priv
->ern_cnt
.err_cond
++;
1243 case QM_MR_RC_ORPWINDOW_EARLY
:
1244 percpu_priv
->ern_cnt
.early_window
++;
1246 case QM_MR_RC_ORPWINDOW_LATE
:
1247 percpu_priv
->ern_cnt
.late_window
++;
1249 case QM_MR_RC_FQ_TAILDROP
:
1250 percpu_priv
->ern_cnt
.fq_tdrop
++;
1252 case QM_MR_RC_ORPWINDOW_RETIRED
:
1253 percpu_priv
->ern_cnt
.fq_retired
++;
1255 case QM_MR_RC_ORP_ZERO
:
1256 percpu_priv
->ern_cnt
.orp_zero
++;
1261 /* Turn on HW checksum computation for this outgoing frame.
1262 * If the current protocol is not something we support in this regard
1263 * (or if the stack has already computed the SW checksum), we do nothing.
1265 * Returns 0 if all goes well (or HW csum doesn't apply), and a negative value
1268 * Note that this function may modify the fd->cmd field and the skb data buffer
1269 * (the Parse Results area).
1271 static int dpaa_enable_tx_csum(struct dpaa_priv
*priv
,
1272 struct sk_buff
*skb
,
1274 char *parse_results
)
1276 struct fman_prs_result
*parse_result
;
1277 u16 ethertype
= ntohs(skb
->protocol
);
1278 struct ipv6hdr
*ipv6h
= NULL
;
1283 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1286 /* Note: L3 csum seems to be already computed in sw, but we can't choose
1287 * L4 alone from the FM configuration anyway.
1290 /* Fill in some fields of the Parse Results array, so the FMan
1291 * can find them as if they came from the FMan Parser.
1293 parse_result
= (struct fman_prs_result
*)parse_results
;
1295 /* If we're dealing with VLAN, get the real Ethernet type */
1296 if (ethertype
== ETH_P_8021Q
) {
1297 /* We can't always assume the MAC header is set correctly
1298 * by the stack, so reset to beginning of skb->data
1300 skb_reset_mac_header(skb
);
1301 ethertype
= ntohs(vlan_eth_hdr(skb
)->h_vlan_encapsulated_proto
);
1304 /* Fill in the relevant L3 parse result fields
1305 * and read the L4 protocol type
1307 switch (ethertype
) {
1309 parse_result
->l3r
= cpu_to_be16(FM_L3_PARSE_RESULT_IPV4
);
1312 l4_proto
= iph
->protocol
;
1315 parse_result
->l3r
= cpu_to_be16(FM_L3_PARSE_RESULT_IPV6
);
1316 ipv6h
= ipv6_hdr(skb
);
1318 l4_proto
= ipv6h
->nexthdr
;
1321 /* We shouldn't even be here */
1322 if (net_ratelimit())
1323 netif_alert(priv
, tx_err
, priv
->net_dev
,
1324 "Can't compute HW csum for L3 proto 0x%x\n",
1325 ntohs(skb
->protocol
));
1330 /* Fill in the relevant L4 parse result fields */
1333 parse_result
->l4r
= FM_L4_PARSE_RESULT_UDP
;
1336 parse_result
->l4r
= FM_L4_PARSE_RESULT_TCP
;
1339 if (net_ratelimit())
1340 netif_alert(priv
, tx_err
, priv
->net_dev
,
1341 "Can't compute HW csum for L4 proto 0x%x\n",
1347 /* At index 0 is IPOffset_1 as defined in the Parse Results */
1348 parse_result
->ip_off
[0] = (u8
)skb_network_offset(skb
);
1349 parse_result
->l4_off
= (u8
)skb_transport_offset(skb
);
1351 /* Enable L3 (and L4, if TCP or UDP) HW checksum. */
1352 fd
->cmd
|= cpu_to_be32(FM_FD_CMD_RPD
| FM_FD_CMD_DTC
);
1354 /* On P1023 and similar platforms fd->cmd interpretation could
1355 * be disabled by setting CONTEXT_A bit ICMD; currently this bit
1356 * is not set so we do not need to check; in the future, if/when
1357 * using context_a we need to check this bit
1364 static int dpaa_bp_add_8_bufs(const struct dpaa_bp
*dpaa_bp
)
1366 struct device
*dev
= dpaa_bp
->dev
;
1367 struct bm_buffer bmb
[8];
1372 for (i
= 0; i
< 8; i
++) {
1373 new_buf
= netdev_alloc_frag(dpaa_bp
->raw_size
);
1374 if (unlikely(!new_buf
)) {
1375 dev_err(dev
, "netdev_alloc_frag() failed, size %zu\n",
1377 goto release_previous_buffs
;
1379 new_buf
= PTR_ALIGN(new_buf
, SMP_CACHE_BYTES
);
1381 addr
= dma_map_single(dev
, new_buf
,
1382 dpaa_bp
->size
, DMA_FROM_DEVICE
);
1383 if (unlikely(dma_mapping_error(dev
, addr
))) {
1384 dev_err(dpaa_bp
->dev
, "DMA map failed");
1385 goto release_previous_buffs
;
1389 bm_buffer_set64(&bmb
[i
], addr
);
1393 return dpaa_bman_release(dpaa_bp
, bmb
, i
);
1395 release_previous_buffs
:
1396 WARN_ONCE(1, "dpaa_eth: failed to add buffers on Rx\n");
1398 bm_buffer_set64(&bmb
[i
], 0);
1399 /* Avoid releasing a completely null buffer; bman_release() requires
1400 * at least one buffer.
1408 static int dpaa_bp_seed(struct dpaa_bp
*dpaa_bp
)
1412 /* Give each CPU an allotment of "config_count" buffers */
1413 for_each_possible_cpu(i
) {
1414 int *count_ptr
= per_cpu_ptr(dpaa_bp
->percpu_count
, i
);
1417 /* Although we access another CPU's counters here
1418 * we do it at boot time so it is safe
1420 for (j
= 0; j
< dpaa_bp
->config_count
; j
+= 8)
1421 *count_ptr
+= dpaa_bp_add_8_bufs(dpaa_bp
);
1426 /* Add buffers/(pages) for Rx processing whenever bpool count falls below
1429 static int dpaa_eth_refill_bpool(struct dpaa_bp
*dpaa_bp
, int *countptr
)
1431 int count
= *countptr
;
1434 if (unlikely(count
< FSL_DPAA_ETH_REFILL_THRESHOLD
)) {
1436 new_bufs
= dpaa_bp_add_8_bufs(dpaa_bp
);
1437 if (unlikely(!new_bufs
)) {
1438 /* Avoid looping forever if we've temporarily
1439 * run out of memory. We'll try again at the
1445 } while (count
< FSL_DPAA_ETH_MAX_BUF_COUNT
);
1448 if (unlikely(count
< FSL_DPAA_ETH_MAX_BUF_COUNT
))
1455 static int dpaa_eth_refill_bpools(struct dpaa_priv
*priv
)
1457 struct dpaa_bp
*dpaa_bp
;
1461 for (i
= 0; i
< DPAA_BPS_NUM
; i
++) {
1462 dpaa_bp
= priv
->dpaa_bps
[i
];
1465 countptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1466 res
= dpaa_eth_refill_bpool(dpaa_bp
, countptr
);
1473 /* Cleanup function for outgoing frame descriptors that were built on Tx path,
1474 * either contiguous frames or scatter/gather ones.
1475 * Skb freeing is not handled here.
1477 * This function may be called on error paths in the Tx function, so guard
1478 * against cases when not all fd relevant fields were filled in.
1480 * Return the skb backpointer, since for S/G frames the buffer containing it
1483 static struct sk_buff
*dpaa_cleanup_tx_fd(const struct dpaa_priv
*priv
,
1484 const struct qm_fd
*fd
)
1486 const enum dma_data_direction dma_dir
= DMA_TO_DEVICE
;
1487 struct device
*dev
= priv
->net_dev
->dev
.parent
;
1488 dma_addr_t addr
= qm_fd_addr(fd
);
1489 const struct qm_sg_entry
*sgt
;
1490 struct sk_buff
**skbh
, *skb
;
1493 skbh
= (struct sk_buff
**)phys_to_virt(addr
);
1496 if (unlikely(qm_fd_get_format(fd
) == qm_fd_sg
)) {
1497 nr_frags
= skb_shinfo(skb
)->nr_frags
;
1498 dma_unmap_single(dev
, addr
, qm_fd_get_offset(fd
) +
1499 sizeof(struct qm_sg_entry
) * (1 + nr_frags
),
1502 /* The sgt buffer has been allocated with netdev_alloc_frag(),
1505 sgt
= phys_to_virt(addr
+ qm_fd_get_offset(fd
));
1507 /* sgt[0] is from lowmem, was dma_map_single()-ed */
1508 dma_unmap_single(dev
, qm_sg_addr(&sgt
[0]),
1509 qm_sg_entry_get_len(&sgt
[0]), dma_dir
);
1511 /* remaining pages were mapped with skb_frag_dma_map() */
1512 for (i
= 1; i
< nr_frags
; i
++) {
1513 WARN_ON(qm_sg_entry_is_ext(&sgt
[i
]));
1515 dma_unmap_page(dev
, qm_sg_addr(&sgt
[i
]),
1516 qm_sg_entry_get_len(&sgt
[i
]), dma_dir
);
1519 /* Free the page frag that we allocated on Tx */
1520 skb_free_frag(phys_to_virt(addr
));
1522 dma_unmap_single(dev
, addr
,
1523 skb_tail_pointer(skb
) - (u8
*)skbh
, dma_dir
);
1529 /* Build a linear skb around the received buffer.
1530 * We are guaranteed there is enough room at the end of the data buffer to
1531 * accommodate the shared info area of the skb.
1533 static struct sk_buff
*contig_fd_to_skb(const struct dpaa_priv
*priv
,
1534 const struct qm_fd
*fd
)
1536 ssize_t fd_off
= qm_fd_get_offset(fd
);
1537 dma_addr_t addr
= qm_fd_addr(fd
);
1538 struct dpaa_bp
*dpaa_bp
;
1539 struct sk_buff
*skb
;
1542 vaddr
= phys_to_virt(addr
);
1543 WARN_ON(!IS_ALIGNED((unsigned long)vaddr
, SMP_CACHE_BYTES
));
1545 dpaa_bp
= dpaa_bpid2pool(fd
->bpid
);
1549 skb
= build_skb(vaddr
, dpaa_bp
->size
+
1550 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)));
1551 if (unlikely(!skb
)) {
1552 WARN_ONCE(1, "Build skb failure on Rx\n");
1555 WARN_ON(fd_off
!= priv
->rx_headroom
);
1556 skb_reserve(skb
, fd_off
);
1557 skb_put(skb
, qm_fd_get_length(fd
));
1559 skb
->ip_summed
= CHECKSUM_NONE
;
1564 skb_free_frag(vaddr
);
1568 /* Build an skb with the data of the first S/G entry in the linear portion and
1569 * the rest of the frame as skb fragments.
1571 * The page fragment holding the S/G Table is recycled here.
1573 static struct sk_buff
*sg_fd_to_skb(const struct dpaa_priv
*priv
,
1574 const struct qm_fd
*fd
)
1576 ssize_t fd_off
= qm_fd_get_offset(fd
);
1577 dma_addr_t addr
= qm_fd_addr(fd
);
1578 const struct qm_sg_entry
*sgt
;
1579 struct page
*page
, *head_page
;
1580 struct dpaa_bp
*dpaa_bp
;
1581 void *vaddr
, *sg_vaddr
;
1582 int frag_off
, frag_len
;
1583 struct sk_buff
*skb
;
1590 vaddr
= phys_to_virt(addr
);
1591 WARN_ON(!IS_ALIGNED((unsigned long)vaddr
, SMP_CACHE_BYTES
));
1593 /* Iterate through the SGT entries and add data buffers to the skb */
1594 sgt
= vaddr
+ fd_off
;
1595 for (i
= 0; i
< DPAA_SGT_MAX_ENTRIES
; i
++) {
1596 /* Extension bit is not supported */
1597 WARN_ON(qm_sg_entry_is_ext(&sgt
[i
]));
1599 sg_addr
= qm_sg_addr(&sgt
[i
]);
1600 sg_vaddr
= phys_to_virt(sg_addr
);
1601 WARN_ON(!IS_ALIGNED((unsigned long)sg_vaddr
,
1604 /* We may use multiple Rx pools */
1605 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1609 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1610 dma_unmap_single(dpaa_bp
->dev
, sg_addr
, dpaa_bp
->size
,
1613 sz
= dpaa_bp
->size
+
1614 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
1615 skb
= build_skb(sg_vaddr
, sz
);
1616 if (WARN_ON(unlikely(!skb
)))
1619 skb
->ip_summed
= CHECKSUM_NONE
;
1621 /* Make sure forwarded skbs will have enough space
1622 * on Tx, if extra headers are added.
1624 WARN_ON(fd_off
!= priv
->rx_headroom
);
1625 skb_reserve(skb
, fd_off
);
1626 skb_put(skb
, qm_sg_entry_get_len(&sgt
[i
]));
1628 /* Not the first S/G entry; all data from buffer will
1629 * be added in an skb fragment; fragment index is offset
1630 * by one since first S/G entry was incorporated in the
1631 * linear part of the skb.
1633 * Caution: 'page' may be a tail page.
1635 page
= virt_to_page(sg_vaddr
);
1636 head_page
= virt_to_head_page(sg_vaddr
);
1638 /* Compute offset in (possibly tail) page */
1639 page_offset
= ((unsigned long)sg_vaddr
&
1641 (page_address(page
) - page_address(head_page
));
1642 /* page_offset only refers to the beginning of sgt[i];
1643 * but the buffer itself may have an internal offset.
1645 frag_off
= qm_sg_entry_get_off(&sgt
[i
]) + page_offset
;
1646 frag_len
= qm_sg_entry_get_len(&sgt
[i
]);
1647 /* skb_add_rx_frag() does no checking on the page; if
1648 * we pass it a tail page, we'll end up with
1649 * bad page accounting and eventually with segafults.
1651 skb_add_rx_frag(skb
, i
- 1, head_page
, frag_off
,
1652 frag_len
, dpaa_bp
->size
);
1654 /* Update the pool count for the current {cpu x bpool} */
1657 if (qm_sg_entry_is_final(&sgt
[i
]))
1660 WARN_ONCE(i
== DPAA_SGT_MAX_ENTRIES
, "No final bit on SGT\n");
1662 /* free the SG table buffer */
1663 skb_free_frag(vaddr
);
1668 /* compensate sw bpool counter changes */
1669 for (i
--; i
>= 0; i
--) {
1670 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1672 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1676 /* free all the SG entries */
1677 for (i
= 0; i
< DPAA_SGT_MAX_ENTRIES
; i
++) {
1678 sg_addr
= qm_sg_addr(&sgt
[i
]);
1679 sg_vaddr
= phys_to_virt(sg_addr
);
1680 skb_free_frag(sg_vaddr
);
1681 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1683 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1687 if (qm_sg_entry_is_final(&sgt
[i
]))
1690 /* free the SGT fragment */
1691 skb_free_frag(vaddr
);
1696 static int skb_to_contig_fd(struct dpaa_priv
*priv
,
1697 struct sk_buff
*skb
, struct qm_fd
*fd
,
1700 struct net_device
*net_dev
= priv
->net_dev
;
1701 struct device
*dev
= net_dev
->dev
.parent
;
1702 enum dma_data_direction dma_dir
;
1703 unsigned char *buffer_start
;
1704 struct sk_buff
**skbh
;
1708 /* We are guaranteed to have at least tx_headroom bytes
1709 * available, so just use that for offset.
1711 fd
->bpid
= FSL_DPAA_BPID_INV
;
1712 buffer_start
= skb
->data
- priv
->tx_headroom
;
1713 dma_dir
= DMA_TO_DEVICE
;
1715 skbh
= (struct sk_buff
**)buffer_start
;
1718 /* Enable L3/L4 hardware checksum computation.
1720 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1721 * need to write into the skb.
1723 err
= dpaa_enable_tx_csum(priv
, skb
, fd
,
1724 ((char *)skbh
) + DPAA_TX_PRIV_DATA_SIZE
);
1725 if (unlikely(err
< 0)) {
1726 if (net_ratelimit())
1727 netif_err(priv
, tx_err
, net_dev
, "HW csum error: %d\n",
1732 /* Fill in the rest of the FD fields */
1733 qm_fd_set_contig(fd
, priv
->tx_headroom
, skb
->len
);
1734 fd
->cmd
|= cpu_to_be32(FM_FD_CMD_FCO
);
1736 /* Map the entire buffer size that may be seen by FMan, but no more */
1737 addr
= dma_map_single(dev
, skbh
,
1738 skb_tail_pointer(skb
) - buffer_start
, dma_dir
);
1739 if (unlikely(dma_mapping_error(dev
, addr
))) {
1740 if (net_ratelimit())
1741 netif_err(priv
, tx_err
, net_dev
, "dma_map_single() failed\n");
1744 qm_fd_addr_set64(fd
, addr
);
1749 static int skb_to_sg_fd(struct dpaa_priv
*priv
,
1750 struct sk_buff
*skb
, struct qm_fd
*fd
)
1752 const enum dma_data_direction dma_dir
= DMA_TO_DEVICE
;
1753 const int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1754 struct net_device
*net_dev
= priv
->net_dev
;
1755 struct device
*dev
= net_dev
->dev
.parent
;
1756 struct qm_sg_entry
*sgt
;
1757 struct sk_buff
**skbh
;
1765 /* get a page frag to store the SGTable */
1766 sz
= SKB_DATA_ALIGN(priv
->tx_headroom
+
1767 sizeof(struct qm_sg_entry
) * (1 + nr_frags
));
1768 sgt_buf
= netdev_alloc_frag(sz
);
1769 if (unlikely(!sgt_buf
)) {
1770 netdev_err(net_dev
, "netdev_alloc_frag() failed for size %d\n",
1775 /* Enable L3/L4 hardware checksum computation.
1777 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1778 * need to write into the skb.
1780 err
= dpaa_enable_tx_csum(priv
, skb
, fd
,
1781 sgt_buf
+ DPAA_TX_PRIV_DATA_SIZE
);
1782 if (unlikely(err
< 0)) {
1783 if (net_ratelimit())
1784 netif_err(priv
, tx_err
, net_dev
, "HW csum error: %d\n",
1789 sgt
= (struct qm_sg_entry
*)(sgt_buf
+ priv
->tx_headroom
);
1790 qm_sg_entry_set_len(&sgt
[0], skb_headlen(skb
));
1791 sgt
[0].bpid
= FSL_DPAA_BPID_INV
;
1793 addr
= dma_map_single(dev
, skb
->data
,
1794 skb_headlen(skb
), dma_dir
);
1795 if (unlikely(dma_mapping_error(dev
, addr
))) {
1796 dev_err(dev
, "DMA mapping failed");
1798 goto sg0_map_failed
;
1800 qm_sg_entry_set64(&sgt
[0], addr
);
1802 /* populate the rest of SGT entries */
1803 frag
= &skb_shinfo(skb
)->frags
[0];
1804 frag_len
= frag
->size
;
1805 for (i
= 1; i
<= nr_frags
; i
++, frag
++) {
1806 WARN_ON(!skb_frag_page(frag
));
1807 addr
= skb_frag_dma_map(dev
, frag
, 0,
1809 if (unlikely(dma_mapping_error(dev
, addr
))) {
1810 dev_err(dev
, "DMA mapping failed");
1815 qm_sg_entry_set_len(&sgt
[i
], frag_len
);
1816 sgt
[i
].bpid
= FSL_DPAA_BPID_INV
;
1819 /* keep the offset in the address */
1820 qm_sg_entry_set64(&sgt
[i
], addr
);
1821 frag_len
= frag
->size
;
1823 qm_sg_entry_set_f(&sgt
[i
- 1], frag_len
);
1825 qm_fd_set_sg(fd
, priv
->tx_headroom
, skb
->len
);
1827 /* DMA map the SGT page */
1828 buffer_start
= (void *)sgt
- priv
->tx_headroom
;
1829 skbh
= (struct sk_buff
**)buffer_start
;
1832 addr
= dma_map_single(dev
, buffer_start
, priv
->tx_headroom
+
1833 sizeof(struct qm_sg_entry
) * (1 + nr_frags
),
1835 if (unlikely(dma_mapping_error(dev
, addr
))) {
1836 dev_err(dev
, "DMA mapping failed");
1838 goto sgt_map_failed
;
1841 fd
->bpid
= FSL_DPAA_BPID_INV
;
1842 fd
->cmd
|= cpu_to_be32(FM_FD_CMD_FCO
);
1843 qm_fd_addr_set64(fd
, addr
);
1849 for (j
= 0; j
< i
; j
++)
1850 dma_unmap_page(dev
, qm_sg_addr(&sgt
[j
]),
1851 qm_sg_entry_get_len(&sgt
[j
]), dma_dir
);
1854 skb_free_frag(sgt_buf
);
1859 static inline int dpaa_xmit(struct dpaa_priv
*priv
,
1860 struct rtnl_link_stats64
*percpu_stats
,
1864 struct qman_fq
*egress_fq
;
1867 egress_fq
= priv
->egress_fqs
[queue
];
1868 if (fd
->bpid
== FSL_DPAA_BPID_INV
)
1869 fd
->cmd
|= cpu_to_be32(qman_fq_fqid(priv
->conf_fqs
[queue
]));
1871 /* Trace this Tx fd */
1872 trace_dpaa_tx_fd(priv
->net_dev
, egress_fq
, fd
);
1874 for (i
= 0; i
< DPAA_ENQUEUE_RETRIES
; i
++) {
1875 err
= qman_enqueue(egress_fq
, fd
);
1880 if (unlikely(err
< 0)) {
1881 percpu_stats
->tx_errors
++;
1882 percpu_stats
->tx_fifo_errors
++;
1886 percpu_stats
->tx_packets
++;
1887 percpu_stats
->tx_bytes
+= qm_fd_get_length(fd
);
1892 static int dpaa_start_xmit(struct sk_buff
*skb
, struct net_device
*net_dev
)
1894 const int queue_mapping
= skb_get_queue_mapping(skb
);
1895 bool nonlinear
= skb_is_nonlinear(skb
);
1896 struct rtnl_link_stats64
*percpu_stats
;
1897 struct dpaa_percpu_priv
*percpu_priv
;
1898 struct dpaa_priv
*priv
;
1903 priv
= netdev_priv(net_dev
);
1904 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
1905 percpu_stats
= &percpu_priv
->stats
;
1907 qm_fd_clear_fd(&fd
);
1910 /* We're going to store the skb backpointer at the beginning
1911 * of the data buffer, so we need a privately owned skb
1913 * We've made sure skb is not shared in dev->priv_flags,
1914 * we need to verify the skb head is not cloned
1916 if (skb_cow_head(skb
, priv
->tx_headroom
))
1919 WARN_ON(skb_is_nonlinear(skb
));
1922 /* MAX_SKB_FRAGS is equal or larger than our dpaa_SGT_MAX_ENTRIES;
1923 * make sure we don't feed FMan with more fragments than it supports.
1926 likely(skb_shinfo(skb
)->nr_frags
< DPAA_SGT_MAX_ENTRIES
)) {
1927 /* Just create a S/G fd based on the skb */
1928 err
= skb_to_sg_fd(priv
, skb
, &fd
);
1929 percpu_priv
->tx_frag_skbuffs
++;
1931 /* If the egress skb contains more fragments than we support
1932 * we have no choice but to linearize it ourselves.
1934 if (unlikely(nonlinear
) && __skb_linearize(skb
))
1937 /* Finally, create a contig FD from this skb */
1938 err
= skb_to_contig_fd(priv
, skb
, &fd
, &offset
);
1940 if (unlikely(err
< 0))
1941 goto skb_to_fd_failed
;
1943 if (likely(dpaa_xmit(priv
, percpu_stats
, queue_mapping
, &fd
) == 0))
1944 return NETDEV_TX_OK
;
1946 dpaa_cleanup_tx_fd(priv
, &fd
);
1949 percpu_stats
->tx_errors
++;
1951 return NETDEV_TX_OK
;
1954 static void dpaa_rx_error(struct net_device
*net_dev
,
1955 const struct dpaa_priv
*priv
,
1956 struct dpaa_percpu_priv
*percpu_priv
,
1957 const struct qm_fd
*fd
,
1960 if (net_ratelimit())
1961 netif_err(priv
, hw
, net_dev
, "Err FD status = 0x%08x\n",
1962 be32_to_cpu(fd
->status
) & FM_FD_STAT_RX_ERRORS
);
1964 percpu_priv
->stats
.rx_errors
++;
1966 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_DMA
)
1967 percpu_priv
->rx_errors
.dme
++;
1968 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_PHYSICAL
)
1969 percpu_priv
->rx_errors
.fpe
++;
1970 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_SIZE
)
1971 percpu_priv
->rx_errors
.fse
++;
1972 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_PRS_HDR_ERR
)
1973 percpu_priv
->rx_errors
.phe
++;
1975 dpaa_fd_release(net_dev
, fd
);
1978 static void dpaa_tx_error(struct net_device
*net_dev
,
1979 const struct dpaa_priv
*priv
,
1980 struct dpaa_percpu_priv
*percpu_priv
,
1981 const struct qm_fd
*fd
,
1984 struct sk_buff
*skb
;
1986 if (net_ratelimit())
1987 netif_warn(priv
, hw
, net_dev
, "FD status = 0x%08x\n",
1988 be32_to_cpu(fd
->status
) & FM_FD_STAT_TX_ERRORS
);
1990 percpu_priv
->stats
.tx_errors
++;
1992 skb
= dpaa_cleanup_tx_fd(priv
, fd
);
1996 static int dpaa_eth_poll(struct napi_struct
*napi
, int budget
)
1998 struct dpaa_napi_portal
*np
=
1999 container_of(napi
, struct dpaa_napi_portal
, napi
);
2001 int cleaned
= qman_p_poll_dqrr(np
->p
, budget
);
2003 if (cleaned
< budget
) {
2004 napi_complete_done(napi
, cleaned
);
2005 qman_p_irqsource_add(np
->p
, QM_PIRQ_DQRI
);
2007 } else if (np
->down
) {
2008 qman_p_irqsource_add(np
->p
, QM_PIRQ_DQRI
);
2014 static void dpaa_tx_conf(struct net_device
*net_dev
,
2015 const struct dpaa_priv
*priv
,
2016 struct dpaa_percpu_priv
*percpu_priv
,
2017 const struct qm_fd
*fd
,
2020 struct sk_buff
*skb
;
2022 if (unlikely(be32_to_cpu(fd
->status
) & FM_FD_STAT_TX_ERRORS
)) {
2023 if (net_ratelimit())
2024 netif_warn(priv
, hw
, net_dev
, "FD status = 0x%08x\n",
2025 be32_to_cpu(fd
->status
) &
2026 FM_FD_STAT_TX_ERRORS
);
2028 percpu_priv
->stats
.tx_errors
++;
2031 percpu_priv
->tx_confirm
++;
2033 skb
= dpaa_cleanup_tx_fd(priv
, fd
);
2038 static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv
*percpu_priv
,
2039 struct qman_portal
*portal
)
2041 if (unlikely(in_irq() || !in_serving_softirq())) {
2042 /* Disable QMan IRQ and invoke NAPI */
2043 qman_p_irqsource_remove(portal
, QM_PIRQ_DQRI
);
2045 percpu_priv
->np
.p
= portal
;
2046 napi_schedule(&percpu_priv
->np
.napi
);
2047 percpu_priv
->in_interrupt
++;
2053 static enum qman_cb_dqrr_result
rx_error_dqrr(struct qman_portal
*portal
,
2055 const struct qm_dqrr_entry
*dq
)
2057 struct dpaa_fq
*dpaa_fq
= container_of(fq
, struct dpaa_fq
, fq_base
);
2058 struct dpaa_percpu_priv
*percpu_priv
;
2059 struct net_device
*net_dev
;
2060 struct dpaa_bp
*dpaa_bp
;
2061 struct dpaa_priv
*priv
;
2063 net_dev
= dpaa_fq
->net_dev
;
2064 priv
= netdev_priv(net_dev
);
2065 dpaa_bp
= dpaa_bpid2pool(dq
->fd
.bpid
);
2067 return qman_cb_dqrr_consume
;
2069 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2071 if (dpaa_eth_napi_schedule(percpu_priv
, portal
))
2072 return qman_cb_dqrr_stop
;
2074 if (dpaa_eth_refill_bpools(priv
))
2075 /* Unable to refill the buffer pool due to insufficient
2076 * system memory. Just release the frame back into the pool,
2077 * otherwise we'll soon end up with an empty buffer pool.
2079 dpaa_fd_release(net_dev
, &dq
->fd
);
2081 dpaa_rx_error(net_dev
, priv
, percpu_priv
, &dq
->fd
, fq
->fqid
);
2083 return qman_cb_dqrr_consume
;
2086 static enum qman_cb_dqrr_result
rx_default_dqrr(struct qman_portal
*portal
,
2088 const struct qm_dqrr_entry
*dq
)
2090 struct rtnl_link_stats64
*percpu_stats
;
2091 struct dpaa_percpu_priv
*percpu_priv
;
2092 const struct qm_fd
*fd
= &dq
->fd
;
2093 dma_addr_t addr
= qm_fd_addr(fd
);
2094 enum qm_fd_format fd_format
;
2095 struct net_device
*net_dev
;
2096 u32 fd_status
= fd
->status
;
2097 struct dpaa_bp
*dpaa_bp
;
2098 struct dpaa_priv
*priv
;
2099 unsigned int skb_len
;
2100 struct sk_buff
*skb
;
2103 fd_status
= be32_to_cpu(fd
->status
);
2104 fd_format
= qm_fd_get_format(fd
);
2105 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2106 priv
= netdev_priv(net_dev
);
2107 dpaa_bp
= dpaa_bpid2pool(dq
->fd
.bpid
);
2109 return qman_cb_dqrr_consume
;
2111 /* Trace the Rx fd */
2112 trace_dpaa_rx_fd(net_dev
, fq
, &dq
->fd
);
2114 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2115 percpu_stats
= &percpu_priv
->stats
;
2117 if (unlikely(dpaa_eth_napi_schedule(percpu_priv
, portal
)))
2118 return qman_cb_dqrr_stop
;
2120 /* Make sure we didn't run out of buffers */
2121 if (unlikely(dpaa_eth_refill_bpools(priv
))) {
2122 /* Unable to refill the buffer pool due to insufficient
2123 * system memory. Just release the frame back into the pool,
2124 * otherwise we'll soon end up with an empty buffer pool.
2126 dpaa_fd_release(net_dev
, &dq
->fd
);
2127 return qman_cb_dqrr_consume
;
2130 if (unlikely(fd_status
& FM_FD_STAT_RX_ERRORS
) != 0) {
2131 if (net_ratelimit())
2132 netif_warn(priv
, hw
, net_dev
, "FD status = 0x%08x\n",
2133 fd_status
& FM_FD_STAT_RX_ERRORS
);
2135 percpu_stats
->rx_errors
++;
2136 dpaa_fd_release(net_dev
, fd
);
2137 return qman_cb_dqrr_consume
;
2140 dpaa_bp
= dpaa_bpid2pool(fd
->bpid
);
2142 return qman_cb_dqrr_consume
;
2144 dma_unmap_single(dpaa_bp
->dev
, addr
, dpaa_bp
->size
, DMA_FROM_DEVICE
);
2146 /* prefetch the first 64 bytes of the frame or the SGT start */
2147 prefetch(phys_to_virt(addr
) + qm_fd_get_offset(fd
));
2149 fd_format
= qm_fd_get_format(fd
);
2150 /* The only FD types that we may receive are contig and S/G */
2151 WARN_ON((fd_format
!= qm_fd_contig
) && (fd_format
!= qm_fd_sg
));
2153 /* Account for either the contig buffer or the SGT buffer (depending on
2154 * which case we were in) having been removed from the pool.
2156 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
2159 if (likely(fd_format
== qm_fd_contig
))
2160 skb
= contig_fd_to_skb(priv
, fd
);
2162 skb
= sg_fd_to_skb(priv
, fd
);
2164 return qman_cb_dqrr_consume
;
2166 skb
->protocol
= eth_type_trans(skb
, net_dev
);
2170 if (unlikely(netif_receive_skb(skb
) == NET_RX_DROP
))
2171 return qman_cb_dqrr_consume
;
2173 percpu_stats
->rx_packets
++;
2174 percpu_stats
->rx_bytes
+= skb_len
;
2176 return qman_cb_dqrr_consume
;
2179 static enum qman_cb_dqrr_result
conf_error_dqrr(struct qman_portal
*portal
,
2181 const struct qm_dqrr_entry
*dq
)
2183 struct dpaa_percpu_priv
*percpu_priv
;
2184 struct net_device
*net_dev
;
2185 struct dpaa_priv
*priv
;
2187 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2188 priv
= netdev_priv(net_dev
);
2190 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2192 if (dpaa_eth_napi_schedule(percpu_priv
, portal
))
2193 return qman_cb_dqrr_stop
;
2195 dpaa_tx_error(net_dev
, priv
, percpu_priv
, &dq
->fd
, fq
->fqid
);
2197 return qman_cb_dqrr_consume
;
2200 static enum qman_cb_dqrr_result
conf_dflt_dqrr(struct qman_portal
*portal
,
2202 const struct qm_dqrr_entry
*dq
)
2204 struct dpaa_percpu_priv
*percpu_priv
;
2205 struct net_device
*net_dev
;
2206 struct dpaa_priv
*priv
;
2208 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2209 priv
= netdev_priv(net_dev
);
2212 trace_dpaa_tx_conf_fd(net_dev
, fq
, &dq
->fd
);
2214 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2216 if (dpaa_eth_napi_schedule(percpu_priv
, portal
))
2217 return qman_cb_dqrr_stop
;
2219 dpaa_tx_conf(net_dev
, priv
, percpu_priv
, &dq
->fd
, fq
->fqid
);
2221 return qman_cb_dqrr_consume
;
2224 static void egress_ern(struct qman_portal
*portal
,
2226 const union qm_mr_entry
*msg
)
2228 const struct qm_fd
*fd
= &msg
->ern
.fd
;
2229 struct dpaa_percpu_priv
*percpu_priv
;
2230 const struct dpaa_priv
*priv
;
2231 struct net_device
*net_dev
;
2232 struct sk_buff
*skb
;
2234 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2235 priv
= netdev_priv(net_dev
);
2236 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2238 percpu_priv
->stats
.tx_dropped
++;
2239 percpu_priv
->stats
.tx_fifo_errors
++;
2240 count_ern(percpu_priv
, msg
);
2242 skb
= dpaa_cleanup_tx_fd(priv
, fd
);
2243 dev_kfree_skb_any(skb
);
2246 static const struct dpaa_fq_cbs dpaa_fq_cbs
= {
2247 .rx_defq
= { .cb
= { .dqrr
= rx_default_dqrr
} },
2248 .tx_defq
= { .cb
= { .dqrr
= conf_dflt_dqrr
} },
2249 .rx_errq
= { .cb
= { .dqrr
= rx_error_dqrr
} },
2250 .tx_errq
= { .cb
= { .dqrr
= conf_error_dqrr
} },
2251 .egress_ern
= { .cb
= { .ern
= egress_ern
} }
2254 static void dpaa_eth_napi_enable(struct dpaa_priv
*priv
)
2256 struct dpaa_percpu_priv
*percpu_priv
;
2259 for_each_possible_cpu(i
) {
2260 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
2262 percpu_priv
->np
.down
= 0;
2263 napi_enable(&percpu_priv
->np
.napi
);
2267 static void dpaa_eth_napi_disable(struct dpaa_priv
*priv
)
2269 struct dpaa_percpu_priv
*percpu_priv
;
2272 for_each_possible_cpu(i
) {
2273 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
2275 percpu_priv
->np
.down
= 1;
2276 napi_disable(&percpu_priv
->np
.napi
);
2280 static int dpaa_open(struct net_device
*net_dev
)
2282 struct mac_device
*mac_dev
;
2283 struct dpaa_priv
*priv
;
2286 priv
= netdev_priv(net_dev
);
2287 mac_dev
= priv
->mac_dev
;
2288 dpaa_eth_napi_enable(priv
);
2290 net_dev
->phydev
= mac_dev
->init_phy(net_dev
, priv
->mac_dev
);
2291 if (!net_dev
->phydev
) {
2292 netif_err(priv
, ifup
, net_dev
, "init_phy() failed\n");
2294 goto phy_init_failed
;
2297 for (i
= 0; i
< ARRAY_SIZE(mac_dev
->port
); i
++) {
2298 err
= fman_port_enable(mac_dev
->port
[i
]);
2300 goto mac_start_failed
;
2303 err
= priv
->mac_dev
->start(mac_dev
);
2305 netif_err(priv
, ifup
, net_dev
, "mac_dev->start() = %d\n", err
);
2306 goto mac_start_failed
;
2309 netif_tx_start_all_queues(net_dev
);
2314 for (i
= 0; i
< ARRAY_SIZE(mac_dev
->port
); i
++)
2315 fman_port_disable(mac_dev
->port
[i
]);
2318 dpaa_eth_napi_disable(priv
);
2323 static int dpaa_eth_stop(struct net_device
*net_dev
)
2325 struct dpaa_priv
*priv
;
2328 err
= dpaa_stop(net_dev
);
2330 priv
= netdev_priv(net_dev
);
2331 dpaa_eth_napi_disable(priv
);
2336 static const struct net_device_ops dpaa_ops
= {
2337 .ndo_open
= dpaa_open
,
2338 .ndo_start_xmit
= dpaa_start_xmit
,
2339 .ndo_stop
= dpaa_eth_stop
,
2340 .ndo_tx_timeout
= dpaa_tx_timeout
,
2341 .ndo_get_stats64
= dpaa_get_stats64
,
2342 .ndo_set_mac_address
= dpaa_set_mac_address
,
2343 .ndo_validate_addr
= eth_validate_addr
,
2344 .ndo_set_rx_mode
= dpaa_set_rx_mode
,
2347 static int dpaa_napi_add(struct net_device
*net_dev
)
2349 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
2350 struct dpaa_percpu_priv
*percpu_priv
;
2353 for_each_possible_cpu(cpu
) {
2354 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, cpu
);
2356 netif_napi_add(net_dev
, &percpu_priv
->np
.napi
,
2357 dpaa_eth_poll
, NAPI_POLL_WEIGHT
);
2363 static void dpaa_napi_del(struct net_device
*net_dev
)
2365 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
2366 struct dpaa_percpu_priv
*percpu_priv
;
2369 for_each_possible_cpu(cpu
) {
2370 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, cpu
);
2372 netif_napi_del(&percpu_priv
->np
.napi
);
2376 static inline void dpaa_bp_free_pf(const struct dpaa_bp
*bp
,
2377 struct bm_buffer
*bmb
)
2379 dma_addr_t addr
= bm_buf_addr(bmb
);
2381 dma_unmap_single(bp
->dev
, addr
, bp
->size
, DMA_FROM_DEVICE
);
2383 skb_free_frag(phys_to_virt(addr
));
2386 /* Alloc the dpaa_bp struct and configure default values */
2387 static struct dpaa_bp
*dpaa_bp_alloc(struct device
*dev
)
2389 struct dpaa_bp
*dpaa_bp
;
2391 dpaa_bp
= devm_kzalloc(dev
, sizeof(*dpaa_bp
), GFP_KERNEL
);
2393 return ERR_PTR(-ENOMEM
);
2395 dpaa_bp
->bpid
= FSL_DPAA_BPID_INV
;
2396 dpaa_bp
->percpu_count
= devm_alloc_percpu(dev
, *dpaa_bp
->percpu_count
);
2397 dpaa_bp
->config_count
= FSL_DPAA_ETH_MAX_BUF_COUNT
;
2399 dpaa_bp
->seed_cb
= dpaa_bp_seed
;
2400 dpaa_bp
->free_buf_cb
= dpaa_bp_free_pf
;
2405 /* Place all ingress FQs (Rx Default, Rx Error) in a dedicated CGR.
2406 * We won't be sending congestion notifications to FMan; for now, we just use
2407 * this CGR to generate enqueue rejections to FMan in order to drop the frames
2408 * before they reach our ingress queues and eat up memory.
2410 static int dpaa_ingress_cgr_init(struct dpaa_priv
*priv
)
2412 struct qm_mcc_initcgr initcgr
;
2416 err
= qman_alloc_cgrid(&priv
->ingress_cgr
.cgrid
);
2418 if (netif_msg_drv(priv
))
2419 pr_err("Error %d allocating CGR ID\n", err
);
2423 /* Enable CS TD, but disable Congestion State Change Notifications. */
2424 memset(&initcgr
, 0, sizeof(initcgr
));
2425 initcgr
.we_mask
= cpu_to_be16(QM_CGR_WE_CS_THRES
);
2426 initcgr
.cgr
.cscn_en
= QM_CGR_EN
;
2427 cs_th
= DPAA_INGRESS_CS_THRESHOLD
;
2428 qm_cgr_cs_thres_set64(&initcgr
.cgr
.cs_thres
, cs_th
, 1);
2430 initcgr
.we_mask
|= cpu_to_be16(QM_CGR_WE_CSTD_EN
);
2431 initcgr
.cgr
.cstd_en
= QM_CGR_EN
;
2433 /* This CGR will be associated with the SWP affined to the current CPU.
2434 * However, we'll place all our ingress FQs in it.
2436 err
= qman_create_cgr(&priv
->ingress_cgr
, QMAN_CGR_FLAG_USE_INIT
,
2439 if (netif_msg_drv(priv
))
2440 pr_err("Error %d creating ingress CGR with ID %d\n",
2441 err
, priv
->ingress_cgr
.cgrid
);
2442 qman_release_cgrid(priv
->ingress_cgr
.cgrid
);
2445 if (netif_msg_drv(priv
))
2446 pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n",
2447 priv
->ingress_cgr
.cgrid
, priv
->mac_dev
->addr
);
2449 priv
->use_ingress_cgr
= true;
2455 static const struct of_device_id dpaa_match
[];
2457 static inline u16
dpaa_get_headroom(struct dpaa_buffer_layout
*bl
)
2461 /* The frame headroom must accommodate:
2462 * - the driver private data area
2463 * - parse results, hash results, timestamp if selected
2464 * If either hash results or time stamp are selected, both will
2465 * be copied to/from the frame headroom, as TS is located between PR and
2466 * HR in the IC and IC copy size has a granularity of 16bytes
2467 * (see description of FMBM_RICP and FMBM_TICP registers in DPAARM)
2469 * Also make sure the headroom is a multiple of data_align bytes
2471 headroom
= (u16
)(bl
->priv_data_size
+ DPAA_PARSE_RESULTS_SIZE
+
2472 DPAA_TIME_STAMP_SIZE
+ DPAA_HASH_RESULTS_SIZE
);
2474 return DPAA_FD_DATA_ALIGNMENT
? ALIGN(headroom
,
2475 DPAA_FD_DATA_ALIGNMENT
) :
2479 static int dpaa_eth_probe(struct platform_device
*pdev
)
2481 struct dpaa_bp
*dpaa_bps
[DPAA_BPS_NUM
] = {NULL
};
2482 struct dpaa_percpu_priv
*percpu_priv
;
2483 struct net_device
*net_dev
= NULL
;
2484 struct dpaa_fq
*dpaa_fq
, *tmp
;
2485 struct dpaa_priv
*priv
= NULL
;
2486 struct fm_port_fqs port_fqs
;
2487 struct mac_device
*mac_dev
;
2488 int err
= 0, i
, channel
;
2493 /* Allocate this early, so we can store relevant information in
2496 net_dev
= alloc_etherdev_mq(sizeof(*priv
), DPAA_ETH_TXQ_NUM
);
2498 dev_err(dev
, "alloc_etherdev_mq() failed\n");
2499 goto alloc_etherdev_mq_failed
;
2502 /* Do this here, so we can be verbose early */
2503 SET_NETDEV_DEV(net_dev
, dev
);
2504 dev_set_drvdata(dev
, net_dev
);
2506 priv
= netdev_priv(net_dev
);
2507 priv
->net_dev
= net_dev
;
2509 priv
->msg_enable
= netif_msg_init(debug
, DPAA_MSG_DEFAULT
);
2511 mac_dev
= dpaa_mac_dev_get(pdev
);
2512 if (IS_ERR(mac_dev
)) {
2513 dev_err(dev
, "dpaa_mac_dev_get() failed\n");
2514 err
= PTR_ERR(mac_dev
);
2515 goto mac_probe_failed
;
2518 /* If fsl_fm_max_frm is set to a higher value than the all-common 1500,
2519 * we choose conservatively and let the user explicitly set a higher
2520 * MTU via ifconfig. Otherwise, the user may end up with different MTUs
2522 * If on the other hand fsl_fm_max_frm has been chosen below 1500,
2523 * start with the maximum allowed.
2525 net_dev
->mtu
= min(dpaa_get_max_mtu(), ETH_DATA_LEN
);
2527 netdev_dbg(net_dev
, "Setting initial MTU on net device: %d\n",
2530 priv
->buf_layout
[RX
].priv_data_size
= DPAA_RX_PRIV_DATA_SIZE
; /* Rx */
2531 priv
->buf_layout
[TX
].priv_data_size
= DPAA_TX_PRIV_DATA_SIZE
; /* Tx */
2533 /* device used for DMA mapping */
2534 arch_setup_dma_ops(dev
, 0, 0, NULL
, false);
2535 err
= dma_coerce_mask_and_coherent(dev
, DMA_BIT_MASK(40));
2537 dev_err(dev
, "dma_coerce_mask_and_coherent() failed\n");
2538 goto dev_mask_failed
;
2542 for (i
= 0; i
< DPAA_BPS_NUM
; i
++) {
2545 dpaa_bps
[i
] = dpaa_bp_alloc(dev
);
2546 if (IS_ERR(dpaa_bps
[i
]))
2547 return PTR_ERR(dpaa_bps
[i
]);
2548 /* the raw size of the buffers used for reception */
2549 dpaa_bps
[i
]->raw_size
= bpool_buffer_raw_size(i
, DPAA_BPS_NUM
);
2550 /* avoid runtime computations by keeping the usable size here */
2551 dpaa_bps
[i
]->size
= dpaa_bp_size(dpaa_bps
[i
]->raw_size
);
2552 dpaa_bps
[i
]->dev
= dev
;
2554 err
= dpaa_bp_alloc_pool(dpaa_bps
[i
]);
2556 dpaa_bps_free(priv
);
2557 priv
->dpaa_bps
[i
] = NULL
;
2558 goto bp_create_failed
;
2560 priv
->dpaa_bps
[i
] = dpaa_bps
[i
];
2563 INIT_LIST_HEAD(&priv
->dpaa_fq_list
);
2565 memset(&port_fqs
, 0, sizeof(port_fqs
));
2567 err
= dpaa_alloc_all_fqs(dev
, &priv
->dpaa_fq_list
, &port_fqs
);
2569 dev_err(dev
, "dpaa_alloc_all_fqs() failed\n");
2570 goto fq_probe_failed
;
2573 priv
->mac_dev
= mac_dev
;
2575 channel
= dpaa_get_channel();
2577 dev_err(dev
, "dpaa_get_channel() failed\n");
2579 goto get_channel_failed
;
2582 priv
->channel
= (u16
)channel
;
2584 /* Start a thread that will walk the CPUs with affine portals
2585 * and add this pool channel to each's dequeue mask.
2587 dpaa_eth_add_channel(priv
->channel
);
2589 dpaa_fq_setup(priv
, &dpaa_fq_cbs
, priv
->mac_dev
->port
[TX
]);
2591 /* Create a congestion group for this netdev, with
2592 * dynamically-allocated CGR ID.
2593 * Must be executed after probing the MAC, but before
2594 * assigning the egress FQs to the CGRs.
2596 err
= dpaa_eth_cgr_init(priv
);
2598 dev_err(dev
, "Error initializing CGR\n");
2599 goto tx_cgr_init_failed
;
2602 err
= dpaa_ingress_cgr_init(priv
);
2604 dev_err(dev
, "Error initializing ingress CGR\n");
2605 goto rx_cgr_init_failed
;
2608 /* Add the FQs to the interface, and make them active */
2609 list_for_each_entry_safe(dpaa_fq
, tmp
, &priv
->dpaa_fq_list
, list
) {
2610 err
= dpaa_fq_init(dpaa_fq
, false);
2612 goto fq_alloc_failed
;
2615 priv
->tx_headroom
= dpaa_get_headroom(&priv
->buf_layout
[TX
]);
2616 priv
->rx_headroom
= dpaa_get_headroom(&priv
->buf_layout
[RX
]);
2618 /* All real interfaces need their ports initialized */
2619 dpaa_eth_init_ports(mac_dev
, dpaa_bps
, DPAA_BPS_NUM
, &port_fqs
,
2620 &priv
->buf_layout
[0], dev
);
2622 priv
->percpu_priv
= devm_alloc_percpu(dev
, *priv
->percpu_priv
);
2623 if (!priv
->percpu_priv
) {
2624 dev_err(dev
, "devm_alloc_percpu() failed\n");
2626 goto alloc_percpu_failed
;
2628 for_each_possible_cpu(i
) {
2629 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
2630 memset(percpu_priv
, 0, sizeof(*percpu_priv
));
2633 /* Initialize NAPI */
2634 err
= dpaa_napi_add(net_dev
);
2636 goto napi_add_failed
;
2638 err
= dpaa_netdev_init(net_dev
, &dpaa_ops
, tx_timeout
);
2640 goto netdev_init_failed
;
2642 dpaa_eth_sysfs_init(&net_dev
->dev
);
2644 netif_info(priv
, probe
, net_dev
, "Probed interface %s\n",
2651 dpaa_napi_del(net_dev
);
2652 alloc_percpu_failed
:
2653 dpaa_fq_free(dev
, &priv
->dpaa_fq_list
);
2655 qman_delete_cgr_safe(&priv
->ingress_cgr
);
2656 qman_release_cgrid(priv
->ingress_cgr
.cgrid
);
2658 qman_delete_cgr_safe(&priv
->cgr_data
.cgr
);
2659 qman_release_cgrid(priv
->cgr_data
.cgr
.cgrid
);
2662 dpaa_bps_free(priv
);
2667 dev_set_drvdata(dev
, NULL
);
2668 free_netdev(net_dev
);
2669 alloc_etherdev_mq_failed
:
2670 for (i
= 0; i
< DPAA_BPS_NUM
&& dpaa_bps
[i
]; i
++) {
2671 if (atomic_read(&dpaa_bps
[i
]->refs
) == 0)
2672 devm_kfree(dev
, dpaa_bps
[i
]);
2677 static int dpaa_remove(struct platform_device
*pdev
)
2679 struct net_device
*net_dev
;
2680 struct dpaa_priv
*priv
;
2685 net_dev
= dev_get_drvdata(dev
);
2687 priv
= netdev_priv(net_dev
);
2689 dpaa_eth_sysfs_remove(dev
);
2691 dev_set_drvdata(dev
, NULL
);
2692 unregister_netdev(net_dev
);
2694 err
= dpaa_fq_free(dev
, &priv
->dpaa_fq_list
);
2696 qman_delete_cgr_safe(&priv
->ingress_cgr
);
2697 qman_release_cgrid(priv
->ingress_cgr
.cgrid
);
2698 qman_delete_cgr_safe(&priv
->cgr_data
.cgr
);
2699 qman_release_cgrid(priv
->cgr_data
.cgr
.cgrid
);
2701 dpaa_napi_del(net_dev
);
2703 dpaa_bps_free(priv
);
2705 free_netdev(net_dev
);
2710 static struct platform_device_id dpaa_devtype
[] = {
2712 .name
= "dpaa-ethernet",
2717 MODULE_DEVICE_TABLE(platform
, dpaa_devtype
);
2719 static struct platform_driver dpaa_driver
= {
2721 .name
= KBUILD_MODNAME
,
2723 .id_table
= dpaa_devtype
,
2724 .probe
= dpaa_eth_probe
,
2725 .remove
= dpaa_remove
2728 static int __init
dpaa_load(void)
2732 pr_debug("FSL DPAA Ethernet driver\n");
2734 /* initialize dpaa_eth mirror values */
2735 dpaa_rx_extra_headroom
= fman_get_rx_extra_headroom();
2736 dpaa_max_frm
= fman_get_max_frm();
2738 err
= platform_driver_register(&dpaa_driver
);
2740 pr_err("Error, platform_driver_register() = %d\n", err
);
2744 module_init(dpaa_load
);
2746 static void __exit
dpaa_unload(void)
2748 platform_driver_unregister(&dpaa_driver
);
2750 /* Only one channel is used and needs to be released after all
2751 * interfaces are removed
2753 dpaa_release_channel();
2755 module_exit(dpaa_unload
);
2757 MODULE_LICENSE("Dual BSD/GPL");
2758 MODULE_DESCRIPTION("FSL DPAA Ethernet driver");