1 /* Copyright 2008 - 2016 Freescale Semiconductor Inc.
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 #include <linux/init.h>
34 #include <linux/module.h>
35 #include <linux/of_platform.h>
36 #include <linux/of_mdio.h>
37 #include <linux/of_net.h>
39 #include <linux/if_arp.h>
40 #include <linux/if_vlan.h>
41 #include <linux/icmp.h>
43 #include <linux/ipv6.h>
44 #include <linux/udp.h>
45 #include <linux/tcp.h>
46 #include <linux/net.h>
47 #include <linux/skbuff.h>
48 #include <linux/etherdevice.h>
49 #include <linux/if_ether.h>
50 #include <linux/highmem.h>
51 #include <linux/percpu.h>
52 #include <linux/dma-mapping.h>
53 #include <linux/sort.h>
54 #include <soc/fsl/bman.h>
55 #include <soc/fsl/qman.h>
58 #include "fman_port.h"
62 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpaa files
63 * using trace events only need to #include <trace/events/sched.h>
65 #define CREATE_TRACE_POINTS
66 #include "dpaa_eth_trace.h"
68 static int debug
= -1;
69 module_param(debug
, int, 0444);
70 MODULE_PARM_DESC(debug
, "Module/Driver verbosity level (0=none,...,16=all)");
72 static u16 tx_timeout
= 1000;
73 module_param(tx_timeout
, ushort
, 0444);
74 MODULE_PARM_DESC(tx_timeout
, "The Tx timeout in ms");
76 #define FM_FD_STAT_RX_ERRORS \
77 (FM_FD_ERR_DMA | FM_FD_ERR_PHYSICAL | \
78 FM_FD_ERR_SIZE | FM_FD_ERR_CLS_DISCARD | \
79 FM_FD_ERR_EXTRACTION | FM_FD_ERR_NO_SCHEME | \
80 FM_FD_ERR_PRS_TIMEOUT | FM_FD_ERR_PRS_ILL_INSTRUCT | \
81 FM_FD_ERR_PRS_HDR_ERR)
83 #define FM_FD_STAT_TX_ERRORS \
84 (FM_FD_ERR_UNSUPPORTED_FORMAT | \
85 FM_FD_ERR_LENGTH | FM_FD_ERR_DMA)
87 #define DPAA_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
88 NETIF_MSG_LINK | NETIF_MSG_IFUP | \
91 #define DPAA_INGRESS_CS_THRESHOLD 0x10000000
92 /* Ingress congestion threshold on FMan ports
93 * The size in bytes of the ingress tail-drop threshold on FMan ports.
94 * Traffic piling up above this value will be rejected by QMan and discarded
98 /* Size in bytes of the FQ taildrop threshold */
99 #define DPAA_FQ_TD 0x200000
101 #define DPAA_CS_THRESHOLD_1G 0x06000000
102 /* Egress congestion threshold on 1G ports, range 0x1000 .. 0x10000000
103 * The size in bytes of the egress Congestion State notification threshold on
104 * 1G ports. The 1G dTSECs can quite easily be flooded by cores doing Tx in a
105 * tight loop (e.g. by sending UDP datagrams at "while(1) speed"),
106 * and the larger the frame size, the more acute the problem.
107 * So we have to find a balance between these factors:
108 * - avoiding the device staying congested for a prolonged time (risking
109 * the netdev watchdog to fire - see also the tx_timeout module param);
110 * - affecting performance of protocols such as TCP, which otherwise
111 * behave well under the congestion notification mechanism;
112 * - preventing the Tx cores from tightly-looping (as if the congestion
113 * threshold was too low to be effective);
114 * - running out of memory if the CS threshold is set too high.
117 #define DPAA_CS_THRESHOLD_10G 0x10000000
118 /* The size in bytes of the egress Congestion State notification threshold on
119 * 10G ports, range 0x1000 .. 0x10000000
122 /* Largest value that the FQD's OAL field can hold */
123 #define FSL_QMAN_MAX_OAL 127
125 /* Default alignment for start of data in an Rx FD */
126 #define DPAA_FD_DATA_ALIGNMENT 16
128 /* Values for the L3R field of the FM Parse Results
130 /* L3 Type field: First IP Present IPv4 */
131 #define FM_L3_PARSE_RESULT_IPV4 0x8000
132 /* L3 Type field: First IP Present IPv6 */
133 #define FM_L3_PARSE_RESULT_IPV6 0x4000
134 /* Values for the L4R field of the FM Parse Results */
135 /* L4 Type field: UDP */
136 #define FM_L4_PARSE_RESULT_UDP 0x40
137 /* L4 Type field: TCP */
138 #define FM_L4_PARSE_RESULT_TCP 0x20
140 #define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
141 #define DPAA_BUFF_RELEASE_MAX 8 /* maximum number of buffers released at once */
143 #define FSL_DPAA_BPID_INV 0xff
144 #define FSL_DPAA_ETH_MAX_BUF_COUNT 128
145 #define FSL_DPAA_ETH_REFILL_THRESHOLD 80
147 #define DPAA_TX_PRIV_DATA_SIZE 16
148 #define DPAA_PARSE_RESULTS_SIZE sizeof(struct fman_prs_result)
149 #define DPAA_TIME_STAMP_SIZE 8
150 #define DPAA_HASH_RESULTS_SIZE 8
151 #define DPAA_RX_PRIV_DATA_SIZE (u16)(DPAA_TX_PRIV_DATA_SIZE + \
152 dpaa_rx_extra_headroom)
154 #define DPAA_ETH_RX_QUEUES 128
156 #define DPAA_ENQUEUE_RETRIES 100000
158 enum port_type
{RX
, TX
};
161 struct dpaa_fq
*tx_defq
;
162 struct dpaa_fq
*tx_errq
;
163 struct dpaa_fq
*rx_defq
;
164 struct dpaa_fq
*rx_errq
;
167 /* All the dpa bps in use at any moment */
168 static struct dpaa_bp
*dpaa_bp_array
[BM_MAX_NUM_OF_POOLS
];
170 /* The raw buffer size must be cacheline aligned */
171 #define DPAA_BP_RAW_SIZE 4096
172 /* When using more than one buffer pool, the raw sizes are as follows:
175 * 3 bp: 1KB, 2KB, 4KB
176 * 4 bp: 1KB, 2KB, 4KB, 8KB
178 static inline size_t bpool_buffer_raw_size(u8 index
, u8 cnt
)
180 size_t res
= DPAA_BP_RAW_SIZE
/ 4;
183 for (i
= (cnt
< 3) ? cnt
: 3; i
< 3 + index
; i
++)
188 /* FMan-DMA requires 16-byte alignment for Rx buffers, but SKB_DATA_ALIGN is
189 * even stronger (SMP_CACHE_BYTES-aligned), so we just get away with that,
190 * via SKB_WITH_OVERHEAD(). We can't rely on netdev_alloc_frag() giving us
191 * half-page-aligned buffers, so we reserve some more space for start-of-buffer
194 #define dpaa_bp_size(raw_size) SKB_WITH_OVERHEAD((raw_size) - SMP_CACHE_BYTES)
196 static int dpaa_max_frm
;
198 static int dpaa_rx_extra_headroom
;
200 #define dpaa_get_max_mtu() \
201 (dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
203 static int dpaa_netdev_init(struct net_device
*net_dev
,
204 const struct net_device_ops
*dpaa_ops
,
207 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
208 struct device
*dev
= net_dev
->dev
.parent
;
209 struct dpaa_percpu_priv
*percpu_priv
;
213 /* Although we access another CPU's private data here
214 * we do it at initialization so it is safe
216 for_each_possible_cpu(i
) {
217 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
218 percpu_priv
->net_dev
= net_dev
;
221 net_dev
->netdev_ops
= dpaa_ops
;
222 mac_addr
= priv
->mac_dev
->addr
;
224 net_dev
->mem_start
= priv
->mac_dev
->res
->start
;
225 net_dev
->mem_end
= priv
->mac_dev
->res
->end
;
227 net_dev
->min_mtu
= ETH_MIN_MTU
;
228 net_dev
->max_mtu
= dpaa_get_max_mtu();
230 net_dev
->hw_features
|= (NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
233 net_dev
->hw_features
|= NETIF_F_SG
| NETIF_F_HIGHDMA
;
234 /* The kernels enables GSO automatically, if we declare NETIF_F_SG.
235 * For conformity, we'll still declare GSO explicitly.
237 net_dev
->features
|= NETIF_F_GSO
;
239 net_dev
->priv_flags
|= IFF_LIVE_ADDR_CHANGE
;
240 /* we do not want shared skbs on TX */
241 net_dev
->priv_flags
&= ~IFF_TX_SKB_SHARING
;
243 net_dev
->features
|= net_dev
->hw_features
;
244 net_dev
->vlan_features
= net_dev
->features
;
246 memcpy(net_dev
->perm_addr
, mac_addr
, net_dev
->addr_len
);
247 memcpy(net_dev
->dev_addr
, mac_addr
, net_dev
->addr_len
);
249 net_dev
->ethtool_ops
= &dpaa_ethtool_ops
;
251 net_dev
->needed_headroom
= priv
->tx_headroom
;
252 net_dev
->watchdog_timeo
= msecs_to_jiffies(tx_timeout
);
254 /* start without the RUNNING flag, phylib controls it later */
255 netif_carrier_off(net_dev
);
257 err
= register_netdev(net_dev
);
259 dev_err(dev
, "register_netdev() = %d\n", err
);
266 static int dpaa_stop(struct net_device
*net_dev
)
268 struct mac_device
*mac_dev
;
269 struct dpaa_priv
*priv
;
272 priv
= netdev_priv(net_dev
);
273 mac_dev
= priv
->mac_dev
;
275 netif_tx_stop_all_queues(net_dev
);
276 /* Allow the Fman (Tx) port to process in-flight frames before we
277 * try switching it off.
279 usleep_range(5000, 10000);
281 err
= mac_dev
->stop(mac_dev
);
283 netif_err(priv
, ifdown
, net_dev
, "mac_dev->stop() = %d\n",
286 for (i
= 0; i
< ARRAY_SIZE(mac_dev
->port
); i
++) {
287 error
= fman_port_disable(mac_dev
->port
[i
]);
293 phy_disconnect(net_dev
->phydev
);
294 net_dev
->phydev
= NULL
;
299 static void dpaa_tx_timeout(struct net_device
*net_dev
)
301 struct dpaa_percpu_priv
*percpu_priv
;
302 const struct dpaa_priv
*priv
;
304 priv
= netdev_priv(net_dev
);
305 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
307 netif_crit(priv
, timer
, net_dev
, "Transmit timeout latency: %u ms\n",
308 jiffies_to_msecs(jiffies
- dev_trans_start(net_dev
)));
310 percpu_priv
->stats
.tx_errors
++;
313 /* Calculates the statistics for the given device by adding the statistics
314 * collected by each CPU.
316 static struct rtnl_link_stats64
*dpaa_get_stats64(struct net_device
*net_dev
,
317 struct rtnl_link_stats64
*s
)
319 int numstats
= sizeof(struct rtnl_link_stats64
) / sizeof(u64
);
320 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
321 struct dpaa_percpu_priv
*percpu_priv
;
322 u64
*netstats
= (u64
*)s
;
326 for_each_possible_cpu(i
) {
327 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
329 cpustats
= (u64
*)&percpu_priv
->stats
;
331 /* add stats from all CPUs */
332 for (j
= 0; j
< numstats
; j
++)
333 netstats
[j
] += cpustats
[j
];
339 static struct mac_device
*dpaa_mac_dev_get(struct platform_device
*pdev
)
341 struct platform_device
*of_dev
;
342 struct dpaa_eth_data
*eth_data
;
343 struct device
*dpaa_dev
, *dev
;
344 struct device_node
*mac_node
;
345 struct mac_device
*mac_dev
;
347 dpaa_dev
= &pdev
->dev
;
348 eth_data
= dpaa_dev
->platform_data
;
350 return ERR_PTR(-ENODEV
);
352 mac_node
= eth_data
->mac_node
;
354 of_dev
= of_find_device_by_node(mac_node
);
356 dev_err(dpaa_dev
, "of_find_device_by_node(%s) failed\n",
357 mac_node
->full_name
);
358 of_node_put(mac_node
);
359 return ERR_PTR(-EINVAL
);
361 of_node_put(mac_node
);
365 mac_dev
= dev_get_drvdata(dev
);
367 dev_err(dpaa_dev
, "dev_get_drvdata(%s) failed\n",
369 return ERR_PTR(-EINVAL
);
375 static int dpaa_set_mac_address(struct net_device
*net_dev
, void *addr
)
377 const struct dpaa_priv
*priv
;
378 struct mac_device
*mac_dev
;
379 struct sockaddr old_addr
;
382 priv
= netdev_priv(net_dev
);
384 memcpy(old_addr
.sa_data
, net_dev
->dev_addr
, ETH_ALEN
);
386 err
= eth_mac_addr(net_dev
, addr
);
388 netif_err(priv
, drv
, net_dev
, "eth_mac_addr() = %d\n", err
);
392 mac_dev
= priv
->mac_dev
;
394 err
= mac_dev
->change_addr(mac_dev
->fman_mac
,
395 (enet_addr_t
*)net_dev
->dev_addr
);
397 netif_err(priv
, drv
, net_dev
, "mac_dev->change_addr() = %d\n",
399 /* reverting to previous address */
400 eth_mac_addr(net_dev
, &old_addr
);
408 static void dpaa_set_rx_mode(struct net_device
*net_dev
)
410 const struct dpaa_priv
*priv
;
413 priv
= netdev_priv(net_dev
);
415 if (!!(net_dev
->flags
& IFF_PROMISC
) != priv
->mac_dev
->promisc
) {
416 priv
->mac_dev
->promisc
= !priv
->mac_dev
->promisc
;
417 err
= priv
->mac_dev
->set_promisc(priv
->mac_dev
->fman_mac
,
418 priv
->mac_dev
->promisc
);
420 netif_err(priv
, drv
, net_dev
,
421 "mac_dev->set_promisc() = %d\n",
425 err
= priv
->mac_dev
->set_multi(net_dev
, priv
->mac_dev
);
427 netif_err(priv
, drv
, net_dev
, "mac_dev->set_multi() = %d\n",
431 static struct dpaa_bp
*dpaa_bpid2pool(int bpid
)
433 if (WARN_ON(bpid
< 0 || bpid
>= BM_MAX_NUM_OF_POOLS
))
436 return dpaa_bp_array
[bpid
];
439 /* checks if this bpool is already allocated */
440 static bool dpaa_bpid2pool_use(int bpid
)
442 if (dpaa_bpid2pool(bpid
)) {
443 atomic_inc(&dpaa_bp_array
[bpid
]->refs
);
450 /* called only once per bpid by dpaa_bp_alloc_pool() */
451 static void dpaa_bpid2pool_map(int bpid
, struct dpaa_bp
*dpaa_bp
)
453 dpaa_bp_array
[bpid
] = dpaa_bp
;
454 atomic_set(&dpaa_bp
->refs
, 1);
457 static int dpaa_bp_alloc_pool(struct dpaa_bp
*dpaa_bp
)
461 if (dpaa_bp
->size
== 0 || dpaa_bp
->config_count
== 0) {
462 pr_err("%s: Buffer pool is not properly initialized! Missing size or initial number of buffers\n",
467 /* If the pool is already specified, we only create one per bpid */
468 if (dpaa_bp
->bpid
!= FSL_DPAA_BPID_INV
&&
469 dpaa_bpid2pool_use(dpaa_bp
->bpid
))
472 if (dpaa_bp
->bpid
== FSL_DPAA_BPID_INV
) {
473 dpaa_bp
->pool
= bman_new_pool();
474 if (!dpaa_bp
->pool
) {
475 pr_err("%s: bman_new_pool() failed\n",
480 dpaa_bp
->bpid
= (u8
)bman_get_bpid(dpaa_bp
->pool
);
483 if (dpaa_bp
->seed_cb
) {
484 err
= dpaa_bp
->seed_cb(dpaa_bp
);
486 goto pool_seed_failed
;
489 dpaa_bpid2pool_map(dpaa_bp
->bpid
, dpaa_bp
);
494 pr_err("%s: pool seeding failed\n", __func__
);
495 bman_free_pool(dpaa_bp
->pool
);
500 /* remove and free all the buffers from the given buffer pool */
501 static void dpaa_bp_drain(struct dpaa_bp
*bp
)
507 struct bm_buffer bmb
[8];
510 ret
= bman_acquire(bp
->pool
, bmb
, num
);
513 /* we have less than 8 buffers left;
514 * drain them one by one
520 /* Pool is fully drained */
526 for (i
= 0; i
< num
; i
++)
527 bp
->free_buf_cb(bp
, &bmb
[i
]);
531 static void dpaa_bp_free(struct dpaa_bp
*dpaa_bp
)
533 struct dpaa_bp
*bp
= dpaa_bpid2pool(dpaa_bp
->bpid
);
535 /* the mapping between bpid and dpaa_bp is done very late in the
536 * allocation procedure; if something failed before the mapping, the bp
537 * was not configured, therefore we don't need the below instructions
542 if (!atomic_dec_and_test(&bp
->refs
))
548 dpaa_bp_array
[bp
->bpid
] = NULL
;
549 bman_free_pool(bp
->pool
);
552 static void dpaa_bps_free(struct dpaa_priv
*priv
)
556 for (i
= 0; i
< DPAA_BPS_NUM
; i
++)
557 dpaa_bp_free(priv
->dpaa_bps
[i
]);
560 /* Use multiple WQs for FQ assignment:
561 * - Tx Confirmation queues go to WQ1.
562 * - Rx Error and Tx Error queues go to WQ2 (giving them a better chance
563 * to be scheduled, in case there are many more FQs in WQ3).
564 * - Rx Default and Tx queues go to WQ3 (no differentiation between
565 * Rx and Tx traffic).
566 * This ensures that Tx-confirmed buffers are timely released. In particular,
567 * it avoids congestion on the Tx Confirm FQs, which can pile up PFDRs if they
568 * are greatly outnumbered by other FQs in the system, while
569 * dequeue scheduling is round-robin.
571 static inline void dpaa_assign_wq(struct dpaa_fq
*fq
)
573 switch (fq
->fq_type
) {
574 case FQ_TYPE_TX_CONFIRM
:
575 case FQ_TYPE_TX_CONF_MQ
:
578 case FQ_TYPE_RX_ERROR
:
579 case FQ_TYPE_TX_ERROR
:
582 case FQ_TYPE_RX_DEFAULT
:
587 WARN(1, "Invalid FQ type %d for FQID %d!\n",
588 fq
->fq_type
, fq
->fqid
);
592 static struct dpaa_fq
*dpaa_fq_alloc(struct device
*dev
,
593 u32 start
, u32 count
,
594 struct list_head
*list
,
595 enum dpaa_fq_type fq_type
)
597 struct dpaa_fq
*dpaa_fq
;
600 dpaa_fq
= devm_kzalloc(dev
, sizeof(*dpaa_fq
) * count
,
605 for (i
= 0; i
< count
; i
++) {
606 dpaa_fq
[i
].fq_type
= fq_type
;
607 dpaa_fq
[i
].fqid
= start
? start
+ i
: 0;
608 list_add_tail(&dpaa_fq
[i
].list
, list
);
611 for (i
= 0; i
< count
; i
++)
612 dpaa_assign_wq(dpaa_fq
+ i
);
617 static int dpaa_alloc_all_fqs(struct device
*dev
, struct list_head
*list
,
618 struct fm_port_fqs
*port_fqs
)
620 struct dpaa_fq
*dpaa_fq
;
622 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_RX_ERROR
);
624 goto fq_alloc_failed
;
626 port_fqs
->rx_errq
= &dpaa_fq
[0];
628 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_RX_DEFAULT
);
630 goto fq_alloc_failed
;
632 port_fqs
->rx_defq
= &dpaa_fq
[0];
634 if (!dpaa_fq_alloc(dev
, 0, DPAA_ETH_TXQ_NUM
, list
, FQ_TYPE_TX_CONF_MQ
))
635 goto fq_alloc_failed
;
637 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_TX_ERROR
);
639 goto fq_alloc_failed
;
641 port_fqs
->tx_errq
= &dpaa_fq
[0];
643 dpaa_fq
= dpaa_fq_alloc(dev
, 0, 1, list
, FQ_TYPE_TX_CONFIRM
);
645 goto fq_alloc_failed
;
647 port_fqs
->tx_defq
= &dpaa_fq
[0];
649 if (!dpaa_fq_alloc(dev
, 0, DPAA_ETH_TXQ_NUM
, list
, FQ_TYPE_TX
))
650 goto fq_alloc_failed
;
655 dev_err(dev
, "dpaa_fq_alloc() failed\n");
659 static u32 rx_pool_channel
;
660 static DEFINE_SPINLOCK(rx_pool_channel_init
);
662 static int dpaa_get_channel(void)
664 spin_lock(&rx_pool_channel_init
);
665 if (!rx_pool_channel
) {
669 ret
= qman_alloc_pool(&pool
);
672 rx_pool_channel
= pool
;
674 spin_unlock(&rx_pool_channel_init
);
675 if (!rx_pool_channel
)
677 return rx_pool_channel
;
680 static void dpaa_release_channel(void)
682 qman_release_pool(rx_pool_channel
);
685 static void dpaa_eth_add_channel(u16 channel
)
687 u32 pool
= QM_SDQCR_CHANNELS_POOL_CONV(channel
);
688 const cpumask_t
*cpus
= qman_affine_cpus();
689 struct qman_portal
*portal
;
692 for_each_cpu(cpu
, cpus
) {
693 portal
= qman_get_affine_portal(cpu
);
694 qman_p_static_dequeue_add(portal
, pool
);
698 /* Congestion group state change notification callback.
699 * Stops the device's egress queues while they are congested and
700 * wakes them upon exiting congested state.
701 * Also updates some CGR-related stats.
703 static void dpaa_eth_cgscn(struct qman_portal
*qm
, struct qman_cgr
*cgr
,
706 struct dpaa_priv
*priv
= (struct dpaa_priv
*)container_of(cgr
,
707 struct dpaa_priv
, cgr_data
.cgr
);
710 priv
->cgr_data
.congestion_start_jiffies
= jiffies
;
711 netif_tx_stop_all_queues(priv
->net_dev
);
712 priv
->cgr_data
.cgr_congested_count
++;
714 priv
->cgr_data
.congested_jiffies
+=
715 (jiffies
- priv
->cgr_data
.congestion_start_jiffies
);
716 netif_tx_wake_all_queues(priv
->net_dev
);
720 static int dpaa_eth_cgr_init(struct dpaa_priv
*priv
)
722 struct qm_mcc_initcgr initcgr
;
726 err
= qman_alloc_cgrid(&priv
->cgr_data
.cgr
.cgrid
);
728 if (netif_msg_drv(priv
))
729 pr_err("%s: Error %d allocating CGR ID\n",
733 priv
->cgr_data
.cgr
.cb
= dpaa_eth_cgscn
;
735 /* Enable Congestion State Change Notifications and CS taildrop */
736 initcgr
.we_mask
= cpu_to_be16(QM_CGR_WE_CSCN_EN
| QM_CGR_WE_CS_THRES
);
737 initcgr
.cgr
.cscn_en
= QM_CGR_EN
;
739 /* Set different thresholds based on the MAC speed.
740 * This may turn suboptimal if the MAC is reconfigured at a speed
741 * lower than its max, e.g. if a dTSEC later negotiates a 100Mbps link.
742 * In such cases, we ought to reconfigure the threshold, too.
744 if (priv
->mac_dev
->if_support
& SUPPORTED_10000baseT_Full
)
745 cs_th
= DPAA_CS_THRESHOLD_10G
;
747 cs_th
= DPAA_CS_THRESHOLD_1G
;
748 qm_cgr_cs_thres_set64(&initcgr
.cgr
.cs_thres
, cs_th
, 1);
750 initcgr
.we_mask
|= cpu_to_be16(QM_CGR_WE_CSTD_EN
);
751 initcgr
.cgr
.cstd_en
= QM_CGR_EN
;
753 err
= qman_create_cgr(&priv
->cgr_data
.cgr
, QMAN_CGR_FLAG_USE_INIT
,
756 if (netif_msg_drv(priv
))
757 pr_err("%s: Error %d creating CGR with ID %d\n",
758 __func__
, err
, priv
->cgr_data
.cgr
.cgrid
);
759 qman_release_cgrid(priv
->cgr_data
.cgr
.cgrid
);
762 if (netif_msg_drv(priv
))
763 pr_debug("Created CGR %d for netdev with hwaddr %pM on QMan channel %d\n",
764 priv
->cgr_data
.cgr
.cgrid
, priv
->mac_dev
->addr
,
765 priv
->cgr_data
.cgr
.chan
);
771 static inline void dpaa_setup_ingress(const struct dpaa_priv
*priv
,
773 const struct qman_fq
*template)
775 fq
->fq_base
= *template;
776 fq
->net_dev
= priv
->net_dev
;
778 fq
->flags
= QMAN_FQ_FLAG_NO_ENQUEUE
;
779 fq
->channel
= priv
->channel
;
782 static inline void dpaa_setup_egress(const struct dpaa_priv
*priv
,
784 struct fman_port
*port
,
785 const struct qman_fq
*template)
787 fq
->fq_base
= *template;
788 fq
->net_dev
= priv
->net_dev
;
791 fq
->flags
= QMAN_FQ_FLAG_TO_DCPORTAL
;
792 fq
->channel
= (u16
)fman_port_get_qman_channel_id(port
);
794 fq
->flags
= QMAN_FQ_FLAG_NO_MODIFY
;
798 static void dpaa_fq_setup(struct dpaa_priv
*priv
,
799 const struct dpaa_fq_cbs
*fq_cbs
,
800 struct fman_port
*tx_port
)
802 int egress_cnt
= 0, conf_cnt
= 0, num_portals
= 0, cpu
;
803 const cpumask_t
*affine_cpus
= qman_affine_cpus();
804 u16 portals
[NR_CPUS
];
807 for_each_cpu(cpu
, affine_cpus
)
808 portals
[num_portals
++] = qman_affine_channel(cpu
);
809 if (num_portals
== 0)
810 dev_err(priv
->net_dev
->dev
.parent
,
811 "No Qman software (affine) channels found");
813 /* Initialize each FQ in the list */
814 list_for_each_entry(fq
, &priv
->dpaa_fq_list
, list
) {
815 switch (fq
->fq_type
) {
816 case FQ_TYPE_RX_DEFAULT
:
817 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->rx_defq
);
819 case FQ_TYPE_RX_ERROR
:
820 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->rx_errq
);
823 dpaa_setup_egress(priv
, fq
, tx_port
,
824 &fq_cbs
->egress_ern
);
825 /* If we have more Tx queues than the number of cores,
826 * just ignore the extra ones.
828 if (egress_cnt
< DPAA_ETH_TXQ_NUM
)
829 priv
->egress_fqs
[egress_cnt
++] = &fq
->fq_base
;
831 case FQ_TYPE_TX_CONF_MQ
:
832 priv
->conf_fqs
[conf_cnt
++] = &fq
->fq_base
;
834 case FQ_TYPE_TX_CONFIRM
:
835 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->tx_defq
);
837 case FQ_TYPE_TX_ERROR
:
838 dpaa_setup_ingress(priv
, fq
, &fq_cbs
->tx_errq
);
841 dev_warn(priv
->net_dev
->dev
.parent
,
842 "Unknown FQ type detected!\n");
847 /* Make sure all CPUs receive a corresponding Tx queue. */
848 while (egress_cnt
< DPAA_ETH_TXQ_NUM
) {
849 list_for_each_entry(fq
, &priv
->dpaa_fq_list
, list
) {
850 if (fq
->fq_type
!= FQ_TYPE_TX
)
852 priv
->egress_fqs
[egress_cnt
++] = &fq
->fq_base
;
853 if (egress_cnt
== DPAA_ETH_TXQ_NUM
)
859 static inline int dpaa_tx_fq_to_id(const struct dpaa_priv
*priv
,
860 struct qman_fq
*tx_fq
)
864 for (i
= 0; i
< DPAA_ETH_TXQ_NUM
; i
++)
865 if (priv
->egress_fqs
[i
] == tx_fq
)
871 static int dpaa_fq_init(struct dpaa_fq
*dpaa_fq
, bool td_enable
)
873 const struct dpaa_priv
*priv
;
874 struct qman_fq
*confq
= NULL
;
875 struct qm_mcc_initfq initfq
;
881 priv
= netdev_priv(dpaa_fq
->net_dev
);
882 dev
= dpaa_fq
->net_dev
->dev
.parent
;
884 if (dpaa_fq
->fqid
== 0)
885 dpaa_fq
->flags
|= QMAN_FQ_FLAG_DYNAMIC_FQID
;
887 dpaa_fq
->init
= !(dpaa_fq
->flags
& QMAN_FQ_FLAG_NO_MODIFY
);
889 err
= qman_create_fq(dpaa_fq
->fqid
, dpaa_fq
->flags
, &dpaa_fq
->fq_base
);
891 dev_err(dev
, "qman_create_fq() failed\n");
894 fq
= &dpaa_fq
->fq_base
;
897 memset(&initfq
, 0, sizeof(initfq
));
899 initfq
.we_mask
= cpu_to_be16(QM_INITFQ_WE_FQCTRL
);
900 /* Note: we may get to keep an empty FQ in cache */
901 initfq
.fqd
.fq_ctrl
= cpu_to_be16(QM_FQCTRL_PREFERINCACHE
);
903 /* Try to reduce the number of portal interrupts for
904 * Tx Confirmation FQs.
906 if (dpaa_fq
->fq_type
== FQ_TYPE_TX_CONFIRM
)
907 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_HOLDACTIVE
);
910 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_DESTWQ
);
912 qm_fqd_set_destwq(&initfq
.fqd
, dpaa_fq
->channel
, dpaa_fq
->wq
);
914 /* Put all egress queues in a congestion group of their own.
915 * Sensu stricto, the Tx confirmation queues are Rx FQs,
916 * rather than Tx - but they nonetheless account for the
917 * memory footprint on behalf of egress traffic. We therefore
918 * place them in the netdev's CGR, along with the Tx FQs.
920 if (dpaa_fq
->fq_type
== FQ_TYPE_TX
||
921 dpaa_fq
->fq_type
== FQ_TYPE_TX_CONFIRM
||
922 dpaa_fq
->fq_type
== FQ_TYPE_TX_CONF_MQ
) {
923 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CGID
);
924 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_CGE
);
925 initfq
.fqd
.cgid
= (u8
)priv
->cgr_data
.cgr
.cgrid
;
926 /* Set a fixed overhead accounting, in an attempt to
927 * reduce the impact of fixed-size skb shells and the
928 * driver's needed headroom on system memory. This is
929 * especially the case when the egress traffic is
930 * composed of small datagrams.
931 * Unfortunately, QMan's OAL value is capped to an
932 * insufficient value, but even that is better than
933 * no overhead accounting at all.
935 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_OAC
);
936 qm_fqd_set_oac(&initfq
.fqd
, QM_OAC_CG
);
937 qm_fqd_set_oal(&initfq
.fqd
,
938 min(sizeof(struct sk_buff
) +
940 (size_t)FSL_QMAN_MAX_OAL
));
944 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_TDTHRESH
);
945 qm_fqd_set_taildrop(&initfq
.fqd
, DPAA_FQ_TD
, 1);
946 initfq
.fqd
.fq_ctrl
= cpu_to_be16(QM_FQCTRL_TDE
);
949 if (dpaa_fq
->fq_type
== FQ_TYPE_TX
) {
950 queue_id
= dpaa_tx_fq_to_id(priv
, &dpaa_fq
->fq_base
);
952 confq
= priv
->conf_fqs
[queue_id
];
955 cpu_to_be16(QM_INITFQ_WE_CONTEXTA
);
956 /* ContextA: OVOM=1(use contextA2 bits instead of ICAD)
957 * A2V=1 (contextA A2 field is valid)
958 * A0V=1 (contextA A0 field is valid)
959 * B0V=1 (contextB field is valid)
960 * ContextA A2: EBD=1 (deallocate buffers inside FMan)
961 * ContextB B0(ASPID): 0 (absolute Virtual Storage ID)
963 qm_fqd_context_a_set64(&initfq
.fqd
,
964 0x1e00000080000000ULL
);
968 /* Put all the ingress queues in our "ingress CGR". */
969 if (priv
->use_ingress_cgr
&&
970 (dpaa_fq
->fq_type
== FQ_TYPE_RX_DEFAULT
||
971 dpaa_fq
->fq_type
== FQ_TYPE_RX_ERROR
)) {
972 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CGID
);
973 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_CGE
);
974 initfq
.fqd
.cgid
= (u8
)priv
->ingress_cgr
.cgrid
;
975 /* Set a fixed overhead accounting, just like for the
978 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_OAC
);
979 qm_fqd_set_oac(&initfq
.fqd
, QM_OAC_CG
);
980 qm_fqd_set_oal(&initfq
.fqd
,
981 min(sizeof(struct sk_buff
) +
983 (size_t)FSL_QMAN_MAX_OAL
));
986 /* Initialization common to all ingress queues */
987 if (dpaa_fq
->flags
& QMAN_FQ_FLAG_NO_ENQUEUE
) {
988 initfq
.we_mask
|= cpu_to_be16(QM_INITFQ_WE_CONTEXTA
);
989 initfq
.fqd
.fq_ctrl
|= cpu_to_be16(QM_FQCTRL_HOLDACTIVE
);
990 initfq
.fqd
.context_a
.stashing
.exclusive
=
991 QM_STASHING_EXCL_DATA
| QM_STASHING_EXCL_CTX
|
992 QM_STASHING_EXCL_ANNOTATION
;
993 qm_fqd_set_stashing(&initfq
.fqd
, 1, 2,
994 DIV_ROUND_UP(sizeof(struct qman_fq
),
998 err
= qman_init_fq(fq
, QMAN_INITFQ_FLAG_SCHED
, &initfq
);
1000 dev_err(dev
, "qman_init_fq(%u) = %d\n",
1001 qman_fq_fqid(fq
), err
);
1002 qman_destroy_fq(fq
);
1007 dpaa_fq
->fqid
= qman_fq_fqid(fq
);
1012 static int dpaa_fq_free_entry(struct device
*dev
, struct qman_fq
*fq
)
1014 const struct dpaa_priv
*priv
;
1015 struct dpaa_fq
*dpaa_fq
;
1020 dpaa_fq
= container_of(fq
, struct dpaa_fq
, fq_base
);
1021 priv
= netdev_priv(dpaa_fq
->net_dev
);
1023 if (dpaa_fq
->init
) {
1024 err
= qman_retire_fq(fq
, NULL
);
1025 if (err
< 0 && netif_msg_drv(priv
))
1026 dev_err(dev
, "qman_retire_fq(%u) = %d\n",
1027 qman_fq_fqid(fq
), err
);
1029 error
= qman_oos_fq(fq
);
1030 if (error
< 0 && netif_msg_drv(priv
)) {
1031 dev_err(dev
, "qman_oos_fq(%u) = %d\n",
1032 qman_fq_fqid(fq
), error
);
1038 qman_destroy_fq(fq
);
1039 list_del(&dpaa_fq
->list
);
1044 static int dpaa_fq_free(struct device
*dev
, struct list_head
*list
)
1046 struct dpaa_fq
*dpaa_fq
, *tmp
;
1050 list_for_each_entry_safe(dpaa_fq
, tmp
, list
, list
) {
1051 error
= dpaa_fq_free_entry(dev
, (struct qman_fq
*)dpaa_fq
);
1052 if (error
< 0 && err
>= 0)
1059 static void dpaa_eth_init_tx_port(struct fman_port
*port
, struct dpaa_fq
*errq
,
1060 struct dpaa_fq
*defq
,
1061 struct dpaa_buffer_layout
*buf_layout
)
1063 struct fman_buffer_prefix_content buf_prefix_content
;
1064 struct fman_port_params params
;
1067 memset(¶ms
, 0, sizeof(params
));
1068 memset(&buf_prefix_content
, 0, sizeof(buf_prefix_content
));
1070 buf_prefix_content
.priv_data_size
= buf_layout
->priv_data_size
;
1071 buf_prefix_content
.pass_prs_result
= true;
1072 buf_prefix_content
.pass_hash_result
= true;
1073 buf_prefix_content
.pass_time_stamp
= false;
1074 buf_prefix_content
.data_align
= DPAA_FD_DATA_ALIGNMENT
;
1076 params
.specific_params
.non_rx_params
.err_fqid
= errq
->fqid
;
1077 params
.specific_params
.non_rx_params
.dflt_fqid
= defq
->fqid
;
1079 err
= fman_port_config(port
, ¶ms
);
1081 pr_err("%s: fman_port_config failed\n", __func__
);
1083 err
= fman_port_cfg_buf_prefix_content(port
, &buf_prefix_content
);
1085 pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1088 err
= fman_port_init(port
);
1090 pr_err("%s: fm_port_init failed\n", __func__
);
1093 static void dpaa_eth_init_rx_port(struct fman_port
*port
, struct dpaa_bp
**bps
,
1094 size_t count
, struct dpaa_fq
*errq
,
1095 struct dpaa_fq
*defq
,
1096 struct dpaa_buffer_layout
*buf_layout
)
1098 struct fman_buffer_prefix_content buf_prefix_content
;
1099 struct fman_port_rx_params
*rx_p
;
1100 struct fman_port_params params
;
1103 memset(¶ms
, 0, sizeof(params
));
1104 memset(&buf_prefix_content
, 0, sizeof(buf_prefix_content
));
1106 buf_prefix_content
.priv_data_size
= buf_layout
->priv_data_size
;
1107 buf_prefix_content
.pass_prs_result
= true;
1108 buf_prefix_content
.pass_hash_result
= true;
1109 buf_prefix_content
.pass_time_stamp
= false;
1110 buf_prefix_content
.data_align
= DPAA_FD_DATA_ALIGNMENT
;
1112 rx_p
= ¶ms
.specific_params
.rx_params
;
1113 rx_p
->err_fqid
= errq
->fqid
;
1114 rx_p
->dflt_fqid
= defq
->fqid
;
1116 count
= min(ARRAY_SIZE(rx_p
->ext_buf_pools
.ext_buf_pool
), count
);
1117 rx_p
->ext_buf_pools
.num_of_pools_used
= (u8
)count
;
1118 for (i
= 0; i
< count
; i
++) {
1119 rx_p
->ext_buf_pools
.ext_buf_pool
[i
].id
= bps
[i
]->bpid
;
1120 rx_p
->ext_buf_pools
.ext_buf_pool
[i
].size
= (u16
)bps
[i
]->size
;
1123 err
= fman_port_config(port
, ¶ms
);
1125 pr_err("%s: fman_port_config failed\n", __func__
);
1127 err
= fman_port_cfg_buf_prefix_content(port
, &buf_prefix_content
);
1129 pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1132 err
= fman_port_init(port
);
1134 pr_err("%s: fm_port_init failed\n", __func__
);
1137 static void dpaa_eth_init_ports(struct mac_device
*mac_dev
,
1138 struct dpaa_bp
**bps
, size_t count
,
1139 struct fm_port_fqs
*port_fqs
,
1140 struct dpaa_buffer_layout
*buf_layout
,
1143 struct fman_port
*rxport
= mac_dev
->port
[RX
];
1144 struct fman_port
*txport
= mac_dev
->port
[TX
];
1146 dpaa_eth_init_tx_port(txport
, port_fqs
->tx_errq
,
1147 port_fqs
->tx_defq
, &buf_layout
[TX
]);
1148 dpaa_eth_init_rx_port(rxport
, bps
, count
, port_fqs
->rx_errq
,
1149 port_fqs
->rx_defq
, &buf_layout
[RX
]);
1152 static int dpaa_bman_release(const struct dpaa_bp
*dpaa_bp
,
1153 struct bm_buffer
*bmb
, int cnt
)
1157 err
= bman_release(dpaa_bp
->pool
, bmb
, cnt
);
1158 /* Should never occur, address anyway to avoid leaking the buffers */
1159 if (unlikely(WARN_ON(err
)) && dpaa_bp
->free_buf_cb
)
1161 dpaa_bp
->free_buf_cb(dpaa_bp
, &bmb
[cnt
]);
1166 static void dpaa_release_sgt_members(struct qm_sg_entry
*sgt
)
1168 struct bm_buffer bmb
[DPAA_BUFF_RELEASE_MAX
];
1169 struct dpaa_bp
*dpaa_bp
;
1172 memset(bmb
, 0, sizeof(bmb
));
1175 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1181 WARN_ON(qm_sg_entry_is_ext(&sgt
[i
]));
1183 bm_buffer_set64(&bmb
[j
], qm_sg_entry_get64(&sgt
[i
]));
1186 } while (j
< ARRAY_SIZE(bmb
) &&
1187 !qm_sg_entry_is_final(&sgt
[i
- 1]) &&
1188 sgt
[i
- 1].bpid
== sgt
[i
].bpid
);
1190 dpaa_bman_release(dpaa_bp
, bmb
, j
);
1191 } while (!qm_sg_entry_is_final(&sgt
[i
- 1]));
1194 static void dpaa_fd_release(const struct net_device
*net_dev
,
1195 const struct qm_fd
*fd
)
1197 struct qm_sg_entry
*sgt
;
1198 struct dpaa_bp
*dpaa_bp
;
1199 struct bm_buffer bmb
;
1204 bm_buffer_set64(&bmb
, qm_fd_addr(fd
));
1206 dpaa_bp
= dpaa_bpid2pool(fd
->bpid
);
1210 if (qm_fd_get_format(fd
) == qm_fd_sg
) {
1211 vaddr
= phys_to_virt(qm_fd_addr(fd
));
1212 sgt
= vaddr
+ qm_fd_get_offset(fd
);
1214 dma_unmap_single(dpaa_bp
->dev
, qm_fd_addr(fd
), dpaa_bp
->size
,
1217 dpaa_release_sgt_members(sgt
);
1219 addr
= dma_map_single(dpaa_bp
->dev
, vaddr
, dpaa_bp
->size
,
1221 if (dma_mapping_error(dpaa_bp
->dev
, addr
)) {
1222 dev_err(dpaa_bp
->dev
, "DMA mapping failed");
1225 bm_buffer_set64(&bmb
, addr
);
1228 dpaa_bman_release(dpaa_bp
, &bmb
, 1);
1231 static void count_ern(struct dpaa_percpu_priv
*percpu_priv
,
1232 const union qm_mr_entry
*msg
)
1234 switch (msg
->ern
.rc
& QM_MR_RC_MASK
) {
1235 case QM_MR_RC_CGR_TAILDROP
:
1236 percpu_priv
->ern_cnt
.cg_tdrop
++;
1239 percpu_priv
->ern_cnt
.wred
++;
1241 case QM_MR_RC_ERROR
:
1242 percpu_priv
->ern_cnt
.err_cond
++;
1244 case QM_MR_RC_ORPWINDOW_EARLY
:
1245 percpu_priv
->ern_cnt
.early_window
++;
1247 case QM_MR_RC_ORPWINDOW_LATE
:
1248 percpu_priv
->ern_cnt
.late_window
++;
1250 case QM_MR_RC_FQ_TAILDROP
:
1251 percpu_priv
->ern_cnt
.fq_tdrop
++;
1253 case QM_MR_RC_ORPWINDOW_RETIRED
:
1254 percpu_priv
->ern_cnt
.fq_retired
++;
1256 case QM_MR_RC_ORP_ZERO
:
1257 percpu_priv
->ern_cnt
.orp_zero
++;
1262 /* Turn on HW checksum computation for this outgoing frame.
1263 * If the current protocol is not something we support in this regard
1264 * (or if the stack has already computed the SW checksum), we do nothing.
1266 * Returns 0 if all goes well (or HW csum doesn't apply), and a negative value
1269 * Note that this function may modify the fd->cmd field and the skb data buffer
1270 * (the Parse Results area).
1272 static int dpaa_enable_tx_csum(struct dpaa_priv
*priv
,
1273 struct sk_buff
*skb
,
1275 char *parse_results
)
1277 struct fman_prs_result
*parse_result
;
1278 u16 ethertype
= ntohs(skb
->protocol
);
1279 struct ipv6hdr
*ipv6h
= NULL
;
1284 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1287 /* Note: L3 csum seems to be already computed in sw, but we can't choose
1288 * L4 alone from the FM configuration anyway.
1291 /* Fill in some fields of the Parse Results array, so the FMan
1292 * can find them as if they came from the FMan Parser.
1294 parse_result
= (struct fman_prs_result
*)parse_results
;
1296 /* If we're dealing with VLAN, get the real Ethernet type */
1297 if (ethertype
== ETH_P_8021Q
) {
1298 /* We can't always assume the MAC header is set correctly
1299 * by the stack, so reset to beginning of skb->data
1301 skb_reset_mac_header(skb
);
1302 ethertype
= ntohs(vlan_eth_hdr(skb
)->h_vlan_encapsulated_proto
);
1305 /* Fill in the relevant L3 parse result fields
1306 * and read the L4 protocol type
1308 switch (ethertype
) {
1310 parse_result
->l3r
= cpu_to_be16(FM_L3_PARSE_RESULT_IPV4
);
1313 l4_proto
= iph
->protocol
;
1316 parse_result
->l3r
= cpu_to_be16(FM_L3_PARSE_RESULT_IPV6
);
1317 ipv6h
= ipv6_hdr(skb
);
1319 l4_proto
= ipv6h
->nexthdr
;
1322 /* We shouldn't even be here */
1323 if (net_ratelimit())
1324 netif_alert(priv
, tx_err
, priv
->net_dev
,
1325 "Can't compute HW csum for L3 proto 0x%x\n",
1326 ntohs(skb
->protocol
));
1331 /* Fill in the relevant L4 parse result fields */
1334 parse_result
->l4r
= FM_L4_PARSE_RESULT_UDP
;
1337 parse_result
->l4r
= FM_L4_PARSE_RESULT_TCP
;
1340 if (net_ratelimit())
1341 netif_alert(priv
, tx_err
, priv
->net_dev
,
1342 "Can't compute HW csum for L4 proto 0x%x\n",
1348 /* At index 0 is IPOffset_1 as defined in the Parse Results */
1349 parse_result
->ip_off
[0] = (u8
)skb_network_offset(skb
);
1350 parse_result
->l4_off
= (u8
)skb_transport_offset(skb
);
1352 /* Enable L3 (and L4, if TCP or UDP) HW checksum. */
1353 fd
->cmd
|= cpu_to_be32(FM_FD_CMD_RPD
| FM_FD_CMD_DTC
);
1355 /* On P1023 and similar platforms fd->cmd interpretation could
1356 * be disabled by setting CONTEXT_A bit ICMD; currently this bit
1357 * is not set so we do not need to check; in the future, if/when
1358 * using context_a we need to check this bit
1365 static int dpaa_bp_add_8_bufs(const struct dpaa_bp
*dpaa_bp
)
1367 struct device
*dev
= dpaa_bp
->dev
;
1368 struct bm_buffer bmb
[8];
1373 for (i
= 0; i
< 8; i
++) {
1374 new_buf
= netdev_alloc_frag(dpaa_bp
->raw_size
);
1375 if (unlikely(!new_buf
)) {
1376 dev_err(dev
, "netdev_alloc_frag() failed, size %zu\n",
1378 goto release_previous_buffs
;
1380 new_buf
= PTR_ALIGN(new_buf
, SMP_CACHE_BYTES
);
1382 addr
= dma_map_single(dev
, new_buf
,
1383 dpaa_bp
->size
, DMA_FROM_DEVICE
);
1384 if (unlikely(dma_mapping_error(dev
, addr
))) {
1385 dev_err(dpaa_bp
->dev
, "DMA map failed");
1386 goto release_previous_buffs
;
1390 bm_buffer_set64(&bmb
[i
], addr
);
1394 return dpaa_bman_release(dpaa_bp
, bmb
, i
);
1396 release_previous_buffs
:
1397 WARN_ONCE(1, "dpaa_eth: failed to add buffers on Rx\n");
1399 bm_buffer_set64(&bmb
[i
], 0);
1400 /* Avoid releasing a completely null buffer; bman_release() requires
1401 * at least one buffer.
1409 static int dpaa_bp_seed(struct dpaa_bp
*dpaa_bp
)
1413 /* Give each CPU an allotment of "config_count" buffers */
1414 for_each_possible_cpu(i
) {
1415 int *count_ptr
= per_cpu_ptr(dpaa_bp
->percpu_count
, i
);
1418 /* Although we access another CPU's counters here
1419 * we do it at boot time so it is safe
1421 for (j
= 0; j
< dpaa_bp
->config_count
; j
+= 8)
1422 *count_ptr
+= dpaa_bp_add_8_bufs(dpaa_bp
);
1427 /* Add buffers/(pages) for Rx processing whenever bpool count falls below
1430 static int dpaa_eth_refill_bpool(struct dpaa_bp
*dpaa_bp
, int *countptr
)
1432 int count
= *countptr
;
1435 if (unlikely(count
< FSL_DPAA_ETH_REFILL_THRESHOLD
)) {
1437 new_bufs
= dpaa_bp_add_8_bufs(dpaa_bp
);
1438 if (unlikely(!new_bufs
)) {
1439 /* Avoid looping forever if we've temporarily
1440 * run out of memory. We'll try again at the
1446 } while (count
< FSL_DPAA_ETH_MAX_BUF_COUNT
);
1449 if (unlikely(count
< FSL_DPAA_ETH_MAX_BUF_COUNT
))
1456 static int dpaa_eth_refill_bpools(struct dpaa_priv
*priv
)
1458 struct dpaa_bp
*dpaa_bp
;
1462 for (i
= 0; i
< DPAA_BPS_NUM
; i
++) {
1463 dpaa_bp
= priv
->dpaa_bps
[i
];
1466 countptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1467 res
= dpaa_eth_refill_bpool(dpaa_bp
, countptr
);
1474 /* Cleanup function for outgoing frame descriptors that were built on Tx path,
1475 * either contiguous frames or scatter/gather ones.
1476 * Skb freeing is not handled here.
1478 * This function may be called on error paths in the Tx function, so guard
1479 * against cases when not all fd relevant fields were filled in.
1481 * Return the skb backpointer, since for S/G frames the buffer containing it
1484 static struct sk_buff
*dpaa_cleanup_tx_fd(const struct dpaa_priv
*priv
,
1485 const struct qm_fd
*fd
)
1487 const enum dma_data_direction dma_dir
= DMA_TO_DEVICE
;
1488 struct device
*dev
= priv
->net_dev
->dev
.parent
;
1489 dma_addr_t addr
= qm_fd_addr(fd
);
1490 const struct qm_sg_entry
*sgt
;
1491 struct sk_buff
**skbh
, *skb
;
1494 skbh
= (struct sk_buff
**)phys_to_virt(addr
);
1497 if (unlikely(qm_fd_get_format(fd
) == qm_fd_sg
)) {
1498 nr_frags
= skb_shinfo(skb
)->nr_frags
;
1499 dma_unmap_single(dev
, addr
, qm_fd_get_offset(fd
) +
1500 sizeof(struct qm_sg_entry
) * (1 + nr_frags
),
1503 /* The sgt buffer has been allocated with netdev_alloc_frag(),
1506 sgt
= phys_to_virt(addr
+ qm_fd_get_offset(fd
));
1508 /* sgt[0] is from lowmem, was dma_map_single()-ed */
1509 dma_unmap_single(dev
, qm_sg_addr(&sgt
[0]),
1510 qm_sg_entry_get_len(&sgt
[0]), dma_dir
);
1512 /* remaining pages were mapped with skb_frag_dma_map() */
1513 for (i
= 1; i
< nr_frags
; i
++) {
1514 WARN_ON(qm_sg_entry_is_ext(&sgt
[i
]));
1516 dma_unmap_page(dev
, qm_sg_addr(&sgt
[i
]),
1517 qm_sg_entry_get_len(&sgt
[i
]), dma_dir
);
1520 /* Free the page frag that we allocated on Tx */
1521 skb_free_frag(phys_to_virt(addr
));
1523 dma_unmap_single(dev
, addr
,
1524 skb_tail_pointer(skb
) - (u8
*)skbh
, dma_dir
);
1530 /* Build a linear skb around the received buffer.
1531 * We are guaranteed there is enough room at the end of the data buffer to
1532 * accommodate the shared info area of the skb.
1534 static struct sk_buff
*contig_fd_to_skb(const struct dpaa_priv
*priv
,
1535 const struct qm_fd
*fd
)
1537 ssize_t fd_off
= qm_fd_get_offset(fd
);
1538 dma_addr_t addr
= qm_fd_addr(fd
);
1539 struct dpaa_bp
*dpaa_bp
;
1540 struct sk_buff
*skb
;
1543 vaddr
= phys_to_virt(addr
);
1544 WARN_ON(!IS_ALIGNED((unsigned long)vaddr
, SMP_CACHE_BYTES
));
1546 dpaa_bp
= dpaa_bpid2pool(fd
->bpid
);
1550 skb
= build_skb(vaddr
, dpaa_bp
->size
+
1551 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
)));
1552 if (unlikely(!skb
)) {
1553 WARN_ONCE(1, "Build skb failure on Rx\n");
1556 WARN_ON(fd_off
!= priv
->rx_headroom
);
1557 skb_reserve(skb
, fd_off
);
1558 skb_put(skb
, qm_fd_get_length(fd
));
1560 skb
->ip_summed
= CHECKSUM_NONE
;
1565 skb_free_frag(vaddr
);
1569 /* Build an skb with the data of the first S/G entry in the linear portion and
1570 * the rest of the frame as skb fragments.
1572 * The page fragment holding the S/G Table is recycled here.
1574 static struct sk_buff
*sg_fd_to_skb(const struct dpaa_priv
*priv
,
1575 const struct qm_fd
*fd
)
1577 ssize_t fd_off
= qm_fd_get_offset(fd
);
1578 dma_addr_t addr
= qm_fd_addr(fd
);
1579 const struct qm_sg_entry
*sgt
;
1580 struct page
*page
, *head_page
;
1581 struct dpaa_bp
*dpaa_bp
;
1582 void *vaddr
, *sg_vaddr
;
1583 int frag_off
, frag_len
;
1584 struct sk_buff
*skb
;
1591 vaddr
= phys_to_virt(addr
);
1592 WARN_ON(!IS_ALIGNED((unsigned long)vaddr
, SMP_CACHE_BYTES
));
1594 /* Iterate through the SGT entries and add data buffers to the skb */
1595 sgt
= vaddr
+ fd_off
;
1596 for (i
= 0; i
< DPAA_SGT_MAX_ENTRIES
; i
++) {
1597 /* Extension bit is not supported */
1598 WARN_ON(qm_sg_entry_is_ext(&sgt
[i
]));
1600 sg_addr
= qm_sg_addr(&sgt
[i
]);
1601 sg_vaddr
= phys_to_virt(sg_addr
);
1602 WARN_ON(!IS_ALIGNED((unsigned long)sg_vaddr
,
1605 /* We may use multiple Rx pools */
1606 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1610 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1611 dma_unmap_single(dpaa_bp
->dev
, sg_addr
, dpaa_bp
->size
,
1614 sz
= dpaa_bp
->size
+
1615 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
1616 skb
= build_skb(sg_vaddr
, sz
);
1617 if (WARN_ON(unlikely(!skb
)))
1620 skb
->ip_summed
= CHECKSUM_NONE
;
1622 /* Make sure forwarded skbs will have enough space
1623 * on Tx, if extra headers are added.
1625 WARN_ON(fd_off
!= priv
->rx_headroom
);
1626 skb_reserve(skb
, fd_off
);
1627 skb_put(skb
, qm_sg_entry_get_len(&sgt
[i
]));
1629 /* Not the first S/G entry; all data from buffer will
1630 * be added in an skb fragment; fragment index is offset
1631 * by one since first S/G entry was incorporated in the
1632 * linear part of the skb.
1634 * Caution: 'page' may be a tail page.
1636 page
= virt_to_page(sg_vaddr
);
1637 head_page
= virt_to_head_page(sg_vaddr
);
1639 /* Compute offset in (possibly tail) page */
1640 page_offset
= ((unsigned long)sg_vaddr
&
1642 (page_address(page
) - page_address(head_page
));
1643 /* page_offset only refers to the beginning of sgt[i];
1644 * but the buffer itself may have an internal offset.
1646 frag_off
= qm_sg_entry_get_off(&sgt
[i
]) + page_offset
;
1647 frag_len
= qm_sg_entry_get_len(&sgt
[i
]);
1648 /* skb_add_rx_frag() does no checking on the page; if
1649 * we pass it a tail page, we'll end up with
1650 * bad page accounting and eventually with segafults.
1652 skb_add_rx_frag(skb
, i
- 1, head_page
, frag_off
,
1653 frag_len
, dpaa_bp
->size
);
1655 /* Update the pool count for the current {cpu x bpool} */
1658 if (qm_sg_entry_is_final(&sgt
[i
]))
1661 WARN_ONCE(i
== DPAA_SGT_MAX_ENTRIES
, "No final bit on SGT\n");
1663 /* free the SG table buffer */
1664 skb_free_frag(vaddr
);
1669 /* compensate sw bpool counter changes */
1670 for (i
--; i
> 0; i
--) {
1671 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1673 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1677 /* free all the SG entries */
1678 for (i
= 0; i
< DPAA_SGT_MAX_ENTRIES
; i
++) {
1679 sg_addr
= qm_sg_addr(&sgt
[i
]);
1680 sg_vaddr
= phys_to_virt(sg_addr
);
1681 skb_free_frag(sg_vaddr
);
1682 dpaa_bp
= dpaa_bpid2pool(sgt
[i
].bpid
);
1684 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
1688 if (qm_sg_entry_is_final(&sgt
[i
]))
1691 /* free the SGT fragment */
1692 skb_free_frag(vaddr
);
1697 static int skb_to_contig_fd(struct dpaa_priv
*priv
,
1698 struct sk_buff
*skb
, struct qm_fd
*fd
,
1701 struct net_device
*net_dev
= priv
->net_dev
;
1702 struct device
*dev
= net_dev
->dev
.parent
;
1703 enum dma_data_direction dma_dir
;
1704 unsigned char *buffer_start
;
1705 struct sk_buff
**skbh
;
1709 /* We are guaranteed to have at least tx_headroom bytes
1710 * available, so just use that for offset.
1712 fd
->bpid
= FSL_DPAA_BPID_INV
;
1713 buffer_start
= skb
->data
- priv
->tx_headroom
;
1714 dma_dir
= DMA_TO_DEVICE
;
1716 skbh
= (struct sk_buff
**)buffer_start
;
1719 /* Enable L3/L4 hardware checksum computation.
1721 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1722 * need to write into the skb.
1724 err
= dpaa_enable_tx_csum(priv
, skb
, fd
,
1725 ((char *)skbh
) + DPAA_TX_PRIV_DATA_SIZE
);
1726 if (unlikely(err
< 0)) {
1727 if (net_ratelimit())
1728 netif_err(priv
, tx_err
, net_dev
, "HW csum error: %d\n",
1733 /* Fill in the rest of the FD fields */
1734 qm_fd_set_contig(fd
, priv
->tx_headroom
, skb
->len
);
1735 fd
->cmd
|= cpu_to_be32(FM_FD_CMD_FCO
);
1737 /* Map the entire buffer size that may be seen by FMan, but no more */
1738 addr
= dma_map_single(dev
, skbh
,
1739 skb_tail_pointer(skb
) - buffer_start
, dma_dir
);
1740 if (unlikely(dma_mapping_error(dev
, addr
))) {
1741 if (net_ratelimit())
1742 netif_err(priv
, tx_err
, net_dev
, "dma_map_single() failed\n");
1745 qm_fd_addr_set64(fd
, addr
);
1750 static int skb_to_sg_fd(struct dpaa_priv
*priv
,
1751 struct sk_buff
*skb
, struct qm_fd
*fd
)
1753 const enum dma_data_direction dma_dir
= DMA_TO_DEVICE
;
1754 const int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1755 struct net_device
*net_dev
= priv
->net_dev
;
1756 struct device
*dev
= net_dev
->dev
.parent
;
1757 struct qm_sg_entry
*sgt
;
1758 struct sk_buff
**skbh
;
1766 /* get a page frag to store the SGTable */
1767 sz
= SKB_DATA_ALIGN(priv
->tx_headroom
+
1768 sizeof(struct qm_sg_entry
) * (1 + nr_frags
));
1769 sgt_buf
= netdev_alloc_frag(sz
);
1770 if (unlikely(!sgt_buf
)) {
1771 netdev_err(net_dev
, "netdev_alloc_frag() failed for size %d\n",
1776 /* Enable L3/L4 hardware checksum computation.
1778 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1779 * need to write into the skb.
1781 err
= dpaa_enable_tx_csum(priv
, skb
, fd
,
1782 sgt_buf
+ DPAA_TX_PRIV_DATA_SIZE
);
1783 if (unlikely(err
< 0)) {
1784 if (net_ratelimit())
1785 netif_err(priv
, tx_err
, net_dev
, "HW csum error: %d\n",
1790 sgt
= (struct qm_sg_entry
*)(sgt_buf
+ priv
->tx_headroom
);
1791 qm_sg_entry_set_len(&sgt
[0], skb_headlen(skb
));
1792 sgt
[0].bpid
= FSL_DPAA_BPID_INV
;
1794 addr
= dma_map_single(dev
, skb
->data
,
1795 skb_headlen(skb
), dma_dir
);
1796 if (unlikely(dma_mapping_error(dev
, addr
))) {
1797 dev_err(dev
, "DMA mapping failed");
1799 goto sg0_map_failed
;
1801 qm_sg_entry_set64(&sgt
[0], addr
);
1803 /* populate the rest of SGT entries */
1804 frag
= &skb_shinfo(skb
)->frags
[0];
1805 frag_len
= frag
->size
;
1806 for (i
= 1; i
<= nr_frags
; i
++, frag
++) {
1807 WARN_ON(!skb_frag_page(frag
));
1808 addr
= skb_frag_dma_map(dev
, frag
, 0,
1810 if (unlikely(dma_mapping_error(dev
, addr
))) {
1811 dev_err(dev
, "DMA mapping failed");
1816 qm_sg_entry_set_len(&sgt
[i
], frag_len
);
1817 sgt
[i
].bpid
= FSL_DPAA_BPID_INV
;
1820 /* keep the offset in the address */
1821 qm_sg_entry_set64(&sgt
[i
], addr
);
1822 frag_len
= frag
->size
;
1824 qm_sg_entry_set_f(&sgt
[i
- 1], frag_len
);
1826 qm_fd_set_sg(fd
, priv
->tx_headroom
, skb
->len
);
1828 /* DMA map the SGT page */
1829 buffer_start
= (void *)sgt
- priv
->tx_headroom
;
1830 skbh
= (struct sk_buff
**)buffer_start
;
1833 addr
= dma_map_single(dev
, buffer_start
, priv
->tx_headroom
+
1834 sizeof(struct qm_sg_entry
) * (1 + nr_frags
),
1836 if (unlikely(dma_mapping_error(dev
, addr
))) {
1837 dev_err(dev
, "DMA mapping failed");
1839 goto sgt_map_failed
;
1842 fd
->bpid
= FSL_DPAA_BPID_INV
;
1843 fd
->cmd
|= cpu_to_be32(FM_FD_CMD_FCO
);
1844 qm_fd_addr_set64(fd
, addr
);
1850 for (j
= 0; j
< i
; j
++)
1851 dma_unmap_page(dev
, qm_sg_addr(&sgt
[j
]),
1852 qm_sg_entry_get_len(&sgt
[j
]), dma_dir
);
1855 skb_free_frag(sgt_buf
);
1860 static inline int dpaa_xmit(struct dpaa_priv
*priv
,
1861 struct rtnl_link_stats64
*percpu_stats
,
1865 struct qman_fq
*egress_fq
;
1868 egress_fq
= priv
->egress_fqs
[queue
];
1869 if (fd
->bpid
== FSL_DPAA_BPID_INV
)
1870 fd
->cmd
|= cpu_to_be32(qman_fq_fqid(priv
->conf_fqs
[queue
]));
1872 /* Trace this Tx fd */
1873 trace_dpaa_tx_fd(priv
->net_dev
, egress_fq
, fd
);
1875 for (i
= 0; i
< DPAA_ENQUEUE_RETRIES
; i
++) {
1876 err
= qman_enqueue(egress_fq
, fd
);
1881 if (unlikely(err
< 0)) {
1882 percpu_stats
->tx_errors
++;
1883 percpu_stats
->tx_fifo_errors
++;
1887 percpu_stats
->tx_packets
++;
1888 percpu_stats
->tx_bytes
+= qm_fd_get_length(fd
);
1893 static int dpaa_start_xmit(struct sk_buff
*skb
, struct net_device
*net_dev
)
1895 const int queue_mapping
= skb_get_queue_mapping(skb
);
1896 bool nonlinear
= skb_is_nonlinear(skb
);
1897 struct rtnl_link_stats64
*percpu_stats
;
1898 struct dpaa_percpu_priv
*percpu_priv
;
1899 struct dpaa_priv
*priv
;
1904 priv
= netdev_priv(net_dev
);
1905 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
1906 percpu_stats
= &percpu_priv
->stats
;
1908 qm_fd_clear_fd(&fd
);
1911 /* We're going to store the skb backpointer at the beginning
1912 * of the data buffer, so we need a privately owned skb
1914 * We've made sure skb is not shared in dev->priv_flags,
1915 * we need to verify the skb head is not cloned
1917 if (skb_cow_head(skb
, priv
->tx_headroom
))
1920 WARN_ON(skb_is_nonlinear(skb
));
1923 /* MAX_SKB_FRAGS is equal or larger than our dpaa_SGT_MAX_ENTRIES;
1924 * make sure we don't feed FMan with more fragments than it supports.
1927 likely(skb_shinfo(skb
)->nr_frags
< DPAA_SGT_MAX_ENTRIES
)) {
1928 /* Just create a S/G fd based on the skb */
1929 err
= skb_to_sg_fd(priv
, skb
, &fd
);
1930 percpu_priv
->tx_frag_skbuffs
++;
1932 /* If the egress skb contains more fragments than we support
1933 * we have no choice but to linearize it ourselves.
1935 if (unlikely(nonlinear
) && __skb_linearize(skb
))
1938 /* Finally, create a contig FD from this skb */
1939 err
= skb_to_contig_fd(priv
, skb
, &fd
, &offset
);
1941 if (unlikely(err
< 0))
1942 goto skb_to_fd_failed
;
1944 if (likely(dpaa_xmit(priv
, percpu_stats
, queue_mapping
, &fd
) == 0))
1945 return NETDEV_TX_OK
;
1947 dpaa_cleanup_tx_fd(priv
, &fd
);
1950 percpu_stats
->tx_errors
++;
1952 return NETDEV_TX_OK
;
1955 static void dpaa_rx_error(struct net_device
*net_dev
,
1956 const struct dpaa_priv
*priv
,
1957 struct dpaa_percpu_priv
*percpu_priv
,
1958 const struct qm_fd
*fd
,
1961 if (net_ratelimit())
1962 netif_err(priv
, hw
, net_dev
, "Err FD status = 0x%08x\n",
1963 be32_to_cpu(fd
->status
) & FM_FD_STAT_RX_ERRORS
);
1965 percpu_priv
->stats
.rx_errors
++;
1967 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_DMA
)
1968 percpu_priv
->rx_errors
.dme
++;
1969 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_PHYSICAL
)
1970 percpu_priv
->rx_errors
.fpe
++;
1971 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_SIZE
)
1972 percpu_priv
->rx_errors
.fse
++;
1973 if (be32_to_cpu(fd
->status
) & FM_FD_ERR_PRS_HDR_ERR
)
1974 percpu_priv
->rx_errors
.phe
++;
1976 dpaa_fd_release(net_dev
, fd
);
1979 static void dpaa_tx_error(struct net_device
*net_dev
,
1980 const struct dpaa_priv
*priv
,
1981 struct dpaa_percpu_priv
*percpu_priv
,
1982 const struct qm_fd
*fd
,
1985 struct sk_buff
*skb
;
1987 if (net_ratelimit())
1988 netif_warn(priv
, hw
, net_dev
, "FD status = 0x%08x\n",
1989 be32_to_cpu(fd
->status
) & FM_FD_STAT_TX_ERRORS
);
1991 percpu_priv
->stats
.tx_errors
++;
1993 skb
= dpaa_cleanup_tx_fd(priv
, fd
);
1997 static int dpaa_eth_poll(struct napi_struct
*napi
, int budget
)
1999 struct dpaa_napi_portal
*np
=
2000 container_of(napi
, struct dpaa_napi_portal
, napi
);
2002 int cleaned
= qman_p_poll_dqrr(np
->p
, budget
);
2004 if (cleaned
< budget
) {
2005 napi_complete(napi
);
2006 qman_p_irqsource_add(np
->p
, QM_PIRQ_DQRI
);
2008 } else if (np
->down
) {
2009 qman_p_irqsource_add(np
->p
, QM_PIRQ_DQRI
);
2015 static void dpaa_tx_conf(struct net_device
*net_dev
,
2016 const struct dpaa_priv
*priv
,
2017 struct dpaa_percpu_priv
*percpu_priv
,
2018 const struct qm_fd
*fd
,
2021 struct sk_buff
*skb
;
2023 if (unlikely(be32_to_cpu(fd
->status
) & FM_FD_STAT_TX_ERRORS
)) {
2024 if (net_ratelimit())
2025 netif_warn(priv
, hw
, net_dev
, "FD status = 0x%08x\n",
2026 be32_to_cpu(fd
->status
) &
2027 FM_FD_STAT_TX_ERRORS
);
2029 percpu_priv
->stats
.tx_errors
++;
2032 percpu_priv
->tx_confirm
++;
2034 skb
= dpaa_cleanup_tx_fd(priv
, fd
);
2039 static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv
*percpu_priv
,
2040 struct qman_portal
*portal
)
2042 if (unlikely(in_irq() || !in_serving_softirq())) {
2043 /* Disable QMan IRQ and invoke NAPI */
2044 qman_p_irqsource_remove(portal
, QM_PIRQ_DQRI
);
2046 percpu_priv
->np
.p
= portal
;
2047 napi_schedule(&percpu_priv
->np
.napi
);
2048 percpu_priv
->in_interrupt
++;
2054 static enum qman_cb_dqrr_result
rx_error_dqrr(struct qman_portal
*portal
,
2056 const struct qm_dqrr_entry
*dq
)
2058 struct dpaa_fq
*dpaa_fq
= container_of(fq
, struct dpaa_fq
, fq_base
);
2059 struct dpaa_percpu_priv
*percpu_priv
;
2060 struct net_device
*net_dev
;
2061 struct dpaa_bp
*dpaa_bp
;
2062 struct dpaa_priv
*priv
;
2064 net_dev
= dpaa_fq
->net_dev
;
2065 priv
= netdev_priv(net_dev
);
2066 dpaa_bp
= dpaa_bpid2pool(dq
->fd
.bpid
);
2068 return qman_cb_dqrr_consume
;
2070 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2072 if (dpaa_eth_napi_schedule(percpu_priv
, portal
))
2073 return qman_cb_dqrr_stop
;
2075 if (dpaa_eth_refill_bpools(priv
))
2076 /* Unable to refill the buffer pool due to insufficient
2077 * system memory. Just release the frame back into the pool,
2078 * otherwise we'll soon end up with an empty buffer pool.
2080 dpaa_fd_release(net_dev
, &dq
->fd
);
2082 dpaa_rx_error(net_dev
, priv
, percpu_priv
, &dq
->fd
, fq
->fqid
);
2084 return qman_cb_dqrr_consume
;
2087 static enum qman_cb_dqrr_result
rx_default_dqrr(struct qman_portal
*portal
,
2089 const struct qm_dqrr_entry
*dq
)
2091 struct rtnl_link_stats64
*percpu_stats
;
2092 struct dpaa_percpu_priv
*percpu_priv
;
2093 const struct qm_fd
*fd
= &dq
->fd
;
2094 dma_addr_t addr
= qm_fd_addr(fd
);
2095 enum qm_fd_format fd_format
;
2096 struct net_device
*net_dev
;
2097 u32 fd_status
= fd
->status
;
2098 struct dpaa_bp
*dpaa_bp
;
2099 struct dpaa_priv
*priv
;
2100 unsigned int skb_len
;
2101 struct sk_buff
*skb
;
2104 fd_status
= be32_to_cpu(fd
->status
);
2105 fd_format
= qm_fd_get_format(fd
);
2106 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2107 priv
= netdev_priv(net_dev
);
2108 dpaa_bp
= dpaa_bpid2pool(dq
->fd
.bpid
);
2110 return qman_cb_dqrr_consume
;
2112 /* Trace the Rx fd */
2113 trace_dpaa_rx_fd(net_dev
, fq
, &dq
->fd
);
2115 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2116 percpu_stats
= &percpu_priv
->stats
;
2118 if (unlikely(dpaa_eth_napi_schedule(percpu_priv
, portal
)))
2119 return qman_cb_dqrr_stop
;
2121 /* Make sure we didn't run out of buffers */
2122 if (unlikely(dpaa_eth_refill_bpools(priv
))) {
2123 /* Unable to refill the buffer pool due to insufficient
2124 * system memory. Just release the frame back into the pool,
2125 * otherwise we'll soon end up with an empty buffer pool.
2127 dpaa_fd_release(net_dev
, &dq
->fd
);
2128 return qman_cb_dqrr_consume
;
2131 if (unlikely(fd_status
& FM_FD_STAT_RX_ERRORS
) != 0) {
2132 if (net_ratelimit())
2133 netif_warn(priv
, hw
, net_dev
, "FD status = 0x%08x\n",
2134 fd_status
& FM_FD_STAT_RX_ERRORS
);
2136 percpu_stats
->rx_errors
++;
2137 dpaa_fd_release(net_dev
, fd
);
2138 return qman_cb_dqrr_consume
;
2141 dpaa_bp
= dpaa_bpid2pool(fd
->bpid
);
2143 return qman_cb_dqrr_consume
;
2145 dma_unmap_single(dpaa_bp
->dev
, addr
, dpaa_bp
->size
, DMA_FROM_DEVICE
);
2147 /* prefetch the first 64 bytes of the frame or the SGT start */
2148 prefetch(phys_to_virt(addr
) + qm_fd_get_offset(fd
));
2150 fd_format
= qm_fd_get_format(fd
);
2151 /* The only FD types that we may receive are contig and S/G */
2152 WARN_ON((fd_format
!= qm_fd_contig
) && (fd_format
!= qm_fd_sg
));
2154 /* Account for either the contig buffer or the SGT buffer (depending on
2155 * which case we were in) having been removed from the pool.
2157 count_ptr
= this_cpu_ptr(dpaa_bp
->percpu_count
);
2160 if (likely(fd_format
== qm_fd_contig
))
2161 skb
= contig_fd_to_skb(priv
, fd
);
2163 skb
= sg_fd_to_skb(priv
, fd
);
2165 return qman_cb_dqrr_consume
;
2167 skb
->protocol
= eth_type_trans(skb
, net_dev
);
2171 if (unlikely(netif_receive_skb(skb
) == NET_RX_DROP
))
2172 return qman_cb_dqrr_consume
;
2174 percpu_stats
->rx_packets
++;
2175 percpu_stats
->rx_bytes
+= skb_len
;
2177 return qman_cb_dqrr_consume
;
2180 static enum qman_cb_dqrr_result
conf_error_dqrr(struct qman_portal
*portal
,
2182 const struct qm_dqrr_entry
*dq
)
2184 struct dpaa_percpu_priv
*percpu_priv
;
2185 struct net_device
*net_dev
;
2186 struct dpaa_priv
*priv
;
2188 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2189 priv
= netdev_priv(net_dev
);
2191 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2193 if (dpaa_eth_napi_schedule(percpu_priv
, portal
))
2194 return qman_cb_dqrr_stop
;
2196 dpaa_tx_error(net_dev
, priv
, percpu_priv
, &dq
->fd
, fq
->fqid
);
2198 return qman_cb_dqrr_consume
;
2201 static enum qman_cb_dqrr_result
conf_dflt_dqrr(struct qman_portal
*portal
,
2203 const struct qm_dqrr_entry
*dq
)
2205 struct dpaa_percpu_priv
*percpu_priv
;
2206 struct net_device
*net_dev
;
2207 struct dpaa_priv
*priv
;
2209 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2210 priv
= netdev_priv(net_dev
);
2213 trace_dpaa_tx_conf_fd(net_dev
, fq
, &dq
->fd
);
2215 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2217 if (dpaa_eth_napi_schedule(percpu_priv
, portal
))
2218 return qman_cb_dqrr_stop
;
2220 dpaa_tx_conf(net_dev
, priv
, percpu_priv
, &dq
->fd
, fq
->fqid
);
2222 return qman_cb_dqrr_consume
;
2225 static void egress_ern(struct qman_portal
*portal
,
2227 const union qm_mr_entry
*msg
)
2229 const struct qm_fd
*fd
= &msg
->ern
.fd
;
2230 struct dpaa_percpu_priv
*percpu_priv
;
2231 const struct dpaa_priv
*priv
;
2232 struct net_device
*net_dev
;
2233 struct sk_buff
*skb
;
2235 net_dev
= ((struct dpaa_fq
*)fq
)->net_dev
;
2236 priv
= netdev_priv(net_dev
);
2237 percpu_priv
= this_cpu_ptr(priv
->percpu_priv
);
2239 percpu_priv
->stats
.tx_dropped
++;
2240 percpu_priv
->stats
.tx_fifo_errors
++;
2241 count_ern(percpu_priv
, msg
);
2243 skb
= dpaa_cleanup_tx_fd(priv
, fd
);
2244 dev_kfree_skb_any(skb
);
2247 static const struct dpaa_fq_cbs dpaa_fq_cbs
= {
2248 .rx_defq
= { .cb
= { .dqrr
= rx_default_dqrr
} },
2249 .tx_defq
= { .cb
= { .dqrr
= conf_dflt_dqrr
} },
2250 .rx_errq
= { .cb
= { .dqrr
= rx_error_dqrr
} },
2251 .tx_errq
= { .cb
= { .dqrr
= conf_error_dqrr
} },
2252 .egress_ern
= { .cb
= { .ern
= egress_ern
} }
2255 static void dpaa_eth_napi_enable(struct dpaa_priv
*priv
)
2257 struct dpaa_percpu_priv
*percpu_priv
;
2260 for_each_possible_cpu(i
) {
2261 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
2263 percpu_priv
->np
.down
= 0;
2264 napi_enable(&percpu_priv
->np
.napi
);
2268 static void dpaa_eth_napi_disable(struct dpaa_priv
*priv
)
2270 struct dpaa_percpu_priv
*percpu_priv
;
2273 for_each_possible_cpu(i
) {
2274 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
2276 percpu_priv
->np
.down
= 1;
2277 napi_disable(&percpu_priv
->np
.napi
);
2281 static int dpaa_open(struct net_device
*net_dev
)
2283 struct mac_device
*mac_dev
;
2284 struct dpaa_priv
*priv
;
2287 priv
= netdev_priv(net_dev
);
2288 mac_dev
= priv
->mac_dev
;
2289 dpaa_eth_napi_enable(priv
);
2291 net_dev
->phydev
= mac_dev
->init_phy(net_dev
, priv
->mac_dev
);
2292 if (!net_dev
->phydev
) {
2293 netif_err(priv
, ifup
, net_dev
, "init_phy() failed\n");
2297 for (i
= 0; i
< ARRAY_SIZE(mac_dev
->port
); i
++) {
2298 err
= fman_port_enable(mac_dev
->port
[i
]);
2300 goto mac_start_failed
;
2303 err
= priv
->mac_dev
->start(mac_dev
);
2305 netif_err(priv
, ifup
, net_dev
, "mac_dev->start() = %d\n", err
);
2306 goto mac_start_failed
;
2309 netif_tx_start_all_queues(net_dev
);
2314 for (i
= 0; i
< ARRAY_SIZE(mac_dev
->port
); i
++)
2315 fman_port_disable(mac_dev
->port
[i
]);
2317 dpaa_eth_napi_disable(priv
);
2322 static int dpaa_eth_stop(struct net_device
*net_dev
)
2324 struct dpaa_priv
*priv
;
2327 err
= dpaa_stop(net_dev
);
2329 priv
= netdev_priv(net_dev
);
2330 dpaa_eth_napi_disable(priv
);
2335 static const struct net_device_ops dpaa_ops
= {
2336 .ndo_open
= dpaa_open
,
2337 .ndo_start_xmit
= dpaa_start_xmit
,
2338 .ndo_stop
= dpaa_eth_stop
,
2339 .ndo_tx_timeout
= dpaa_tx_timeout
,
2340 .ndo_get_stats64
= dpaa_get_stats64
,
2341 .ndo_set_mac_address
= dpaa_set_mac_address
,
2342 .ndo_validate_addr
= eth_validate_addr
,
2343 .ndo_set_rx_mode
= dpaa_set_rx_mode
,
2346 static int dpaa_napi_add(struct net_device
*net_dev
)
2348 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
2349 struct dpaa_percpu_priv
*percpu_priv
;
2352 for_each_possible_cpu(cpu
) {
2353 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, cpu
);
2355 netif_napi_add(net_dev
, &percpu_priv
->np
.napi
,
2356 dpaa_eth_poll
, NAPI_POLL_WEIGHT
);
2362 static void dpaa_napi_del(struct net_device
*net_dev
)
2364 struct dpaa_priv
*priv
= netdev_priv(net_dev
);
2365 struct dpaa_percpu_priv
*percpu_priv
;
2368 for_each_possible_cpu(cpu
) {
2369 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, cpu
);
2371 netif_napi_del(&percpu_priv
->np
.napi
);
2375 static inline void dpaa_bp_free_pf(const struct dpaa_bp
*bp
,
2376 struct bm_buffer
*bmb
)
2378 dma_addr_t addr
= bm_buf_addr(bmb
);
2380 dma_unmap_single(bp
->dev
, addr
, bp
->size
, DMA_FROM_DEVICE
);
2382 skb_free_frag(phys_to_virt(addr
));
2385 /* Alloc the dpaa_bp struct and configure default values */
2386 static struct dpaa_bp
*dpaa_bp_alloc(struct device
*dev
)
2388 struct dpaa_bp
*dpaa_bp
;
2390 dpaa_bp
= devm_kzalloc(dev
, sizeof(*dpaa_bp
), GFP_KERNEL
);
2392 return ERR_PTR(-ENOMEM
);
2394 dpaa_bp
->bpid
= FSL_DPAA_BPID_INV
;
2395 dpaa_bp
->percpu_count
= devm_alloc_percpu(dev
, *dpaa_bp
->percpu_count
);
2396 dpaa_bp
->config_count
= FSL_DPAA_ETH_MAX_BUF_COUNT
;
2398 dpaa_bp
->seed_cb
= dpaa_bp_seed
;
2399 dpaa_bp
->free_buf_cb
= dpaa_bp_free_pf
;
2404 /* Place all ingress FQs (Rx Default, Rx Error) in a dedicated CGR.
2405 * We won't be sending congestion notifications to FMan; for now, we just use
2406 * this CGR to generate enqueue rejections to FMan in order to drop the frames
2407 * before they reach our ingress queues and eat up memory.
2409 static int dpaa_ingress_cgr_init(struct dpaa_priv
*priv
)
2411 struct qm_mcc_initcgr initcgr
;
2415 err
= qman_alloc_cgrid(&priv
->ingress_cgr
.cgrid
);
2417 if (netif_msg_drv(priv
))
2418 pr_err("Error %d allocating CGR ID\n", err
);
2422 /* Enable CS TD, but disable Congestion State Change Notifications. */
2423 initcgr
.we_mask
= cpu_to_be16(QM_CGR_WE_CS_THRES
);
2424 initcgr
.cgr
.cscn_en
= QM_CGR_EN
;
2425 cs_th
= DPAA_INGRESS_CS_THRESHOLD
;
2426 qm_cgr_cs_thres_set64(&initcgr
.cgr
.cs_thres
, cs_th
, 1);
2428 initcgr
.we_mask
|= cpu_to_be16(QM_CGR_WE_CSTD_EN
);
2429 initcgr
.cgr
.cstd_en
= QM_CGR_EN
;
2431 /* This CGR will be associated with the SWP affined to the current CPU.
2432 * However, we'll place all our ingress FQs in it.
2434 err
= qman_create_cgr(&priv
->ingress_cgr
, QMAN_CGR_FLAG_USE_INIT
,
2437 if (netif_msg_drv(priv
))
2438 pr_err("Error %d creating ingress CGR with ID %d\n",
2439 err
, priv
->ingress_cgr
.cgrid
);
2440 qman_release_cgrid(priv
->ingress_cgr
.cgrid
);
2443 if (netif_msg_drv(priv
))
2444 pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n",
2445 priv
->ingress_cgr
.cgrid
, priv
->mac_dev
->addr
);
2447 priv
->use_ingress_cgr
= true;
2453 static const struct of_device_id dpaa_match
[];
2455 static inline u16
dpaa_get_headroom(struct dpaa_buffer_layout
*bl
)
2459 /* The frame headroom must accommodate:
2460 * - the driver private data area
2461 * - parse results, hash results, timestamp if selected
2462 * If either hash results or time stamp are selected, both will
2463 * be copied to/from the frame headroom, as TS is located between PR and
2464 * HR in the IC and IC copy size has a granularity of 16bytes
2465 * (see description of FMBM_RICP and FMBM_TICP registers in DPAARM)
2467 * Also make sure the headroom is a multiple of data_align bytes
2469 headroom
= (u16
)(bl
->priv_data_size
+ DPAA_PARSE_RESULTS_SIZE
+
2470 DPAA_TIME_STAMP_SIZE
+ DPAA_HASH_RESULTS_SIZE
);
2472 return DPAA_FD_DATA_ALIGNMENT
? ALIGN(headroom
,
2473 DPAA_FD_DATA_ALIGNMENT
) :
2477 static int dpaa_eth_probe(struct platform_device
*pdev
)
2479 struct dpaa_bp
*dpaa_bps
[DPAA_BPS_NUM
] = {NULL
};
2480 struct dpaa_percpu_priv
*percpu_priv
;
2481 struct net_device
*net_dev
= NULL
;
2482 struct dpaa_fq
*dpaa_fq
, *tmp
;
2483 struct dpaa_priv
*priv
= NULL
;
2484 struct fm_port_fqs port_fqs
;
2485 struct mac_device
*mac_dev
;
2486 int err
= 0, i
, channel
;
2491 /* Allocate this early, so we can store relevant information in
2494 net_dev
= alloc_etherdev_mq(sizeof(*priv
), DPAA_ETH_TXQ_NUM
);
2496 dev_err(dev
, "alloc_etherdev_mq() failed\n");
2497 goto alloc_etherdev_mq_failed
;
2500 /* Do this here, so we can be verbose early */
2501 SET_NETDEV_DEV(net_dev
, dev
);
2502 dev_set_drvdata(dev
, net_dev
);
2504 priv
= netdev_priv(net_dev
);
2505 priv
->net_dev
= net_dev
;
2507 priv
->msg_enable
= netif_msg_init(debug
, DPAA_MSG_DEFAULT
);
2509 mac_dev
= dpaa_mac_dev_get(pdev
);
2510 if (IS_ERR(mac_dev
)) {
2511 dev_err(dev
, "dpaa_mac_dev_get() failed\n");
2512 err
= PTR_ERR(mac_dev
);
2513 goto mac_probe_failed
;
2516 /* If fsl_fm_max_frm is set to a higher value than the all-common 1500,
2517 * we choose conservatively and let the user explicitly set a higher
2518 * MTU via ifconfig. Otherwise, the user may end up with different MTUs
2520 * If on the other hand fsl_fm_max_frm has been chosen below 1500,
2521 * start with the maximum allowed.
2523 net_dev
->mtu
= min(dpaa_get_max_mtu(), ETH_DATA_LEN
);
2525 netdev_dbg(net_dev
, "Setting initial MTU on net device: %d\n",
2528 priv
->buf_layout
[RX
].priv_data_size
= DPAA_RX_PRIV_DATA_SIZE
; /* Rx */
2529 priv
->buf_layout
[TX
].priv_data_size
= DPAA_TX_PRIV_DATA_SIZE
; /* Tx */
2531 /* device used for DMA mapping */
2532 arch_setup_dma_ops(dev
, 0, 0, NULL
, false);
2533 err
= dma_coerce_mask_and_coherent(dev
, DMA_BIT_MASK(40));
2535 dev_err(dev
, "dma_coerce_mask_and_coherent() failed\n");
2536 goto dev_mask_failed
;
2540 for (i
= 0; i
< DPAA_BPS_NUM
; i
++) {
2543 dpaa_bps
[i
] = dpaa_bp_alloc(dev
);
2544 if (IS_ERR(dpaa_bps
[i
]))
2545 return PTR_ERR(dpaa_bps
[i
]);
2546 /* the raw size of the buffers used for reception */
2547 dpaa_bps
[i
]->raw_size
= bpool_buffer_raw_size(i
, DPAA_BPS_NUM
);
2548 /* avoid runtime computations by keeping the usable size here */
2549 dpaa_bps
[i
]->size
= dpaa_bp_size(dpaa_bps
[i
]->raw_size
);
2550 dpaa_bps
[i
]->dev
= dev
;
2552 err
= dpaa_bp_alloc_pool(dpaa_bps
[i
]);
2554 dpaa_bps_free(priv
);
2555 priv
->dpaa_bps
[i
] = NULL
;
2556 goto bp_create_failed
;
2558 priv
->dpaa_bps
[i
] = dpaa_bps
[i
];
2561 INIT_LIST_HEAD(&priv
->dpaa_fq_list
);
2563 memset(&port_fqs
, 0, sizeof(port_fqs
));
2565 err
= dpaa_alloc_all_fqs(dev
, &priv
->dpaa_fq_list
, &port_fqs
);
2567 dev_err(dev
, "dpaa_alloc_all_fqs() failed\n");
2568 goto fq_probe_failed
;
2571 priv
->mac_dev
= mac_dev
;
2573 channel
= dpaa_get_channel();
2575 dev_err(dev
, "dpaa_get_channel() failed\n");
2577 goto get_channel_failed
;
2580 priv
->channel
= (u16
)channel
;
2582 /* Start a thread that will walk the CPUs with affine portals
2583 * and add this pool channel to each's dequeue mask.
2585 dpaa_eth_add_channel(priv
->channel
);
2587 dpaa_fq_setup(priv
, &dpaa_fq_cbs
, priv
->mac_dev
->port
[TX
]);
2589 /* Create a congestion group for this netdev, with
2590 * dynamically-allocated CGR ID.
2591 * Must be executed after probing the MAC, but before
2592 * assigning the egress FQs to the CGRs.
2594 err
= dpaa_eth_cgr_init(priv
);
2596 dev_err(dev
, "Error initializing CGR\n");
2597 goto tx_cgr_init_failed
;
2600 err
= dpaa_ingress_cgr_init(priv
);
2602 dev_err(dev
, "Error initializing ingress CGR\n");
2603 goto rx_cgr_init_failed
;
2606 /* Add the FQs to the interface, and make them active */
2607 list_for_each_entry_safe(dpaa_fq
, tmp
, &priv
->dpaa_fq_list
, list
) {
2608 err
= dpaa_fq_init(dpaa_fq
, false);
2610 goto fq_alloc_failed
;
2613 priv
->tx_headroom
= dpaa_get_headroom(&priv
->buf_layout
[TX
]);
2614 priv
->rx_headroom
= dpaa_get_headroom(&priv
->buf_layout
[RX
]);
2616 /* All real interfaces need their ports initialized */
2617 dpaa_eth_init_ports(mac_dev
, dpaa_bps
, DPAA_BPS_NUM
, &port_fqs
,
2618 &priv
->buf_layout
[0], dev
);
2620 priv
->percpu_priv
= devm_alloc_percpu(dev
, *priv
->percpu_priv
);
2621 if (!priv
->percpu_priv
) {
2622 dev_err(dev
, "devm_alloc_percpu() failed\n");
2624 goto alloc_percpu_failed
;
2626 for_each_possible_cpu(i
) {
2627 percpu_priv
= per_cpu_ptr(priv
->percpu_priv
, i
);
2628 memset(percpu_priv
, 0, sizeof(*percpu_priv
));
2631 /* Initialize NAPI */
2632 err
= dpaa_napi_add(net_dev
);
2634 goto napi_add_failed
;
2636 err
= dpaa_netdev_init(net_dev
, &dpaa_ops
, tx_timeout
);
2638 goto netdev_init_failed
;
2640 dpaa_eth_sysfs_init(&net_dev
->dev
);
2642 netif_info(priv
, probe
, net_dev
, "Probed interface %s\n",
2649 dpaa_napi_del(net_dev
);
2650 alloc_percpu_failed
:
2651 dpaa_fq_free(dev
, &priv
->dpaa_fq_list
);
2653 qman_delete_cgr_safe(&priv
->ingress_cgr
);
2654 qman_release_cgrid(priv
->ingress_cgr
.cgrid
);
2656 qman_delete_cgr_safe(&priv
->cgr_data
.cgr
);
2657 qman_release_cgrid(priv
->cgr_data
.cgr
.cgrid
);
2660 dpaa_bps_free(priv
);
2665 dev_set_drvdata(dev
, NULL
);
2666 free_netdev(net_dev
);
2667 alloc_etherdev_mq_failed
:
2668 for (i
= 0; i
< DPAA_BPS_NUM
&& dpaa_bps
[i
]; i
++) {
2669 if (atomic_read(&dpaa_bps
[i
]->refs
) == 0)
2670 devm_kfree(dev
, dpaa_bps
[i
]);
2675 static int dpaa_remove(struct platform_device
*pdev
)
2677 struct net_device
*net_dev
;
2678 struct dpaa_priv
*priv
;
2683 net_dev
= dev_get_drvdata(dev
);
2685 priv
= netdev_priv(net_dev
);
2687 dpaa_eth_sysfs_remove(dev
);
2689 dev_set_drvdata(dev
, NULL
);
2690 unregister_netdev(net_dev
);
2692 err
= dpaa_fq_free(dev
, &priv
->dpaa_fq_list
);
2694 qman_delete_cgr_safe(&priv
->ingress_cgr
);
2695 qman_release_cgrid(priv
->ingress_cgr
.cgrid
);
2696 qman_delete_cgr_safe(&priv
->cgr_data
.cgr
);
2697 qman_release_cgrid(priv
->cgr_data
.cgr
.cgrid
);
2699 dpaa_napi_del(net_dev
);
2701 dpaa_bps_free(priv
);
2703 free_netdev(net_dev
);
2708 static struct platform_device_id dpaa_devtype
[] = {
2710 .name
= "dpaa-ethernet",
2715 MODULE_DEVICE_TABLE(platform
, dpaa_devtype
);
2717 static struct platform_driver dpaa_driver
= {
2719 .name
= KBUILD_MODNAME
,
2721 .id_table
= dpaa_devtype
,
2722 .probe
= dpaa_eth_probe
,
2723 .remove
= dpaa_remove
2726 static int __init
dpaa_load(void)
2730 pr_debug("FSL DPAA Ethernet driver\n");
2732 /* initialize dpaa_eth mirror values */
2733 dpaa_rx_extra_headroom
= fman_get_rx_extra_headroom();
2734 dpaa_max_frm
= fman_get_max_frm();
2736 err
= platform_driver_register(&dpaa_driver
);
2738 pr_err("Error, platform_driver_register() = %d\n", err
);
2742 module_init(dpaa_load
);
2744 static void __exit
dpaa_unload(void)
2746 platform_driver_unregister(&dpaa_driver
);
2748 /* Only one channel is used and needs to be released after all
2749 * interfaces are removed
2751 dpaa_release_channel();
2753 module_exit(dpaa_unload
);
2755 MODULE_LICENSE("Dual BSD/GPL");
2756 MODULE_DESCRIPTION("FSL DPAA Ethernet driver");