]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * | |
3 | * Copyright 2016 Freescale Semiconductor, Inc. All rights reserved. | |
9f95a23c | 4 | * Copyright 2017,2019 NXP |
11fdf7f2 TL |
5 | * |
6 | */ | |
7 | ||
8 | /* System headers */ | |
9 | #include <inttypes.h> | |
10 | #include <unistd.h> | |
11 | #include <stdio.h> | |
12 | #include <limits.h> | |
13 | #include <sched.h> | |
14 | #include <pthread.h> | |
15 | ||
16 | #include <rte_byteorder.h> | |
17 | #include <rte_common.h> | |
18 | #include <rte_interrupts.h> | |
19 | #include <rte_log.h> | |
20 | #include <rte_debug.h> | |
21 | #include <rte_pci.h> | |
22 | #include <rte_atomic.h> | |
23 | #include <rte_branch_prediction.h> | |
24 | #include <rte_memory.h> | |
25 | #include <rte_tailq.h> | |
26 | #include <rte_eal.h> | |
27 | #include <rte_alarm.h> | |
28 | #include <rte_ether.h> | |
29 | #include <rte_ethdev_driver.h> | |
30 | #include <rte_malloc.h> | |
31 | #include <rte_ring.h> | |
32 | #include <rte_ip.h> | |
33 | #include <rte_tcp.h> | |
34 | #include <rte_udp.h> | |
35 | #include <rte_net.h> | |
36 | #include <rte_eventdev.h> | |
37 | ||
38 | #include "dpaa_ethdev.h" | |
39 | #include "dpaa_rxtx.h" | |
40 | #include <rte_dpaa_bus.h> | |
41 | #include <dpaa_mempool.h> | |
42 | ||
43 | #include <qman.h> | |
44 | #include <fsl_usd.h> | |
45 | #include <fsl_qman.h> | |
46 | #include <fsl_bman.h> | |
f67539c2 | 47 | #include <dpaa_of.h> |
11fdf7f2 TL |
48 | #include <netcfg.h> |
49 | ||
50 | #define DPAA_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid) \ | |
51 | do { \ | |
52 | (_fd)->cmd = 0; \ | |
53 | (_fd)->opaque_addr = 0; \ | |
54 | (_fd)->opaque = QM_FD_CONTIG << DPAA_FD_FORMAT_SHIFT; \ | |
55 | (_fd)->opaque |= ((_mbuf)->data_off) << DPAA_FD_OFFSET_SHIFT; \ | |
56 | (_fd)->opaque |= (_mbuf)->pkt_len; \ | |
57 | (_fd)->addr = (_mbuf)->buf_iova; \ | |
58 | (_fd)->bpid = _bpid; \ | |
59 | } while (0) | |
60 | ||
61 | #if (defined RTE_LIBRTE_DPAA_DEBUG_DRIVER) | |
62 | static void dpaa_display_frame(const struct qm_fd *fd) | |
63 | { | |
64 | int ii; | |
65 | char *ptr; | |
66 | ||
67 | printf("%s::bpid %x addr %08x%08x, format %d off %d, len %d stat %x\n", | |
68 | __func__, fd->bpid, fd->addr_hi, fd->addr_lo, fd->format, | |
69 | fd->offset, fd->length20, fd->status); | |
70 | ||
71 | ptr = (char *)rte_dpaa_mem_ptov(fd->addr); | |
72 | ptr += fd->offset; | |
73 | printf("%02x ", *ptr); | |
74 | for (ii = 1; ii < fd->length20; ii++) { | |
75 | printf("%02x ", *ptr); | |
76 | if ((ii % 16) == 0) | |
77 | printf("\n"); | |
78 | ptr++; | |
79 | } | |
80 | printf("\n"); | |
81 | } | |
82 | #else | |
83 | #define dpaa_display_frame(a) | |
84 | #endif | |
85 | ||
86 | static inline void dpaa_slow_parsing(struct rte_mbuf *m __rte_unused, | |
87 | uint64_t prs __rte_unused) | |
88 | { | |
89 | DPAA_DP_LOG(DEBUG, "Slow parsing"); | |
90 | /*TBD:XXX: to be implemented*/ | |
91 | } | |
92 | ||
93 | static inline void dpaa_eth_packet_info(struct rte_mbuf *m, void *fd_virt_addr) | |
94 | { | |
95 | struct annotations_t *annot = GET_ANNOTATIONS(fd_virt_addr); | |
96 | uint64_t prs = *((uintptr_t *)(&annot->parse)) & DPAA_PARSE_MASK; | |
97 | ||
98 | DPAA_DP_LOG(DEBUG, " Parsing mbuf: %p with annotations: %p", m, annot); | |
99 | ||
100 | switch (prs) { | |
101 | case DPAA_PKT_TYPE_IPV4: | |
102 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
103 | RTE_PTYPE_L3_IPV4; | |
104 | break; | |
105 | case DPAA_PKT_TYPE_IPV6: | |
106 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
107 | RTE_PTYPE_L3_IPV6; | |
108 | break; | |
109 | case DPAA_PKT_TYPE_ETHER: | |
110 | m->packet_type = RTE_PTYPE_L2_ETHER; | |
111 | break; | |
112 | case DPAA_PKT_TYPE_IPV4_FRAG: | |
113 | case DPAA_PKT_TYPE_IPV4_FRAG_UDP: | |
114 | case DPAA_PKT_TYPE_IPV4_FRAG_TCP: | |
115 | case DPAA_PKT_TYPE_IPV4_FRAG_SCTP: | |
116 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
117 | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG; | |
118 | break; | |
119 | case DPAA_PKT_TYPE_IPV6_FRAG: | |
120 | case DPAA_PKT_TYPE_IPV6_FRAG_UDP: | |
121 | case DPAA_PKT_TYPE_IPV6_FRAG_TCP: | |
122 | case DPAA_PKT_TYPE_IPV6_FRAG_SCTP: | |
123 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
124 | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG; | |
125 | break; | |
126 | case DPAA_PKT_TYPE_IPV4_EXT: | |
127 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
128 | RTE_PTYPE_L3_IPV4_EXT; | |
129 | break; | |
130 | case DPAA_PKT_TYPE_IPV6_EXT: | |
131 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
132 | RTE_PTYPE_L3_IPV6_EXT; | |
133 | break; | |
134 | case DPAA_PKT_TYPE_IPV4_TCP: | |
135 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
136 | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP; | |
137 | break; | |
138 | case DPAA_PKT_TYPE_IPV6_TCP: | |
139 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
140 | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP; | |
141 | break; | |
142 | case DPAA_PKT_TYPE_IPV4_UDP: | |
143 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
144 | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP; | |
145 | break; | |
146 | case DPAA_PKT_TYPE_IPV6_UDP: | |
147 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
148 | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP; | |
149 | break; | |
150 | case DPAA_PKT_TYPE_IPV4_EXT_UDP: | |
151 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
152 | RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP; | |
153 | break; | |
154 | case DPAA_PKT_TYPE_IPV6_EXT_UDP: | |
155 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
156 | RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP; | |
157 | break; | |
158 | case DPAA_PKT_TYPE_IPV4_EXT_TCP: | |
159 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
160 | RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP; | |
161 | break; | |
162 | case DPAA_PKT_TYPE_IPV6_EXT_TCP: | |
163 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
164 | RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP; | |
165 | break; | |
166 | case DPAA_PKT_TYPE_IPV4_SCTP: | |
167 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
168 | RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP; | |
169 | break; | |
170 | case DPAA_PKT_TYPE_IPV6_SCTP: | |
171 | m->packet_type = RTE_PTYPE_L2_ETHER | | |
172 | RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP; | |
173 | break; | |
174 | case DPAA_PKT_TYPE_NONE: | |
175 | m->packet_type = 0; | |
176 | break; | |
177 | /* More switch cases can be added */ | |
178 | default: | |
179 | dpaa_slow_parsing(m, prs); | |
180 | } | |
181 | ||
182 | m->tx_offload = annot->parse.ip_off[0]; | |
183 | m->tx_offload |= (annot->parse.l4_off - annot->parse.ip_off[0]) | |
184 | << DPAA_PKT_L3_LEN_SHIFT; | |
185 | ||
186 | /* Set the hash values */ | |
187 | m->hash.rss = (uint32_t)(annot->hash); | |
188 | /* All packets with Bad checksum are dropped by interface (and | |
189 | * corresponding notification issued to RX error queues). | |
190 | */ | |
191 | m->ol_flags = PKT_RX_RSS_HASH | PKT_RX_IP_CKSUM_GOOD; | |
192 | ||
193 | /* Check if Vlan is present */ | |
194 | if (prs & DPAA_PARSE_VLAN_MASK) | |
195 | m->ol_flags |= PKT_RX_VLAN; | |
196 | /* Packet received without stripping the vlan */ | |
197 | } | |
198 | ||
199 | static inline void dpaa_checksum(struct rte_mbuf *mbuf) | |
200 | { | |
f67539c2 TL |
201 | struct rte_ether_hdr *eth_hdr = |
202 | rte_pktmbuf_mtod(mbuf, struct rte_ether_hdr *); | |
11fdf7f2 | 203 | char *l3_hdr = (char *)eth_hdr + mbuf->l2_len; |
f67539c2 TL |
204 | struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; |
205 | struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; | |
11fdf7f2 TL |
206 | |
207 | DPAA_DP_LOG(DEBUG, "Calculating checksum for mbuf: %p", mbuf); | |
208 | ||
209 | if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || | |
210 | ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == | |
211 | RTE_PTYPE_L3_IPV4_EXT)) { | |
f67539c2 | 212 | ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; |
11fdf7f2 TL |
213 | ipv4_hdr->hdr_checksum = 0; |
214 | ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); | |
215 | } else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == | |
216 | RTE_PTYPE_L3_IPV6) || | |
217 | ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == | |
218 | RTE_PTYPE_L3_IPV6_EXT)) | |
f67539c2 | 219 | ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; |
11fdf7f2 TL |
220 | |
221 | if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) { | |
f67539c2 | 222 | struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)(l3_hdr + |
11fdf7f2 TL |
223 | mbuf->l3_len); |
224 | tcp_hdr->cksum = 0; | |
f67539c2 | 225 | if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4)) |
11fdf7f2 TL |
226 | tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, |
227 | tcp_hdr); | |
f67539c2 | 228 | else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */ |
11fdf7f2 TL |
229 | tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, |
230 | tcp_hdr); | |
231 | } else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == | |
232 | RTE_PTYPE_L4_UDP) { | |
f67539c2 | 233 | struct rte_udp_hdr *udp_hdr = (struct rte_udp_hdr *)(l3_hdr + |
11fdf7f2 TL |
234 | mbuf->l3_len); |
235 | udp_hdr->dgram_cksum = 0; | |
f67539c2 | 236 | if (eth_hdr->ether_type == htons(RTE_ETHER_TYPE_IPV4)) |
11fdf7f2 TL |
237 | udp_hdr->dgram_cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, |
238 | udp_hdr); | |
f67539c2 | 239 | else /* assume ethertype == RTE_ETHER_TYPE_IPV6 */ |
11fdf7f2 TL |
240 | udp_hdr->dgram_cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, |
241 | udp_hdr); | |
242 | } | |
243 | } | |
244 | ||
245 | static inline void dpaa_checksum_offload(struct rte_mbuf *mbuf, | |
246 | struct qm_fd *fd, char *prs_buf) | |
247 | { | |
248 | struct dpaa_eth_parse_results_t *prs; | |
249 | ||
250 | DPAA_DP_LOG(DEBUG, " Offloading checksum for mbuf: %p", mbuf); | |
251 | ||
252 | prs = GET_TX_PRS(prs_buf); | |
253 | prs->l3r = 0; | |
254 | prs->l4r = 0; | |
255 | if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4) || | |
256 | ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == | |
257 | RTE_PTYPE_L3_IPV4_EXT)) | |
258 | prs->l3r = DPAA_L3_PARSE_RESULT_IPV4; | |
259 | else if (((mbuf->packet_type & RTE_PTYPE_L3_MASK) == | |
260 | RTE_PTYPE_L3_IPV6) || | |
261 | ((mbuf->packet_type & RTE_PTYPE_L3_MASK) == | |
262 | RTE_PTYPE_L3_IPV6_EXT)) | |
263 | prs->l3r = DPAA_L3_PARSE_RESULT_IPV6; | |
264 | ||
265 | if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) | |
266 | prs->l4r = DPAA_L4_PARSE_RESULT_TCP; | |
267 | else if ((mbuf->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP) | |
268 | prs->l4r = DPAA_L4_PARSE_RESULT_UDP; | |
269 | ||
270 | prs->ip_off[0] = mbuf->l2_len; | |
271 | prs->l4_off = mbuf->l3_len + mbuf->l2_len; | |
272 | /* Enable L3 (and L4, if TCP or UDP) HW checksum*/ | |
273 | fd->cmd = DPAA_FD_CMD_RPD | DPAA_FD_CMD_DTC; | |
274 | } | |
275 | ||
276 | static inline void | |
277 | dpaa_unsegmented_checksum(struct rte_mbuf *mbuf, struct qm_fd *fd_arr) | |
278 | { | |
279 | if (!mbuf->packet_type) { | |
280 | struct rte_net_hdr_lens hdr_lens; | |
281 | ||
282 | mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, | |
283 | RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK | |
284 | | RTE_PTYPE_L4_MASK); | |
285 | mbuf->l2_len = hdr_lens.l2_len; | |
286 | mbuf->l3_len = hdr_lens.l3_len; | |
287 | } | |
288 | if (mbuf->data_off < (DEFAULT_TX_ICEOF + | |
289 | sizeof(struct dpaa_eth_parse_results_t))) { | |
290 | DPAA_DP_LOG(DEBUG, "Checksum offload Err: " | |
291 | "Not enough Headroom " | |
292 | "space for correct Checksum offload." | |
293 | "So Calculating checksum in Software."); | |
294 | dpaa_checksum(mbuf); | |
295 | } else { | |
296 | dpaa_checksum_offload(mbuf, fd_arr, mbuf->buf_addr); | |
297 | } | |
298 | } | |
299 | ||
300 | struct rte_mbuf * | |
301 | dpaa_eth_sg_to_mbuf(const struct qm_fd *fd, uint32_t ifid) | |
302 | { | |
303 | struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); | |
304 | struct rte_mbuf *first_seg, *prev_seg, *cur_seg, *temp; | |
305 | struct qm_sg_entry *sgt, *sg_temp; | |
306 | void *vaddr, *sg_vaddr; | |
307 | int i = 0; | |
308 | uint8_t fd_offset = fd->offset; | |
309 | ||
11fdf7f2 TL |
310 | vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); |
311 | if (!vaddr) { | |
312 | DPAA_PMD_ERR("unable to convert physical address"); | |
313 | return NULL; | |
314 | } | |
315 | sgt = vaddr + fd_offset; | |
316 | sg_temp = &sgt[i++]; | |
317 | hw_sg_to_cpu(sg_temp); | |
318 | temp = (struct rte_mbuf *)((char *)vaddr - bp_info->meta_data_size); | |
319 | sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, qm_sg_entry_get64(sg_temp)); | |
320 | ||
321 | first_seg = (struct rte_mbuf *)((char *)sg_vaddr - | |
322 | bp_info->meta_data_size); | |
323 | first_seg->data_off = sg_temp->offset; | |
324 | first_seg->data_len = sg_temp->length; | |
325 | first_seg->pkt_len = sg_temp->length; | |
326 | rte_mbuf_refcnt_set(first_seg, 1); | |
327 | ||
328 | first_seg->port = ifid; | |
329 | first_seg->nb_segs = 1; | |
330 | first_seg->ol_flags = 0; | |
331 | prev_seg = first_seg; | |
332 | while (i < DPAA_SGT_MAX_ENTRIES) { | |
333 | sg_temp = &sgt[i++]; | |
334 | hw_sg_to_cpu(sg_temp); | |
335 | sg_vaddr = DPAA_MEMPOOL_PTOV(bp_info, | |
336 | qm_sg_entry_get64(sg_temp)); | |
337 | cur_seg = (struct rte_mbuf *)((char *)sg_vaddr - | |
338 | bp_info->meta_data_size); | |
339 | cur_seg->data_off = sg_temp->offset; | |
340 | cur_seg->data_len = sg_temp->length; | |
341 | first_seg->pkt_len += sg_temp->length; | |
342 | first_seg->nb_segs += 1; | |
343 | rte_mbuf_refcnt_set(cur_seg, 1); | |
344 | prev_seg->next = cur_seg; | |
345 | if (sg_temp->final) { | |
346 | cur_seg->next = NULL; | |
347 | break; | |
348 | } | |
349 | prev_seg = cur_seg; | |
350 | } | |
9f95a23c TL |
351 | DPAA_DP_LOG(DEBUG, "Received an SG frame len =%d, num_sg =%d", |
352 | first_seg->pkt_len, first_seg->nb_segs); | |
11fdf7f2 TL |
353 | |
354 | dpaa_eth_packet_info(first_seg, vaddr); | |
355 | rte_pktmbuf_free_seg(temp); | |
356 | ||
357 | return first_seg; | |
358 | } | |
359 | ||
360 | static inline struct rte_mbuf * | |
361 | dpaa_eth_fd_to_mbuf(const struct qm_fd *fd, uint32_t ifid) | |
362 | { | |
363 | struct rte_mbuf *mbuf; | |
364 | struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(fd->bpid); | |
365 | void *ptr; | |
366 | uint8_t format = | |
367 | (fd->opaque & DPAA_FD_FORMAT_MASK) >> DPAA_FD_FORMAT_SHIFT; | |
368 | uint16_t offset; | |
369 | uint32_t length; | |
370 | ||
11fdf7f2 TL |
371 | if (unlikely(format == qm_fd_sg)) |
372 | return dpaa_eth_sg_to_mbuf(fd, ifid); | |
373 | ||
11fdf7f2 TL |
374 | offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> DPAA_FD_OFFSET_SHIFT; |
375 | length = fd->opaque & DPAA_FD_LENGTH_MASK; | |
376 | ||
9f95a23c TL |
377 | DPAA_DP_LOG(DEBUG, " FD--->MBUF off %d len = %d", offset, length); |
378 | ||
11fdf7f2 TL |
379 | /* Ignoring case when format != qm_fd_contig */ |
380 | dpaa_display_frame(fd); | |
9f95a23c | 381 | ptr = DPAA_MEMPOOL_PTOV(bp_info, qm_fd_addr(fd)); |
11fdf7f2 TL |
382 | |
383 | mbuf = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); | |
9f95a23c TL |
384 | /* Prefetch the Parse results and packet data to L1 */ |
385 | rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); | |
11fdf7f2 TL |
386 | |
387 | mbuf->data_off = offset; | |
388 | mbuf->data_len = length; | |
389 | mbuf->pkt_len = length; | |
390 | ||
391 | mbuf->port = ifid; | |
392 | mbuf->nb_segs = 1; | |
393 | mbuf->ol_flags = 0; | |
394 | mbuf->next = NULL; | |
395 | rte_mbuf_refcnt_set(mbuf, 1); | |
396 | dpaa_eth_packet_info(mbuf, mbuf->buf_addr); | |
397 | ||
398 | return mbuf; | |
399 | } | |
400 | ||
9f95a23c | 401 | /* Specific for LS1043 */ |
11fdf7f2 | 402 | void |
9f95a23c | 403 | dpaa_rx_cb_no_prefetch(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, |
11fdf7f2 TL |
404 | void **bufs, int num_bufs) |
405 | { | |
406 | struct rte_mbuf *mbuf; | |
407 | struct dpaa_bp_info *bp_info; | |
408 | const struct qm_fd *fd; | |
409 | void *ptr; | |
410 | struct dpaa_if *dpaa_intf; | |
411 | uint16_t offset, i; | |
412 | uint32_t length; | |
413 | uint8_t format; | |
414 | ||
9f95a23c TL |
415 | bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[0]->fd.bpid); |
416 | ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[0]->fd)); | |
417 | rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); | |
418 | bufs[0] = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); | |
11fdf7f2 TL |
419 | |
420 | for (i = 0; i < num_bufs; i++) { | |
9f95a23c | 421 | if (i < num_bufs - 1) { |
11fdf7f2 TL |
422 | bp_info = DPAA_BPID_TO_POOL_INFO(dqrr[i + 1]->fd.bpid); |
423 | ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dqrr[i + 1]->fd)); | |
424 | rte_prefetch0((void *)((uint8_t *)ptr + | |
425 | DEFAULT_RX_ICEOF)); | |
426 | bufs[i + 1] = (struct rte_mbuf *)((char *)ptr - | |
427 | bp_info->meta_data_size); | |
428 | } | |
429 | ||
430 | fd = &dqrr[i]->fd; | |
431 | dpaa_intf = fq[0]->dpaa_intf; | |
432 | ||
433 | format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> | |
434 | DPAA_FD_FORMAT_SHIFT; | |
435 | if (unlikely(format == qm_fd_sg)) { | |
436 | bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); | |
437 | continue; | |
438 | } | |
439 | ||
440 | offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> | |
441 | DPAA_FD_OFFSET_SHIFT; | |
442 | length = fd->opaque & DPAA_FD_LENGTH_MASK; | |
443 | ||
444 | mbuf = bufs[i]; | |
445 | mbuf->data_off = offset; | |
446 | mbuf->data_len = length; | |
447 | mbuf->pkt_len = length; | |
448 | mbuf->port = dpaa_intf->ifid; | |
449 | ||
450 | mbuf->nb_segs = 1; | |
451 | mbuf->ol_flags = 0; | |
452 | mbuf->next = NULL; | |
453 | rte_mbuf_refcnt_set(mbuf, 1); | |
454 | dpaa_eth_packet_info(mbuf, mbuf->buf_addr); | |
455 | } | |
456 | } | |
457 | ||
9f95a23c TL |
458 | void |
459 | dpaa_rx_cb(struct qman_fq **fq, struct qm_dqrr_entry **dqrr, | |
460 | void **bufs, int num_bufs) | |
461 | { | |
462 | struct rte_mbuf *mbuf; | |
463 | const struct qm_fd *fd; | |
464 | struct dpaa_if *dpaa_intf; | |
465 | uint16_t offset, i; | |
466 | uint32_t length; | |
467 | uint8_t format; | |
468 | ||
469 | for (i = 0; i < num_bufs; i++) { | |
470 | fd = &dqrr[i]->fd; | |
471 | dpaa_intf = fq[0]->dpaa_intf; | |
472 | ||
473 | format = (fd->opaque & DPAA_FD_FORMAT_MASK) >> | |
474 | DPAA_FD_FORMAT_SHIFT; | |
475 | if (unlikely(format == qm_fd_sg)) { | |
476 | bufs[i] = dpaa_eth_sg_to_mbuf(fd, dpaa_intf->ifid); | |
477 | continue; | |
478 | } | |
479 | ||
480 | offset = (fd->opaque & DPAA_FD_OFFSET_MASK) >> | |
481 | DPAA_FD_OFFSET_SHIFT; | |
482 | length = fd->opaque & DPAA_FD_LENGTH_MASK; | |
483 | ||
484 | mbuf = bufs[i]; | |
485 | mbuf->data_off = offset; | |
486 | mbuf->data_len = length; | |
487 | mbuf->pkt_len = length; | |
488 | mbuf->port = dpaa_intf->ifid; | |
489 | ||
490 | mbuf->nb_segs = 1; | |
491 | mbuf->ol_flags = 0; | |
492 | mbuf->next = NULL; | |
493 | rte_mbuf_refcnt_set(mbuf, 1); | |
494 | dpaa_eth_packet_info(mbuf, mbuf->buf_addr); | |
495 | } | |
496 | } | |
497 | ||
11fdf7f2 TL |
498 | void dpaa_rx_cb_prepare(struct qm_dqrr_entry *dq, void **bufs) |
499 | { | |
500 | struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(dq->fd.bpid); | |
501 | void *ptr = rte_dpaa_mem_ptov(qm_fd_addr(&dq->fd)); | |
502 | ||
503 | /* In case of LS1046, annotation stashing is disabled due to L2 cache | |
504 | * being bottleneck in case of multicore scanario for this platform. | |
505 | * So we prefetch the annoation beforehand, so that it is available | |
506 | * in cache when accessed. | |
507 | */ | |
9f95a23c | 508 | rte_prefetch0((void *)((uint8_t *)ptr + DEFAULT_RX_ICEOF)); |
11fdf7f2 TL |
509 | |
510 | *bufs = (struct rte_mbuf *)((char *)ptr - bp_info->meta_data_size); | |
511 | } | |
512 | ||
513 | static uint16_t | |
514 | dpaa_eth_queue_portal_rx(struct qman_fq *fq, | |
515 | struct rte_mbuf **bufs, | |
516 | uint16_t nb_bufs) | |
517 | { | |
518 | int ret; | |
519 | ||
f67539c2 | 520 | if (unlikely(!fq->qp_initialized)) { |
11fdf7f2 TL |
521 | ret = rte_dpaa_portal_fq_init((void *)0, fq); |
522 | if (ret) { | |
523 | DPAA_PMD_ERR("Failure in affining portal %d", ret); | |
524 | return 0; | |
525 | } | |
f67539c2 | 526 | fq->qp_initialized = 1; |
11fdf7f2 TL |
527 | } |
528 | ||
529 | return qman_portal_poll_rx(nb_bufs, (void **)bufs, fq->qp); | |
530 | } | |
531 | ||
532 | enum qman_cb_dqrr_result | |
533 | dpaa_rx_cb_parallel(void *event, | |
534 | struct qman_portal *qm __always_unused, | |
535 | struct qman_fq *fq, | |
536 | const struct qm_dqrr_entry *dqrr, | |
537 | void **bufs) | |
538 | { | |
539 | u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; | |
540 | struct rte_mbuf *mbuf; | |
541 | struct rte_event *ev = (struct rte_event *)event; | |
542 | ||
543 | mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); | |
544 | ev->event_ptr = (void *)mbuf; | |
545 | ev->flow_id = fq->ev.flow_id; | |
546 | ev->sub_event_type = fq->ev.sub_event_type; | |
547 | ev->event_type = RTE_EVENT_TYPE_ETHDEV; | |
548 | ev->op = RTE_EVENT_OP_NEW; | |
549 | ev->sched_type = fq->ev.sched_type; | |
550 | ev->queue_id = fq->ev.queue_id; | |
551 | ev->priority = fq->ev.priority; | |
552 | ev->impl_opaque = (uint8_t)DPAA_INVALID_MBUF_SEQN; | |
553 | mbuf->seqn = DPAA_INVALID_MBUF_SEQN; | |
554 | *bufs = mbuf; | |
555 | ||
556 | return qman_cb_dqrr_consume; | |
557 | } | |
558 | ||
559 | enum qman_cb_dqrr_result | |
560 | dpaa_rx_cb_atomic(void *event, | |
561 | struct qman_portal *qm __always_unused, | |
562 | struct qman_fq *fq, | |
563 | const struct qm_dqrr_entry *dqrr, | |
564 | void **bufs) | |
565 | { | |
566 | u8 index; | |
567 | u32 ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; | |
568 | struct rte_mbuf *mbuf; | |
569 | struct rte_event *ev = (struct rte_event *)event; | |
570 | ||
571 | mbuf = dpaa_eth_fd_to_mbuf(&dqrr->fd, ifid); | |
572 | ev->event_ptr = (void *)mbuf; | |
573 | ev->flow_id = fq->ev.flow_id; | |
574 | ev->sub_event_type = fq->ev.sub_event_type; | |
575 | ev->event_type = RTE_EVENT_TYPE_ETHDEV; | |
576 | ev->op = RTE_EVENT_OP_NEW; | |
577 | ev->sched_type = fq->ev.sched_type; | |
578 | ev->queue_id = fq->ev.queue_id; | |
579 | ev->priority = fq->ev.priority; | |
580 | ||
581 | /* Save active dqrr entries */ | |
582 | index = DQRR_PTR2IDX(dqrr); | |
583 | DPAA_PER_LCORE_DQRR_SIZE++; | |
584 | DPAA_PER_LCORE_DQRR_HELD |= 1 << index; | |
585 | DPAA_PER_LCORE_DQRR_MBUF(index) = mbuf; | |
586 | ev->impl_opaque = index + 1; | |
587 | mbuf->seqn = (uint32_t)index + 1; | |
588 | *bufs = mbuf; | |
589 | ||
590 | return qman_cb_dqrr_defer; | |
591 | } | |
592 | ||
593 | uint16_t dpaa_eth_queue_rx(void *q, | |
594 | struct rte_mbuf **bufs, | |
595 | uint16_t nb_bufs) | |
596 | { | |
597 | struct qman_fq *fq = q; | |
598 | struct qm_dqrr_entry *dq; | |
599 | uint32_t num_rx = 0, ifid = ((struct dpaa_if *)fq->dpaa_intf)->ifid; | |
600 | int num_rx_bufs, ret; | |
601 | uint32_t vdqcr_flags = 0; | |
602 | ||
9f95a23c TL |
603 | if (unlikely(rte_dpaa_bpid_info == NULL && |
604 | rte_eal_process_type() == RTE_PROC_SECONDARY)) | |
605 | rte_dpaa_bpid_info = fq->bp_array; | |
606 | ||
11fdf7f2 TL |
607 | if (likely(fq->is_static)) |
608 | return dpaa_eth_queue_portal_rx(fq, bufs, nb_bufs); | |
609 | ||
610 | if (unlikely(!RTE_PER_LCORE(dpaa_io))) { | |
611 | ret = rte_dpaa_portal_init((void *)0); | |
612 | if (ret) { | |
613 | DPAA_PMD_ERR("Failure in affining portal"); | |
614 | return 0; | |
615 | } | |
616 | } | |
617 | ||
618 | /* Until request for four buffers, we provide exact number of buffers. | |
619 | * Otherwise we do not set the QM_VDQCR_EXACT flag. | |
620 | * Not setting QM_VDQCR_EXACT flag can provide two more buffers than | |
621 | * requested, so we request two less in this case. | |
622 | */ | |
623 | if (nb_bufs < 4) { | |
624 | vdqcr_flags = QM_VDQCR_EXACT; | |
625 | num_rx_bufs = nb_bufs; | |
626 | } else { | |
627 | num_rx_bufs = nb_bufs > DPAA_MAX_DEQUEUE_NUM_FRAMES ? | |
628 | (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_bufs - 2); | |
629 | } | |
630 | ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags); | |
631 | if (ret) | |
632 | return 0; | |
633 | ||
634 | do { | |
635 | dq = qman_dequeue(fq); | |
636 | if (!dq) | |
637 | continue; | |
638 | bufs[num_rx++] = dpaa_eth_fd_to_mbuf(&dq->fd, ifid); | |
639 | qman_dqrr_consume(fq, dq); | |
640 | } while (fq->flags & QMAN_FQ_STATE_VDQCR); | |
641 | ||
642 | return num_rx; | |
643 | } | |
644 | ||
11fdf7f2 TL |
645 | int |
646 | dpaa_eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, | |
647 | struct qm_fd *fd, | |
648 | uint32_t bpid) | |
649 | { | |
650 | struct rte_mbuf *cur_seg = mbuf, *prev_seg = NULL; | |
651 | struct dpaa_bp_info *bp_info = DPAA_BPID_TO_POOL_INFO(bpid); | |
652 | struct rte_mbuf *temp, *mi; | |
653 | struct qm_sg_entry *sg_temp, *sgt; | |
654 | int i = 0; | |
655 | ||
656 | DPAA_DP_LOG(DEBUG, "Creating SG FD to transmit"); | |
657 | ||
658 | temp = rte_pktmbuf_alloc(bp_info->mp); | |
659 | if (!temp) { | |
660 | DPAA_PMD_ERR("Failure in allocation of mbuf"); | |
661 | return -1; | |
662 | } | |
663 | if (temp->buf_len < ((mbuf->nb_segs * sizeof(struct qm_sg_entry)) | |
664 | + temp->data_off)) { | |
665 | DPAA_PMD_ERR("Insufficient space in mbuf for SG entries"); | |
666 | return -1; | |
667 | } | |
668 | ||
669 | fd->cmd = 0; | |
670 | fd->opaque_addr = 0; | |
671 | ||
672 | if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) { | |
673 | if (!mbuf->packet_type) { | |
674 | struct rte_net_hdr_lens hdr_lens; | |
675 | ||
676 | mbuf->packet_type = rte_net_get_ptype(mbuf, &hdr_lens, | |
677 | RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK | |
678 | | RTE_PTYPE_L4_MASK); | |
679 | mbuf->l2_len = hdr_lens.l2_len; | |
680 | mbuf->l3_len = hdr_lens.l3_len; | |
681 | } | |
682 | if (temp->data_off < DEFAULT_TX_ICEOF | |
683 | + sizeof(struct dpaa_eth_parse_results_t)) | |
684 | temp->data_off = DEFAULT_TX_ICEOF | |
685 | + sizeof(struct dpaa_eth_parse_results_t); | |
686 | dcbz_64(temp->buf_addr); | |
687 | dpaa_checksum_offload(mbuf, fd, temp->buf_addr); | |
688 | } | |
689 | ||
690 | sgt = temp->buf_addr + temp->data_off; | |
691 | fd->format = QM_FD_SG; | |
692 | fd->addr = temp->buf_iova; | |
693 | fd->offset = temp->data_off; | |
694 | fd->bpid = bpid; | |
695 | fd->length20 = mbuf->pkt_len; | |
696 | ||
697 | while (i < DPAA_SGT_MAX_ENTRIES) { | |
698 | sg_temp = &sgt[i++]; | |
699 | sg_temp->opaque = 0; | |
700 | sg_temp->val = 0; | |
701 | sg_temp->addr = cur_seg->buf_iova; | |
702 | sg_temp->offset = cur_seg->data_off; | |
703 | sg_temp->length = cur_seg->data_len; | |
704 | if (RTE_MBUF_DIRECT(cur_seg)) { | |
705 | if (rte_mbuf_refcnt_read(cur_seg) > 1) { | |
706 | /*If refcnt > 1, invalid bpid is set to ensure | |
707 | * buffer is not freed by HW. | |
708 | */ | |
709 | sg_temp->bpid = 0xff; | |
710 | rte_mbuf_refcnt_update(cur_seg, -1); | |
711 | } else { | |
712 | sg_temp->bpid = | |
713 | DPAA_MEMPOOL_TO_BPID(cur_seg->pool); | |
714 | } | |
715 | cur_seg = cur_seg->next; | |
716 | } else { | |
717 | /* Get owner MBUF from indirect buffer */ | |
718 | mi = rte_mbuf_from_indirect(cur_seg); | |
719 | if (rte_mbuf_refcnt_read(mi) > 1) { | |
720 | /*If refcnt > 1, invalid bpid is set to ensure | |
721 | * owner buffer is not freed by HW. | |
722 | */ | |
723 | sg_temp->bpid = 0xff; | |
724 | } else { | |
725 | sg_temp->bpid = DPAA_MEMPOOL_TO_BPID(mi->pool); | |
726 | rte_mbuf_refcnt_update(mi, 1); | |
727 | } | |
728 | prev_seg = cur_seg; | |
729 | cur_seg = cur_seg->next; | |
730 | prev_seg->next = NULL; | |
731 | rte_pktmbuf_free(prev_seg); | |
732 | } | |
733 | if (cur_seg == NULL) { | |
734 | sg_temp->final = 1; | |
735 | cpu_to_hw_sg(sg_temp); | |
736 | break; | |
737 | } | |
738 | cpu_to_hw_sg(sg_temp); | |
739 | } | |
740 | return 0; | |
741 | } | |
742 | ||
743 | /* Handle mbufs which are not segmented (non SG) */ | |
744 | static inline void | |
745 | tx_on_dpaa_pool_unsegmented(struct rte_mbuf *mbuf, | |
746 | struct dpaa_bp_info *bp_info, | |
747 | struct qm_fd *fd_arr) | |
748 | { | |
749 | struct rte_mbuf *mi = NULL; | |
750 | ||
751 | if (RTE_MBUF_DIRECT(mbuf)) { | |
752 | if (rte_mbuf_refcnt_read(mbuf) > 1) { | |
753 | /* In case of direct mbuf and mbuf being cloned, | |
754 | * BMAN should _not_ release buffer. | |
755 | */ | |
756 | DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); | |
757 | /* Buffer should be releasd by EAL */ | |
758 | rte_mbuf_refcnt_update(mbuf, -1); | |
759 | } else { | |
760 | /* In case of direct mbuf and no cloning, mbuf can be | |
761 | * released by BMAN. | |
762 | */ | |
763 | DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); | |
764 | } | |
765 | } else { | |
766 | /* This is data-containing core mbuf: 'mi' */ | |
767 | mi = rte_mbuf_from_indirect(mbuf); | |
768 | if (rte_mbuf_refcnt_read(mi) > 1) { | |
769 | /* In case of indirect mbuf, and mbuf being cloned, | |
770 | * BMAN should _not_ release it and let EAL release | |
771 | * it through pktmbuf_free below. | |
772 | */ | |
773 | DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, 0xff); | |
774 | } else { | |
775 | /* In case of indirect mbuf, and no cloning, core mbuf | |
776 | * should be released by BMAN. | |
777 | * Increate refcnt of core mbuf so that when | |
778 | * pktmbuf_free is called and mbuf is released, EAL | |
779 | * doesn't try to release core mbuf which would have | |
780 | * been released by BMAN. | |
781 | */ | |
782 | rte_mbuf_refcnt_update(mi, 1); | |
783 | DPAA_MBUF_TO_CONTIG_FD(mbuf, fd_arr, bp_info->bpid); | |
784 | } | |
785 | rte_pktmbuf_free(mbuf); | |
786 | } | |
787 | ||
788 | if (mbuf->ol_flags & DPAA_TX_CKSUM_OFFLOAD_MASK) | |
789 | dpaa_unsegmented_checksum(mbuf, fd_arr); | |
790 | } | |
791 | ||
792 | /* Handle all mbufs on dpaa BMAN managed pool */ | |
793 | static inline uint16_t | |
794 | tx_on_dpaa_pool(struct rte_mbuf *mbuf, | |
795 | struct dpaa_bp_info *bp_info, | |
796 | struct qm_fd *fd_arr) | |
797 | { | |
798 | DPAA_DP_LOG(DEBUG, "BMAN offloaded buffer, mbuf: %p", mbuf); | |
799 | ||
800 | if (mbuf->nb_segs == 1) { | |
801 | /* Case for non-segmented buffers */ | |
802 | tx_on_dpaa_pool_unsegmented(mbuf, bp_info, fd_arr); | |
803 | } else if (mbuf->nb_segs > 1 && | |
804 | mbuf->nb_segs <= DPAA_SGT_MAX_ENTRIES) { | |
805 | if (dpaa_eth_mbuf_to_sg_fd(mbuf, fd_arr, bp_info->bpid)) { | |
806 | DPAA_PMD_DEBUG("Unable to create Scatter Gather FD"); | |
807 | return 1; | |
808 | } | |
809 | } else { | |
810 | DPAA_PMD_DEBUG("Number of Segments not supported"); | |
811 | return 1; | |
812 | } | |
813 | ||
814 | return 0; | |
815 | } | |
816 | ||
817 | /* Handle all mbufs on an external pool (non-dpaa) */ | |
f67539c2 TL |
818 | static inline struct rte_mbuf * |
819 | reallocate_mbuf(struct qman_fq *txq, struct rte_mbuf *mbuf) | |
11fdf7f2 TL |
820 | { |
821 | struct dpaa_if *dpaa_intf = txq->dpaa_intf; | |
f67539c2 TL |
822 | struct dpaa_bp_info *bp_info = dpaa_intf->bp_info; |
823 | struct rte_mbuf *new_mbufs[DPAA_SGT_MAX_ENTRIES + 1] = {0}; | |
824 | struct rte_mbuf *temp_mbuf; | |
825 | int num_new_segs, mbuf_greater, ret, extra_seg = 0, i = 0; | |
826 | uint64_t mbufs_size, bytes_to_copy, offset1 = 0, offset2 = 0; | |
827 | char *data; | |
828 | ||
829 | DPAA_DP_LOG(DEBUG, "Reallocating transmit buffer"); | |
830 | ||
831 | mbufs_size = bp_info->size - | |
832 | bp_info->meta_data_size - RTE_PKTMBUF_HEADROOM; | |
833 | extra_seg = !!(mbuf->pkt_len % mbufs_size); | |
834 | num_new_segs = (mbuf->pkt_len / mbufs_size) + extra_seg; | |
835 | ||
836 | ret = rte_pktmbuf_alloc_bulk(bp_info->mp, new_mbufs, num_new_segs); | |
837 | if (ret != 0) { | |
838 | DPAA_DP_LOG(DEBUG, "Allocation for new buffers failed"); | |
839 | return NULL; | |
840 | } | |
11fdf7f2 | 841 | |
f67539c2 TL |
842 | temp_mbuf = mbuf; |
843 | ||
844 | while (temp_mbuf) { | |
845 | /* If mbuf data is less than new mbuf remaining memory */ | |
846 | if ((temp_mbuf->data_len - offset1) < (mbufs_size - offset2)) { | |
847 | bytes_to_copy = temp_mbuf->data_len - offset1; | |
848 | mbuf_greater = -1; | |
849 | /* If mbuf data is greater than new mbuf remaining memory */ | |
850 | } else if ((temp_mbuf->data_len - offset1) > | |
851 | (mbufs_size - offset2)) { | |
852 | bytes_to_copy = mbufs_size - offset2; | |
853 | mbuf_greater = 1; | |
854 | /* if mbuf data is equal to new mbuf remaining memory */ | |
855 | } else { | |
856 | bytes_to_copy = temp_mbuf->data_len - offset1; | |
857 | mbuf_greater = 0; | |
858 | } | |
859 | ||
860 | /* Copy the data */ | |
861 | data = rte_pktmbuf_append(new_mbufs[0], bytes_to_copy); | |
862 | ||
863 | rte_memcpy((uint8_t *)data, rte_pktmbuf_mtod_offset(mbuf, | |
864 | void *, offset1), bytes_to_copy); | |
865 | ||
866 | /* Set new offsets and the temp buffers */ | |
867 | if (mbuf_greater == -1) { | |
868 | offset1 = 0; | |
869 | offset2 += bytes_to_copy; | |
870 | temp_mbuf = temp_mbuf->next; | |
871 | } else if (mbuf_greater == 1) { | |
872 | offset2 = 0; | |
873 | offset1 += bytes_to_copy; | |
874 | new_mbufs[i]->next = new_mbufs[i + 1]; | |
875 | new_mbufs[0]->nb_segs++; | |
876 | i++; | |
877 | } else { | |
878 | offset1 = 0; | |
879 | offset2 = 0; | |
880 | temp_mbuf = temp_mbuf->next; | |
881 | new_mbufs[i]->next = new_mbufs[i + 1]; | |
882 | if (new_mbufs[i + 1]) | |
883 | new_mbufs[0]->nb_segs++; | |
884 | i++; | |
885 | } | |
11fdf7f2 TL |
886 | } |
887 | ||
f67539c2 TL |
888 | /* Copy other required fields */ |
889 | new_mbufs[0]->ol_flags = mbuf->ol_flags; | |
890 | new_mbufs[0]->packet_type = mbuf->packet_type; | |
891 | new_mbufs[0]->tx_offload = mbuf->tx_offload; | |
11fdf7f2 | 892 | |
f67539c2 TL |
893 | rte_pktmbuf_free(mbuf); |
894 | ||
895 | return new_mbufs[0]; | |
11fdf7f2 TL |
896 | } |
897 | ||
898 | uint16_t | |
899 | dpaa_eth_queue_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) | |
900 | { | |
901 | struct rte_mbuf *mbuf, *mi = NULL; | |
902 | struct rte_mempool *mp; | |
903 | struct dpaa_bp_info *bp_info; | |
904 | struct qm_fd fd_arr[DPAA_TX_BURST_SIZE]; | |
905 | uint32_t frames_to_send, loop, sent = 0; | |
906 | uint16_t state; | |
f67539c2 | 907 | int ret, realloc_mbuf = 0; |
11fdf7f2 TL |
908 | uint32_t seqn, index, flags[DPAA_TX_BURST_SIZE] = {0}; |
909 | ||
910 | if (unlikely(!RTE_PER_LCORE(dpaa_io))) { | |
911 | ret = rte_dpaa_portal_init((void *)0); | |
912 | if (ret) { | |
913 | DPAA_PMD_ERR("Failure in affining portal"); | |
914 | return 0; | |
915 | } | |
916 | } | |
917 | ||
918 | DPAA_DP_LOG(DEBUG, "Transmitting %d buffers on queue: %p", nb_bufs, q); | |
919 | ||
920 | while (nb_bufs) { | |
921 | frames_to_send = (nb_bufs > DPAA_TX_BURST_SIZE) ? | |
922 | DPAA_TX_BURST_SIZE : nb_bufs; | |
923 | for (loop = 0; loop < frames_to_send; loop++) { | |
924 | mbuf = *(bufs++); | |
f67539c2 TL |
925 | /* In case the data offset is not multiple of 16, |
926 | * FMAN can stall because of an errata. So reallocate | |
927 | * the buffer in such case. | |
928 | */ | |
929 | if (dpaa_svr_family == SVR_LS1043A_FAMILY && | |
930 | (mbuf->data_off & 0x7F) != 0x0) | |
931 | realloc_mbuf = 1; | |
9f95a23c TL |
932 | seqn = mbuf->seqn; |
933 | if (seqn != DPAA_INVALID_MBUF_SEQN) { | |
934 | index = seqn - 1; | |
935 | if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { | |
936 | flags[loop] = | |
937 | ((index & QM_EQCR_DCA_IDXMASK) << 8); | |
938 | flags[loop] |= QMAN_ENQUEUE_FLAG_DCA; | |
939 | DPAA_PER_LCORE_DQRR_SIZE--; | |
940 | DPAA_PER_LCORE_DQRR_HELD &= | |
941 | ~(1 << index); | |
942 | } | |
943 | } | |
944 | ||
11fdf7f2 TL |
945 | if (likely(RTE_MBUF_DIRECT(mbuf))) { |
946 | mp = mbuf->pool; | |
947 | bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); | |
948 | if (likely(mp->ops_index == | |
949 | bp_info->dpaa_ops_index && | |
950 | mbuf->nb_segs == 1 && | |
f67539c2 | 951 | realloc_mbuf == 0 && |
11fdf7f2 TL |
952 | rte_mbuf_refcnt_read(mbuf) == 1)) { |
953 | DPAA_MBUF_TO_CONTIG_FD(mbuf, | |
954 | &fd_arr[loop], bp_info->bpid); | |
955 | if (mbuf->ol_flags & | |
956 | DPAA_TX_CKSUM_OFFLOAD_MASK) | |
957 | dpaa_unsegmented_checksum(mbuf, | |
958 | &fd_arr[loop]); | |
959 | continue; | |
960 | } | |
961 | } else { | |
962 | mi = rte_mbuf_from_indirect(mbuf); | |
963 | mp = mi->pool; | |
964 | } | |
965 | ||
966 | bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp); | |
f67539c2 TL |
967 | if (unlikely(mp->ops_index != bp_info->dpaa_ops_index || |
968 | realloc_mbuf == 1)) { | |
969 | struct rte_mbuf *temp_mbuf; | |
970 | ||
971 | temp_mbuf = reallocate_mbuf(q, mbuf); | |
972 | if (!temp_mbuf) { | |
11fdf7f2 TL |
973 | /* Set frames_to_send & nb_bufs so |
974 | * that packets are transmitted till | |
975 | * previous frame. | |
976 | */ | |
977 | frames_to_send = loop; | |
978 | nb_bufs = loop; | |
979 | goto send_pkts; | |
980 | } | |
f67539c2 TL |
981 | mbuf = temp_mbuf; |
982 | realloc_mbuf = 0; | |
983 | } | |
984 | ||
985 | state = tx_on_dpaa_pool(mbuf, bp_info, | |
986 | &fd_arr[loop]); | |
987 | if (unlikely(state)) { | |
988 | /* Set frames_to_send & nb_bufs so | |
989 | * that packets are transmitted till | |
990 | * previous frame. | |
991 | */ | |
992 | frames_to_send = loop; | |
993 | nb_bufs = loop; | |
994 | goto send_pkts; | |
11fdf7f2 | 995 | } |
11fdf7f2 TL |
996 | } |
997 | ||
998 | send_pkts: | |
999 | loop = 0; | |
1000 | while (loop < frames_to_send) { | |
1001 | loop += qman_enqueue_multi(q, &fd_arr[loop], | |
1002 | &flags[loop], | |
1003 | frames_to_send - loop); | |
1004 | } | |
1005 | nb_bufs -= frames_to_send; | |
1006 | sent += frames_to_send; | |
1007 | } | |
1008 | ||
1009 | DPAA_DP_LOG(DEBUG, "Transmitted %d buffers on queue: %p", sent, q); | |
1010 | ||
1011 | return sent; | |
1012 | } | |
1013 | ||
1014 | uint16_t dpaa_eth_tx_drop_all(void *q __rte_unused, | |
1015 | struct rte_mbuf **bufs __rte_unused, | |
1016 | uint16_t nb_bufs __rte_unused) | |
1017 | { | |
1018 | DPAA_DP_LOG(DEBUG, "Drop all packets"); | |
1019 | ||
1020 | /* Drop all incoming packets. No need to free packets here | |
1021 | * because the rte_eth f/w frees up the packets through tx_buffer | |
1022 | * callback in case this functions returns count less than nb_bufs | |
1023 | */ | |
1024 | return 0; | |
1025 | } |