]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * | |
3 | * Copyright (c) 2016-2018 Solarflare Communications Inc. | |
4 | * All rights reserved. | |
5 | * | |
6 | * This software was jointly developed between OKTET Labs (under contract | |
7 | * for Solarflare) and Solarflare Communications, Inc. | |
8 | */ | |
9 | ||
10 | /* EF10 native datapath implementation */ | |
11 | ||
12 | #include <stdbool.h> | |
13 | ||
14 | #include <rte_byteorder.h> | |
15 | #include <rte_mbuf_ptype.h> | |
16 | #include <rte_mbuf.h> | |
17 | #include <rte_io.h> | |
18 | ||
19 | #include "efx.h" | |
20 | #include "efx_types.h" | |
21 | #include "efx_regs.h" | |
22 | #include "efx_regs_ef10.h" | |
23 | ||
24 | #include "sfc_tweak.h" | |
25 | #include "sfc_dp_rx.h" | |
26 | #include "sfc_kvargs.h" | |
27 | #include "sfc_ef10.h" | |
28 | ||
29 | #define SFC_EF10_RX_EV_ENCAP_SUPPORT 1 | |
30 | #include "sfc_ef10_rx_ev.h" | |
31 | ||
32 | #define sfc_ef10_rx_err(dpq, ...) \ | |
33 | SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) | |
34 | ||
35 | /** | |
36 | * Maximum number of descriptors/buffers in the Rx ring. | |
37 | * It should guarantee that corresponding event queue never overfill. | |
38 | * EF10 native datapath uses event queue of the same size as Rx queue. | |
39 | * Maximum number of events on datapath can be estimated as number of | |
40 | * Rx queue entries (one event per Rx buffer in the worst case) plus | |
41 | * Rx error and flush events. | |
42 | */ | |
43 | #define SFC_EF10_RXQ_LIMIT(_ndesc) \ | |
44 | ((_ndesc) - 1 /* head must not step on tail */ - \ | |
45 | (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ | |
46 | 1 /* Rx error */ - 1 /* flush */) | |
47 | ||
48 | struct sfc_ef10_rx_sw_desc { | |
49 | struct rte_mbuf *mbuf; | |
50 | }; | |
51 | ||
52 | struct sfc_ef10_rxq { | |
53 | /* Used on data path */ | |
54 | unsigned int flags; | |
55 | #define SFC_EF10_RXQ_STARTED 0x1 | |
56 | #define SFC_EF10_RXQ_NOT_RUNNING 0x2 | |
57 | #define SFC_EF10_RXQ_EXCEPTION 0x4 | |
58 | #define SFC_EF10_RXQ_RSS_HASH 0x8 | |
59 | unsigned int ptr_mask; | |
9f95a23c | 60 | unsigned int pending; |
11fdf7f2 TL |
61 | unsigned int completed; |
62 | unsigned int evq_read_ptr; | |
63 | efx_qword_t *evq_hw_ring; | |
64 | struct sfc_ef10_rx_sw_desc *sw_ring; | |
65 | uint64_t rearm_data; | |
9f95a23c | 66 | struct rte_mbuf *scatter_pkt; |
11fdf7f2 TL |
67 | uint16_t prefix_size; |
68 | ||
69 | /* Used on refill */ | |
70 | uint16_t buf_size; | |
71 | unsigned int added; | |
72 | unsigned int max_fill_level; | |
73 | unsigned int refill_threshold; | |
74 | struct rte_mempool *refill_mb_pool; | |
75 | efx_qword_t *rxq_hw_ring; | |
76 | volatile void *doorbell; | |
77 | ||
78 | /* Datapath receive queue anchor */ | |
79 | struct sfc_dp_rxq dp; | |
80 | }; | |
81 | ||
82 | static inline struct sfc_ef10_rxq * | |
83 | sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq) | |
84 | { | |
85 | return container_of(dp_rxq, struct sfc_ef10_rxq, dp); | |
86 | } | |
87 | ||
88 | static void | |
89 | sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq) | |
90 | { | |
91 | const unsigned int ptr_mask = rxq->ptr_mask; | |
92 | const uint32_t buf_size = rxq->buf_size; | |
93 | unsigned int free_space; | |
94 | unsigned int bulks; | |
95 | void *objs[SFC_RX_REFILL_BULK]; | |
96 | unsigned int added = rxq->added; | |
97 | ||
98 | RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0); | |
99 | ||
100 | free_space = rxq->max_fill_level - (added - rxq->completed); | |
101 | ||
102 | if (free_space < rxq->refill_threshold) | |
103 | return; | |
104 | ||
105 | bulks = free_space / RTE_DIM(objs); | |
106 | /* refill_threshold guarantees that bulks is positive */ | |
107 | SFC_ASSERT(bulks > 0); | |
108 | ||
109 | do { | |
110 | unsigned int id; | |
111 | unsigned int i; | |
112 | ||
113 | if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs, | |
114 | RTE_DIM(objs)) < 0)) { | |
115 | struct rte_eth_dev_data *dev_data = | |
116 | rte_eth_devices[rxq->dp.dpq.port_id].data; | |
117 | ||
118 | /* | |
119 | * It is hardly a safe way to increment counter | |
120 | * from different contexts, but all PMDs do it. | |
121 | */ | |
122 | dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs); | |
123 | /* Return if we have posted nothing yet */ | |
124 | if (added == rxq->added) | |
125 | return; | |
126 | /* Push posted */ | |
127 | break; | |
128 | } | |
129 | ||
130 | for (i = 0, id = added & ptr_mask; | |
131 | i < RTE_DIM(objs); | |
132 | ++i, ++id) { | |
133 | struct rte_mbuf *m = objs[i]; | |
134 | struct sfc_ef10_rx_sw_desc *rxd; | |
135 | rte_iova_t phys_addr; | |
136 | ||
9f95a23c TL |
137 | MBUF_RAW_ALLOC_CHECK(m); |
138 | ||
11fdf7f2 TL |
139 | SFC_ASSERT((id & ~ptr_mask) == 0); |
140 | rxd = &rxq->sw_ring[id]; | |
141 | rxd->mbuf = m; | |
142 | ||
143 | /* | |
144 | * Avoid writing to mbuf. It is cheaper to do it | |
145 | * when we receive packet and fill in nearby | |
146 | * structure members. | |
147 | */ | |
148 | ||
149 | phys_addr = rte_mbuf_data_iova_default(m); | |
150 | EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id], | |
151 | ESF_DZ_RX_KER_BYTE_CNT, buf_size, | |
152 | ESF_DZ_RX_KER_BUF_ADDR, phys_addr); | |
153 | } | |
154 | ||
155 | added += RTE_DIM(objs); | |
156 | } while (--bulks > 0); | |
157 | ||
158 | SFC_ASSERT(rxq->added != added); | |
159 | rxq->added = added; | |
160 | sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask); | |
161 | } | |
162 | ||
163 | static void | |
164 | sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id) | |
165 | { | |
166 | struct rte_mbuf *next_mbuf; | |
167 | ||
168 | /* Prefetch next bunch of software descriptors */ | |
169 | if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0) | |
170 | rte_prefetch0(&rxq->sw_ring[next_id]); | |
171 | ||
172 | /* | |
173 | * It looks strange to prefetch depending on previous prefetch | |
174 | * data, but measurements show that it is really efficient and | |
175 | * increases packet rate. | |
176 | */ | |
177 | next_mbuf = rxq->sw_ring[next_id].mbuf; | |
178 | if (likely(next_mbuf != NULL)) { | |
179 | /* Prefetch the next mbuf structure */ | |
180 | rte_mbuf_prefetch_part1(next_mbuf); | |
181 | ||
182 | /* Prefetch pseudo header of the next packet */ | |
183 | /* data_off is not filled in yet */ | |
184 | /* Yes, data could be not ready yet, but we hope */ | |
185 | rte_prefetch0((uint8_t *)next_mbuf->buf_addr + | |
186 | RTE_PKTMBUF_HEADROOM); | |
187 | } | |
188 | } | |
189 | ||
9f95a23c TL |
190 | static struct rte_mbuf ** |
191 | sfc_ef10_rx_pending(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts, | |
192 | uint16_t nb_pkts) | |
11fdf7f2 | 193 | { |
9f95a23c | 194 | uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending - rxq->completed); |
11fdf7f2 | 195 | |
9f95a23c | 196 | SFC_ASSERT(rxq->pending == rxq->completed || rxq->scatter_pkt == NULL); |
11fdf7f2 | 197 | |
9f95a23c TL |
198 | if (n_rx_pkts != 0) { |
199 | unsigned int completed = rxq->completed; | |
200 | ||
201 | rxq->completed = completed + n_rx_pkts; | |
202 | ||
203 | do { | |
204 | *rx_pkts++ = | |
205 | rxq->sw_ring[completed++ & rxq->ptr_mask].mbuf; | |
206 | } while (completed != rxq->completed); | |
207 | } | |
11fdf7f2 | 208 | |
9f95a23c | 209 | return rx_pkts; |
11fdf7f2 TL |
210 | } |
211 | ||
212 | static uint16_t | |
213 | sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr) | |
214 | { | |
215 | return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]); | |
216 | } | |
217 | ||
218 | static uint32_t | |
219 | sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr) | |
220 | { | |
221 | return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr); | |
222 | } | |
223 | ||
9f95a23c | 224 | static struct rte_mbuf ** |
11fdf7f2 | 225 | sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev, |
9f95a23c TL |
226 | struct rte_mbuf **rx_pkts, |
227 | struct rte_mbuf ** const rx_pkts_end) | |
11fdf7f2 TL |
228 | { |
229 | const unsigned int ptr_mask = rxq->ptr_mask; | |
9f95a23c | 230 | unsigned int pending = rxq->pending; |
11fdf7f2 TL |
231 | unsigned int ready; |
232 | struct sfc_ef10_rx_sw_desc *rxd; | |
233 | struct rte_mbuf *m; | |
234 | struct rte_mbuf *m0; | |
11fdf7f2 | 235 | const uint8_t *pseudo_hdr; |
9f95a23c | 236 | uint16_t seg_len; |
11fdf7f2 | 237 | |
9f95a23c | 238 | ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - pending) & |
11fdf7f2 | 239 | EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); |
9f95a23c TL |
240 | |
241 | if (ready == 0) { | |
242 | /* Rx abort - it was no enough descriptors for Rx packet */ | |
243 | rte_pktmbuf_free(rxq->scatter_pkt); | |
244 | rxq->scatter_pkt = NULL; | |
245 | return rx_pkts; | |
246 | } | |
247 | ||
248 | rxq->pending = pending + ready; | |
11fdf7f2 TL |
249 | |
250 | if (rx_ev.eq_u64[0] & | |
251 | rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) | | |
252 | (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) { | |
9f95a23c TL |
253 | SFC_ASSERT(rxq->completed == pending); |
254 | do { | |
255 | rxd = &rxq->sw_ring[pending++ & ptr_mask]; | |
256 | rte_mbuf_raw_free(rxd->mbuf); | |
257 | } while (pending != rxq->pending); | |
258 | rxq->completed = pending; | |
259 | return rx_pkts; | |
11fdf7f2 TL |
260 | } |
261 | ||
9f95a23c TL |
262 | /* If scattered packet is in progress */ |
263 | if (rxq->scatter_pkt != NULL) { | |
264 | /* Events for scattered packet frags are not merged */ | |
265 | SFC_ASSERT(ready == 1); | |
266 | SFC_ASSERT(rxq->completed == pending); | |
11fdf7f2 | 267 | |
9f95a23c TL |
268 | /* There is no pseudo-header in scatter segments. */ |
269 | seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES); | |
11fdf7f2 | 270 | |
9f95a23c TL |
271 | rxd = &rxq->sw_ring[pending++ & ptr_mask]; |
272 | m = rxd->mbuf; | |
11fdf7f2 | 273 | |
9f95a23c TL |
274 | MBUF_RAW_ALLOC_CHECK(m); |
275 | ||
276 | m->data_off = RTE_PKTMBUF_HEADROOM; | |
277 | rte_pktmbuf_data_len(m) = seg_len; | |
278 | rte_pktmbuf_pkt_len(m) = seg_len; | |
11fdf7f2 | 279 | |
9f95a23c TL |
280 | rxq->scatter_pkt->nb_segs++; |
281 | rte_pktmbuf_pkt_len(rxq->scatter_pkt) += seg_len; | |
282 | rte_pktmbuf_lastseg(rxq->scatter_pkt)->next = m; | |
283 | ||
284 | if (~rx_ev.eq_u64[0] & | |
285 | rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { | |
286 | *rx_pkts++ = rxq->scatter_pkt; | |
287 | rxq->scatter_pkt = NULL; | |
288 | } | |
289 | rxq->completed = pending; | |
290 | return rx_pkts; | |
291 | } | |
292 | ||
293 | rxd = &rxq->sw_ring[pending++ & ptr_mask]; | |
294 | ||
295 | sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); | |
296 | ||
297 | m = rxd->mbuf; | |
11fdf7f2 TL |
298 | |
299 | RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data)); | |
300 | m->rearm_data[0] = rxq->rearm_data; | |
301 | ||
302 | /* Classify packet based on Rx event */ | |
303 | /* Mask RSS hash offload flag if RSS is not enabled */ | |
304 | sfc_ef10_rx_ev_to_offloads(rx_ev, m, | |
305 | (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ? | |
306 | ~0ull : ~PKT_RX_RSS_HASH); | |
307 | ||
308 | /* data_off already moved past pseudo header */ | |
309 | pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; | |
310 | ||
311 | /* | |
312 | * Always get RSS hash from pseudo header to avoid | |
313 | * condition/branching. If it is valid or not depends on | |
314 | * PKT_RX_RSS_HASH in m->ol_flags. | |
315 | */ | |
316 | m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); | |
317 | ||
318 | if (ready == 1) | |
9f95a23c | 319 | seg_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) - |
11fdf7f2 TL |
320 | rxq->prefix_size; |
321 | else | |
9f95a23c TL |
322 | seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); |
323 | SFC_ASSERT(seg_len > 0); | |
324 | rte_pktmbuf_data_len(m) = seg_len; | |
325 | rte_pktmbuf_pkt_len(m) = seg_len; | |
11fdf7f2 TL |
326 | |
327 | SFC_ASSERT(m->next == NULL); | |
328 | ||
9f95a23c TL |
329 | if (~rx_ev.eq_u64[0] & rte_cpu_to_le_64(1ull << ESF_DZ_RX_CONT_LBN)) { |
330 | *rx_pkts++ = m; | |
331 | rxq->completed = pending; | |
332 | } else { | |
333 | /* Events with CONT bit are not merged */ | |
334 | SFC_ASSERT(ready == 1); | |
335 | rxq->scatter_pkt = m; | |
336 | rxq->completed = pending; | |
337 | return rx_pkts; | |
338 | } | |
339 | ||
11fdf7f2 TL |
340 | /* Remember mbuf to copy offload flags and packet type from */ |
341 | m0 = m; | |
9f95a23c TL |
342 | while (pending != rxq->pending) { |
343 | rxd = &rxq->sw_ring[pending++ & ptr_mask]; | |
11fdf7f2 | 344 | |
9f95a23c | 345 | sfc_ef10_rx_prefetch_next(rxq, pending & ptr_mask); |
11fdf7f2 TL |
346 | |
347 | m = rxd->mbuf; | |
348 | ||
9f95a23c | 349 | if (rx_pkts != rx_pkts_end) { |
11fdf7f2 | 350 | *rx_pkts++ = m; |
9f95a23c TL |
351 | rxq->completed = pending; |
352 | } | |
11fdf7f2 TL |
353 | |
354 | RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != | |
355 | sizeof(rxq->rearm_data)); | |
356 | m->rearm_data[0] = rxq->rearm_data; | |
357 | ||
358 | /* Event-dependent information is the same */ | |
359 | m->ol_flags = m0->ol_flags; | |
360 | m->packet_type = m0->packet_type; | |
361 | ||
362 | /* data_off already moved past pseudo header */ | |
363 | pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM; | |
364 | ||
365 | /* | |
366 | * Always get RSS hash from pseudo header to avoid | |
367 | * condition/branching. If it is valid or not depends on | |
368 | * PKT_RX_RSS_HASH in m->ol_flags. | |
369 | */ | |
370 | m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr); | |
371 | ||
9f95a23c TL |
372 | seg_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr); |
373 | SFC_ASSERT(seg_len > 0); | |
374 | rte_pktmbuf_data_len(m) = seg_len; | |
375 | rte_pktmbuf_pkt_len(m) = seg_len; | |
11fdf7f2 TL |
376 | |
377 | SFC_ASSERT(m->next == NULL); | |
378 | } | |
379 | ||
9f95a23c | 380 | return rx_pkts; |
11fdf7f2 TL |
381 | } |
382 | ||
383 | static bool | |
384 | sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev) | |
385 | { | |
386 | *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask]; | |
387 | ||
388 | if (!sfc_ef10_ev_present(*rx_ev)) | |
389 | return false; | |
390 | ||
391 | if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) != | |
392 | FSE_AZ_EV_CODE_RX_EV)) { | |
393 | /* | |
394 | * Do not move read_ptr to keep the event for exception | |
395 | * handling by the control path. | |
396 | */ | |
397 | rxq->flags |= SFC_EF10_RXQ_EXCEPTION; | |
398 | sfc_ef10_rx_err(&rxq->dp.dpq, | |
399 | "RxQ exception at EvQ read ptr %#x", | |
400 | rxq->evq_read_ptr); | |
401 | return false; | |
402 | } | |
403 | ||
404 | rxq->evq_read_ptr++; | |
405 | return true; | |
406 | } | |
407 | ||
408 | static uint16_t | |
409 | sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) | |
410 | { | |
411 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue); | |
9f95a23c | 412 | struct rte_mbuf ** const rx_pkts_end = &rx_pkts[nb_pkts]; |
11fdf7f2 | 413 | unsigned int evq_old_read_ptr; |
11fdf7f2 TL |
414 | efx_qword_t rx_ev; |
415 | ||
9f95a23c TL |
416 | rx_pkts = sfc_ef10_rx_pending(rxq, rx_pkts, nb_pkts); |
417 | ||
11fdf7f2 TL |
418 | if (unlikely(rxq->flags & |
419 | (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) | |
9f95a23c | 420 | goto done; |
11fdf7f2 TL |
421 | |
422 | evq_old_read_ptr = rxq->evq_read_ptr; | |
9f95a23c | 423 | while (rx_pkts != rx_pkts_end && sfc_ef10_rx_get_event(rxq, &rx_ev)) { |
11fdf7f2 TL |
424 | /* |
425 | * DROP_EVENT is an internal to the NIC, software should | |
426 | * never see it and, therefore, may ignore it. | |
427 | */ | |
428 | ||
9f95a23c TL |
429 | rx_pkts = sfc_ef10_rx_process_event(rxq, rx_ev, |
430 | rx_pkts, rx_pkts_end); | |
11fdf7f2 TL |
431 | } |
432 | ||
433 | sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr, | |
434 | rxq->evq_read_ptr); | |
435 | ||
436 | /* It is not a problem if we refill in the case of exception */ | |
437 | sfc_ef10_rx_qrefill(rxq); | |
438 | ||
9f95a23c TL |
439 | done: |
440 | return nb_pkts - (rx_pkts_end - rx_pkts); | |
11fdf7f2 TL |
441 | } |
442 | ||
443 | const uint32_t * | |
444 | sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps) | |
445 | { | |
446 | static const uint32_t ef10_native_ptypes[] = { | |
447 | RTE_PTYPE_L2_ETHER, | |
448 | RTE_PTYPE_L2_ETHER_ARP, | |
449 | RTE_PTYPE_L2_ETHER_VLAN, | |
450 | RTE_PTYPE_L2_ETHER_QINQ, | |
451 | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, | |
452 | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, | |
453 | RTE_PTYPE_L4_FRAG, | |
454 | RTE_PTYPE_L4_TCP, | |
455 | RTE_PTYPE_L4_UDP, | |
456 | RTE_PTYPE_UNKNOWN | |
457 | }; | |
458 | static const uint32_t ef10_overlay_ptypes[] = { | |
459 | RTE_PTYPE_L2_ETHER, | |
460 | RTE_PTYPE_L2_ETHER_ARP, | |
461 | RTE_PTYPE_L2_ETHER_VLAN, | |
462 | RTE_PTYPE_L2_ETHER_QINQ, | |
463 | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, | |
464 | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, | |
465 | RTE_PTYPE_L4_FRAG, | |
466 | RTE_PTYPE_L4_TCP, | |
467 | RTE_PTYPE_L4_UDP, | |
468 | RTE_PTYPE_TUNNEL_VXLAN, | |
469 | RTE_PTYPE_TUNNEL_NVGRE, | |
470 | RTE_PTYPE_INNER_L2_ETHER, | |
471 | RTE_PTYPE_INNER_L2_ETHER_VLAN, | |
472 | RTE_PTYPE_INNER_L2_ETHER_QINQ, | |
473 | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, | |
474 | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, | |
475 | RTE_PTYPE_INNER_L4_FRAG, | |
476 | RTE_PTYPE_INNER_L4_TCP, | |
477 | RTE_PTYPE_INNER_L4_UDP, | |
478 | RTE_PTYPE_UNKNOWN | |
479 | }; | |
480 | ||
481 | /* | |
482 | * The function returns static set of supported packet types, | |
483 | * so we can't build it dynamically based on supported tunnel | |
484 | * encapsulations and should limit to known sets. | |
485 | */ | |
486 | switch (tunnel_encaps) { | |
487 | case (1u << EFX_TUNNEL_PROTOCOL_VXLAN | | |
488 | 1u << EFX_TUNNEL_PROTOCOL_GENEVE | | |
489 | 1u << EFX_TUNNEL_PROTOCOL_NVGRE): | |
490 | return ef10_overlay_ptypes; | |
491 | default: | |
492 | SFC_GENERIC_LOG(ERR, | |
493 | "Unexpected set of supported tunnel encapsulations: %#x", | |
494 | tunnel_encaps); | |
495 | /* FALLTHROUGH */ | |
496 | case 0: | |
497 | return ef10_native_ptypes; | |
498 | } | |
499 | } | |
500 | ||
501 | static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending; | |
502 | static unsigned int | |
9f95a23c | 503 | sfc_ef10_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq) |
11fdf7f2 | 504 | { |
9f95a23c TL |
505 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); |
506 | efx_qword_t rx_ev; | |
507 | const unsigned int evq_old_read_ptr = rxq->evq_read_ptr; | |
508 | unsigned int pending = rxq->pending; | |
509 | unsigned int ready; | |
510 | ||
511 | if (unlikely(rxq->flags & | |
512 | (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION))) | |
513 | goto done; | |
514 | ||
515 | while (sfc_ef10_rx_get_event(rxq, &rx_ev)) { | |
516 | ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - | |
517 | pending) & | |
518 | EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); | |
519 | pending += ready; | |
520 | } | |
521 | ||
11fdf7f2 | 522 | /* |
9f95a23c TL |
523 | * The function does not process events, so return event queue read |
524 | * pointer to the original position to allow the events that were | |
525 | * read to be processed later | |
11fdf7f2 | 526 | */ |
9f95a23c TL |
527 | rxq->evq_read_ptr = evq_old_read_ptr; |
528 | ||
529 | done: | |
530 | return pending - rxq->completed; | |
11fdf7f2 TL |
531 | } |
532 | ||
533 | static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status; | |
534 | static int | |
9f95a23c | 535 | sfc_ef10_rx_qdesc_status(struct sfc_dp_rxq *dp_rxq, uint16_t offset) |
11fdf7f2 | 536 | { |
9f95a23c TL |
537 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); |
538 | unsigned int npending = sfc_ef10_rx_qdesc_npending(dp_rxq); | |
539 | ||
540 | if (unlikely(offset > rxq->ptr_mask)) | |
541 | return -EINVAL; | |
542 | ||
543 | if (offset < npending) | |
544 | return RTE_ETH_RX_DESC_DONE; | |
545 | ||
546 | if (offset < (rxq->added - rxq->completed)) | |
547 | return RTE_ETH_RX_DESC_AVAIL; | |
548 | ||
549 | return RTE_ETH_RX_DESC_UNAVAIL; | |
11fdf7f2 TL |
550 | } |
551 | ||
552 | ||
553 | static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info; | |
554 | static void | |
555 | sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info) | |
556 | { | |
557 | /* | |
558 | * Number of descriptors just defines maximum number of pushed | |
559 | * descriptors (fill level). | |
560 | */ | |
561 | dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK; | |
562 | dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK; | |
563 | } | |
564 | ||
565 | ||
566 | static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings; | |
567 | static int | |
568 | sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc, | |
9f95a23c | 569 | struct sfc_dp_rx_hw_limits *limits, |
11fdf7f2 TL |
570 | __rte_unused struct rte_mempool *mb_pool, |
571 | unsigned int *rxq_entries, | |
572 | unsigned int *evq_entries, | |
573 | unsigned int *rxq_max_fill_level) | |
574 | { | |
575 | /* | |
576 | * rte_ethdev API guarantees that the number meets min, max and | |
577 | * alignment requirements. | |
578 | */ | |
9f95a23c TL |
579 | if (nb_rx_desc <= limits->rxq_min_entries) |
580 | *rxq_entries = limits->rxq_min_entries; | |
11fdf7f2 TL |
581 | else |
582 | *rxq_entries = rte_align32pow2(nb_rx_desc); | |
583 | ||
584 | *evq_entries = *rxq_entries; | |
585 | ||
586 | *rxq_max_fill_level = RTE_MIN(nb_rx_desc, | |
587 | SFC_EF10_RXQ_LIMIT(*evq_entries)); | |
588 | return 0; | |
589 | } | |
590 | ||
591 | ||
592 | static uint64_t | |
593 | sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size) | |
594 | { | |
595 | struct rte_mbuf m; | |
596 | ||
597 | memset(&m, 0, sizeof(m)); | |
598 | ||
599 | rte_mbuf_refcnt_set(&m, 1); | |
600 | m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size; | |
601 | m.nb_segs = 1; | |
602 | m.port = port_id; | |
603 | ||
604 | /* rearm_data covers structure members filled in above */ | |
605 | rte_compiler_barrier(); | |
606 | RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t)); | |
607 | return m.rearm_data[0]; | |
608 | } | |
609 | ||
610 | static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate; | |
611 | static int | |
612 | sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id, | |
613 | const struct rte_pci_addr *pci_addr, int socket_id, | |
614 | const struct sfc_dp_rx_qcreate_info *info, | |
615 | struct sfc_dp_rxq **dp_rxqp) | |
616 | { | |
617 | struct sfc_ef10_rxq *rxq; | |
618 | int rc; | |
619 | ||
620 | rc = EINVAL; | |
621 | if (info->rxq_entries != info->evq_entries) | |
622 | goto fail_rxq_args; | |
623 | ||
624 | rc = ENOMEM; | |
625 | rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq), | |
626 | RTE_CACHE_LINE_SIZE, socket_id); | |
627 | if (rxq == NULL) | |
628 | goto fail_rxq_alloc; | |
629 | ||
630 | sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr); | |
631 | ||
632 | rc = ENOMEM; | |
633 | rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring", | |
634 | info->rxq_entries, | |
635 | sizeof(*rxq->sw_ring), | |
636 | RTE_CACHE_LINE_SIZE, socket_id); | |
637 | if (rxq->sw_ring == NULL) | |
638 | goto fail_desc_alloc; | |
639 | ||
640 | rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; | |
641 | if (info->flags & SFC_RXQ_FLAG_RSS_HASH) | |
642 | rxq->flags |= SFC_EF10_RXQ_RSS_HASH; | |
643 | rxq->ptr_mask = info->rxq_entries - 1; | |
644 | rxq->evq_hw_ring = info->evq_hw_ring; | |
645 | rxq->max_fill_level = info->max_fill_level; | |
646 | rxq->refill_threshold = info->refill_threshold; | |
647 | rxq->rearm_data = | |
648 | sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size); | |
649 | rxq->prefix_size = info->prefix_size; | |
650 | rxq->buf_size = info->buf_size; | |
651 | rxq->refill_mb_pool = info->refill_mb_pool; | |
652 | rxq->rxq_hw_ring = info->rxq_hw_ring; | |
653 | rxq->doorbell = (volatile uint8_t *)info->mem_bar + | |
654 | ER_DZ_RX_DESC_UPD_REG_OFST + | |
655 | (info->hw_index << info->vi_window_shift); | |
656 | ||
657 | *dp_rxqp = &rxq->dp; | |
658 | return 0; | |
659 | ||
660 | fail_desc_alloc: | |
661 | rte_free(rxq); | |
662 | ||
663 | fail_rxq_alloc: | |
664 | fail_rxq_args: | |
665 | return rc; | |
666 | } | |
667 | ||
668 | static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy; | |
669 | static void | |
670 | sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq) | |
671 | { | |
672 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); | |
673 | ||
674 | rte_free(rxq->sw_ring); | |
675 | rte_free(rxq); | |
676 | } | |
677 | ||
678 | static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart; | |
679 | static int | |
680 | sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr) | |
681 | { | |
682 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); | |
683 | ||
9f95a23c TL |
684 | SFC_ASSERT(rxq->completed == 0); |
685 | SFC_ASSERT(rxq->pending == 0); | |
686 | SFC_ASSERT(rxq->added == 0); | |
11fdf7f2 TL |
687 | |
688 | sfc_ef10_rx_qrefill(rxq); | |
689 | ||
690 | rxq->evq_read_ptr = evq_read_ptr; | |
691 | ||
692 | rxq->flags |= SFC_EF10_RXQ_STARTED; | |
693 | rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION); | |
694 | ||
695 | return 0; | |
696 | } | |
697 | ||
698 | static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop; | |
699 | static void | |
700 | sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr) | |
701 | { | |
702 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); | |
703 | ||
704 | rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING; | |
705 | ||
706 | *evq_read_ptr = rxq->evq_read_ptr; | |
707 | } | |
708 | ||
709 | static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev; | |
710 | static bool | |
711 | sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id) | |
712 | { | |
713 | __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); | |
714 | ||
715 | SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING); | |
716 | ||
717 | /* | |
718 | * It is safe to ignore Rx event since we free all mbufs on | |
719 | * queue purge anyway. | |
720 | */ | |
721 | ||
722 | return false; | |
723 | } | |
724 | ||
725 | static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge; | |
726 | static void | |
727 | sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq) | |
728 | { | |
729 | struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq); | |
730 | unsigned int i; | |
731 | struct sfc_ef10_rx_sw_desc *rxd; | |
732 | ||
9f95a23c TL |
733 | rte_pktmbuf_free(rxq->scatter_pkt); |
734 | rxq->scatter_pkt = NULL; | |
735 | ||
11fdf7f2 TL |
736 | for (i = rxq->completed; i != rxq->added; ++i) { |
737 | rxd = &rxq->sw_ring[i & rxq->ptr_mask]; | |
9f95a23c | 738 | rte_mbuf_raw_free(rxd->mbuf); |
11fdf7f2 TL |
739 | rxd->mbuf = NULL; |
740 | } | |
741 | ||
9f95a23c TL |
742 | rxq->completed = rxq->pending = rxq->added = 0; |
743 | ||
11fdf7f2 TL |
744 | rxq->flags &= ~SFC_EF10_RXQ_STARTED; |
745 | } | |
746 | ||
747 | struct sfc_dp_rx sfc_ef10_rx = { | |
748 | .dp = { | |
749 | .name = SFC_KVARG_DATAPATH_EF10, | |
750 | .type = SFC_DP_RX, | |
751 | .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, | |
752 | }, | |
9f95a23c TL |
753 | .features = SFC_DP_RX_FEAT_SCATTER | |
754 | SFC_DP_RX_FEAT_MULTI_PROCESS | | |
11fdf7f2 TL |
755 | SFC_DP_RX_FEAT_TUNNELS | |
756 | SFC_DP_RX_FEAT_CHECKSUM, | |
757 | .get_dev_info = sfc_ef10_rx_get_dev_info, | |
758 | .qsize_up_rings = sfc_ef10_rx_qsize_up_rings, | |
759 | .qcreate = sfc_ef10_rx_qcreate, | |
760 | .qdestroy = sfc_ef10_rx_qdestroy, | |
761 | .qstart = sfc_ef10_rx_qstart, | |
762 | .qstop = sfc_ef10_rx_qstop, | |
763 | .qrx_ev = sfc_ef10_rx_qrx_ev, | |
764 | .qpurge = sfc_ef10_rx_qpurge, | |
765 | .supported_ptypes_get = sfc_ef10_supported_ptypes_get, | |
766 | .qdesc_npending = sfc_ef10_rx_qdesc_npending, | |
767 | .qdesc_status = sfc_ef10_rx_qdesc_status, | |
768 | .pkt_burst = sfc_ef10_recv_pkts, | |
769 | }; |