]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2016-2017 Intel Corporation | |
11fdf7f2 TL |
3 | */ |
4 | ||
5 | #ifndef _SW_EVDEV_H_ | |
6 | #define _SW_EVDEV_H_ | |
7 | ||
9f95a23c | 8 | #include "sw_evdev_log.h" |
11fdf7f2 | 9 | #include <rte_eventdev.h> |
9f95a23c | 10 | #include <rte_eventdev_pmd_vdev.h> |
11fdf7f2 TL |
11 | #include <rte_atomic.h> |
12 | ||
13 | #define SW_DEFAULT_CREDIT_QUANTA 32 | |
14 | #define SW_DEFAULT_SCHED_QUANTA 128 | |
15 | #define SW_QID_NUM_FIDS 16384 | |
16 | #define SW_IQS_MAX 4 | |
17 | #define SW_Q_PRIORITY_MAX 255 | |
18 | #define SW_PORTS_MAX 64 | |
19 | #define MAX_SW_CONS_Q_DEPTH 128 | |
20 | #define SW_INFLIGHT_EVENTS_TOTAL 4096 | |
21 | /* allow for lots of over-provisioning */ | |
22 | #define MAX_SW_PROD_Q_DEPTH 4096 | |
23 | #define SW_FRAGMENTS_MAX 16 | |
24 | ||
9f95a23c TL |
25 | /* Should be power-of-two minus one, to leave room for the next pointer */ |
26 | #define SW_EVS_PER_Q_CHUNK 255 | |
27 | #define SW_Q_CHUNK_SIZE ((SW_EVS_PER_Q_CHUNK + 1) * sizeof(struct rte_event)) | |
28 | ||
11fdf7f2 TL |
29 | /* report dequeue burst sizes in buckets */ |
30 | #define SW_DEQ_STAT_BUCKET_SHIFT 2 | |
31 | /* how many packets pulled from port by sched */ | |
32 | #define SCHED_DEQUEUE_BURST_SIZE 32 | |
33 | ||
34 | #define SW_PORT_HIST_LIST (MAX_SW_PROD_Q_DEPTH) /* size of our history list */ | |
35 | #define NUM_SAMPLES 64 /* how many data points use for average stats */ | |
36 | ||
37 | #define EVENTDEV_NAME_SW_PMD event_sw | |
38 | #define SW_PMD_NAME RTE_STR(event_sw) | |
9f95a23c | 39 | #define SW_PMD_NAME_MAX 64 |
11fdf7f2 TL |
40 | |
41 | #define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1) | |
42 | ||
43 | #define SW_NUM_POLL_BUCKETS (MAX_SW_CONS_Q_DEPTH >> SW_DEQ_STAT_BUCKET_SHIFT) | |
44 | ||
45 | enum { | |
46 | QE_FLAG_VALID_SHIFT = 0, | |
47 | QE_FLAG_COMPLETE_SHIFT, | |
48 | QE_FLAG_NOT_EOP_SHIFT, | |
49 | _QE_FLAG_COUNT | |
50 | }; | |
51 | ||
52 | #define QE_FLAG_VALID (1 << QE_FLAG_VALID_SHIFT) /* for NEW FWD, FRAG */ | |
53 | #define QE_FLAG_COMPLETE (1 << QE_FLAG_COMPLETE_SHIFT) /* set for FWD, DROP */ | |
54 | #define QE_FLAG_NOT_EOP (1 << QE_FLAG_NOT_EOP_SHIFT) /* set for FRAG only */ | |
55 | ||
56 | static const uint8_t sw_qe_flag_map[] = { | |
57 | QE_FLAG_VALID /* NEW Event */, | |
58 | QE_FLAG_VALID | QE_FLAG_COMPLETE /* FWD Event */, | |
59 | QE_FLAG_COMPLETE /* RELEASE Event */, | |
60 | ||
61 | /* Values which can be used for future support for partial | |
62 | * events, i.e. where one event comes back to the scheduler | |
63 | * as multiple which need to be tracked together | |
64 | */ | |
65 | QE_FLAG_VALID | QE_FLAG_COMPLETE | QE_FLAG_NOT_EOP, | |
66 | }; | |
67 | ||
11fdf7f2 TL |
68 | /* Records basic event stats at a given point. Used in port and qid structs */ |
69 | struct sw_point_stats { | |
70 | uint64_t rx_pkts; | |
71 | uint64_t rx_dropped; | |
72 | uint64_t tx_pkts; | |
73 | }; | |
74 | ||
75 | /* structure used to track what port a flow (FID) is pinned to */ | |
76 | struct sw_fid_t { | |
77 | /* which CQ this FID is currently pinned to */ | |
78 | int32_t cq; | |
79 | /* number of packets gone to the CQ with this FID */ | |
80 | uint32_t pcount; | |
81 | }; | |
82 | ||
83 | struct reorder_buffer_entry { | |
84 | uint16_t num_fragments; /**< Number of packet fragments */ | |
85 | uint16_t fragment_index; /**< Points to the oldest valid frag */ | |
86 | uint8_t ready; /**< Entry is ready to be reordered */ | |
87 | struct rte_event fragments[SW_FRAGMENTS_MAX]; | |
88 | }; | |
89 | ||
9f95a23c TL |
90 | struct sw_iq { |
91 | struct sw_queue_chunk *head; | |
92 | struct sw_queue_chunk *tail; | |
93 | uint16_t head_idx; | |
94 | uint16_t tail_idx; | |
95 | uint16_t count; | |
96 | }; | |
97 | ||
11fdf7f2 TL |
98 | struct sw_qid { |
99 | /* set when the QID has been initialized */ | |
100 | uint8_t initialized; | |
101 | /* The type of this QID */ | |
102 | int8_t type; | |
103 | /* Integer ID representing the queue. This is used in history lists, | |
104 | * to identify the stage of processing. | |
105 | */ | |
106 | uint32_t id; | |
107 | struct sw_point_stats stats; | |
108 | ||
109 | /* Internal priority rings for packets */ | |
9f95a23c | 110 | struct sw_iq iq[SW_IQS_MAX]; |
11fdf7f2 TL |
111 | uint32_t iq_pkt_mask; /* A mask to indicate packets in an IQ */ |
112 | uint64_t iq_pkt_count[SW_IQS_MAX]; | |
113 | ||
114 | /* Information on what CQs are polling this IQ */ | |
115 | uint32_t cq_num_mapped_cqs; | |
116 | uint32_t cq_next_tx; /* cq to write next (non-atomic) packet */ | |
117 | uint32_t cq_map[SW_PORTS_MAX]; | |
9f95a23c | 118 | uint64_t to_port[SW_PORTS_MAX]; |
11fdf7f2 TL |
119 | |
120 | /* Track flow ids for atomic load balancing */ | |
121 | struct sw_fid_t fids[SW_QID_NUM_FIDS]; | |
122 | ||
123 | /* Track packet order for reordering when needed */ | |
124 | struct reorder_buffer_entry *reorder_buffer; /*< pkts await reorder */ | |
125 | struct rte_ring *reorder_buffer_freelist; /* available reorder slots */ | |
126 | uint32_t reorder_buffer_index; /* oldest valid reorder buffer entry */ | |
127 | uint32_t window_size; /* Used to wrap reorder_buffer_index */ | |
128 | ||
129 | uint8_t priority; | |
130 | }; | |
131 | ||
132 | struct sw_hist_list_entry { | |
133 | int32_t qid; | |
134 | int32_t fid; | |
135 | struct reorder_buffer_entry *rob_entry; | |
136 | }; | |
137 | ||
138 | struct sw_evdev; | |
139 | ||
140 | struct sw_port { | |
141 | /* new enqueue / dequeue API doesn't have an instance pointer, only the | |
142 | * pointer to the port being enqueue/dequeued from | |
143 | */ | |
144 | struct sw_evdev *sw; | |
145 | ||
146 | /* set when the port is initialized */ | |
147 | uint8_t initialized; | |
148 | /* A numeric ID for the port */ | |
149 | uint8_t id; | |
150 | ||
9f95a23c TL |
151 | /* An atomic counter for when the port has been unlinked, and the |
152 | * scheduler has not yet acked this unlink - hence there may still be | |
153 | * events in the buffers going to the port. When the unlinks in | |
154 | * progress is read by the scheduler, no more events will be pushed to | |
155 | * the port - hence the scheduler core can just assign zero. | |
156 | */ | |
157 | uint8_t unlinks_in_progress; | |
158 | ||
11fdf7f2 TL |
159 | int16_t is_directed; /** Takes from a single directed QID */ |
160 | /** | |
161 | * For loadbalanced we can optimise pulling packets from | |
162 | * producers if there is no reordering involved | |
163 | */ | |
164 | int16_t num_ordered_qids; | |
165 | ||
166 | /** Ring and buffer for pulling events from workers for scheduling */ | |
9f95a23c | 167 | struct rte_event_ring *rx_worker_ring __rte_cache_aligned; |
11fdf7f2 | 168 | /** Ring and buffer for pushing packets to workers after scheduling */ |
9f95a23c | 169 | struct rte_event_ring *cq_worker_ring; |
11fdf7f2 TL |
170 | |
171 | /* hole */ | |
172 | ||
173 | /* num releases yet to be completed on this port */ | |
174 | uint16_t outstanding_releases __rte_cache_aligned; | |
175 | uint16_t inflight_max; /* app requested max inflights for this port */ | |
176 | uint16_t inflight_credits; /* num credits this port has right now */ | |
9f95a23c | 177 | uint8_t implicit_release; /* release events before dequeueing */ |
11fdf7f2 TL |
178 | |
179 | uint16_t last_dequeue_burst_sz; /* how big the burst was */ | |
180 | uint64_t last_dequeue_ticks; /* used to track burst processing time */ | |
181 | uint64_t avg_pkt_ticks; /* tracks average over NUM_SAMPLES burst */ | |
182 | uint64_t total_polls; /* how many polls were counted in stats */ | |
183 | uint64_t zero_polls; /* tracks polls returning nothing */ | |
184 | uint32_t poll_buckets[SW_NUM_POLL_BUCKETS]; | |
185 | /* bucket values in 4s for shorter reporting */ | |
186 | ||
187 | /* History list structs, containing info on pkts egressed to worker */ | |
188 | uint16_t hist_head __rte_cache_aligned; | |
189 | uint16_t hist_tail; | |
190 | uint16_t inflights; | |
191 | struct sw_hist_list_entry hist_list[SW_PORT_HIST_LIST]; | |
192 | ||
193 | /* track packets in and out of this port */ | |
194 | struct sw_point_stats stats; | |
195 | ||
196 | ||
197 | uint32_t pp_buf_start; | |
198 | uint32_t pp_buf_count; | |
199 | uint16_t cq_buf_count; | |
200 | struct rte_event pp_buf[SCHED_DEQUEUE_BURST_SIZE]; | |
201 | struct rte_event cq_buf[MAX_SW_CONS_Q_DEPTH]; | |
202 | ||
203 | uint8_t num_qids_mapped; | |
204 | }; | |
205 | ||
206 | struct sw_evdev { | |
207 | struct rte_eventdev_data *data; | |
208 | ||
209 | uint32_t port_count; | |
210 | uint32_t qid_count; | |
211 | uint32_t xstats_count; | |
212 | struct sw_xstats_entry *xstats; | |
213 | uint32_t xstats_count_mode_dev; | |
214 | uint32_t xstats_count_mode_port; | |
215 | uint32_t xstats_count_mode_queue; | |
216 | ||
217 | /* Contains all ports - load balanced and directed */ | |
218 | struct sw_port ports[SW_PORTS_MAX] __rte_cache_aligned; | |
219 | ||
220 | rte_atomic32_t inflights __rte_cache_aligned; | |
221 | ||
222 | /* | |
223 | * max events in this instance. Cached here for performance. | |
224 | * (also available in data->conf.nb_events_limit) | |
225 | */ | |
226 | uint32_t nb_events_limit; | |
227 | ||
228 | /* Internal queues - one per logical queue */ | |
229 | struct sw_qid qids[RTE_EVENT_MAX_QUEUES_PER_DEV] __rte_cache_aligned; | |
9f95a23c TL |
230 | struct sw_queue_chunk *chunk_list_head; |
231 | struct sw_queue_chunk *chunks; | |
11fdf7f2 TL |
232 | |
233 | /* Cache how many packets are in each cq */ | |
234 | uint16_t cq_ring_space[SW_PORTS_MAX] __rte_cache_aligned; | |
235 | ||
236 | /* Array of pointers to load-balanced QIDs sorted by priority level */ | |
237 | struct sw_qid *qids_prioritized[RTE_EVENT_MAX_QUEUES_PER_DEV]; | |
238 | ||
239 | /* Stats */ | |
240 | struct sw_point_stats stats __rte_cache_aligned; | |
241 | uint64_t sched_called; | |
242 | int32_t sched_quanta; | |
243 | uint64_t sched_no_iq_enqueues; | |
244 | uint64_t sched_no_cq_enqueues; | |
245 | uint64_t sched_cq_qid_called; | |
246 | ||
247 | uint8_t started; | |
248 | uint32_t credit_update_quanta; | |
249 | ||
250 | /* store num stats and offset of the stats for each port */ | |
251 | uint16_t xstats_count_per_port[SW_PORTS_MAX]; | |
252 | uint16_t xstats_offset_for_port[SW_PORTS_MAX]; | |
253 | /* store num stats and offset of the stats for each queue */ | |
254 | uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV]; | |
255 | uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV]; | |
9f95a23c TL |
256 | |
257 | uint32_t service_id; | |
258 | char service_name[SW_PMD_NAME_MAX]; | |
11fdf7f2 TL |
259 | }; |
260 | ||
261 | static inline struct sw_evdev * | |
262 | sw_pmd_priv(const struct rte_eventdev *eventdev) | |
263 | { | |
264 | return eventdev->data->dev_private; | |
265 | } | |
266 | ||
267 | static inline const struct sw_evdev * | |
268 | sw_pmd_priv_const(const struct rte_eventdev *eventdev) | |
269 | { | |
270 | return eventdev->data->dev_private; | |
271 | } | |
272 | ||
273 | uint16_t sw_event_enqueue(void *port, const struct rte_event *ev); | |
274 | uint16_t sw_event_enqueue_burst(void *port, const struct rte_event ev[], | |
275 | uint16_t num); | |
276 | ||
277 | uint16_t sw_event_dequeue(void *port, struct rte_event *ev, uint64_t wait); | |
278 | uint16_t sw_event_dequeue_burst(void *port, struct rte_event *ev, uint16_t num, | |
279 | uint64_t wait); | |
280 | void sw_event_schedule(struct rte_eventdev *dev); | |
281 | int sw_xstats_init(struct sw_evdev *dev); | |
282 | int sw_xstats_uninit(struct sw_evdev *dev); | |
283 | int sw_xstats_get_names(const struct rte_eventdev *dev, | |
284 | enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, | |
285 | struct rte_event_dev_xstats_name *xstats_names, | |
286 | unsigned int *ids, unsigned int size); | |
287 | int sw_xstats_get(const struct rte_eventdev *dev, | |
288 | enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, | |
289 | const unsigned int ids[], uint64_t values[], unsigned int n); | |
290 | uint64_t sw_xstats_get_by_name(const struct rte_eventdev *dev, | |
291 | const char *name, unsigned int *id); | |
292 | int sw_xstats_reset(struct rte_eventdev *dev, | |
293 | enum rte_event_dev_xstats_mode mode, | |
294 | int16_t queue_port_id, | |
295 | const uint32_t ids[], | |
296 | uint32_t nb_ids); | |
297 | ||
9f95a23c | 298 | int test_sw_eventdev(void); |
11fdf7f2 TL |
299 | |
300 | #endif /* _SW_EVDEV_H_ */ |