]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2010-2016 Intel Corporation | |
7c673cae FG |
3 | */ |
4 | ||
5 | #include <string.h> | |
6 | #include <stdio.h> | |
7 | ||
8 | #include <rte_common.h> | |
9 | #include <rte_memory.h> | |
7c673cae FG |
10 | #include <rte_cycles.h> |
11 | #include <rte_prefetch.h> | |
12 | #include <rte_branch_prediction.h> | |
13 | #include <rte_mbuf.h> | |
14 | #include <rte_malloc.h> | |
15 | #include <rte_string_fns.h> | |
16 | ||
17 | #include "rte_pipeline.h" | |
18 | ||
19 | #define RTE_TABLE_INVALID UINT32_MAX | |
20 | ||
21 | #ifdef RTE_PIPELINE_STATS_COLLECT | |
22 | ||
23 | #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \ | |
24 | ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); }) | |
25 | ||
26 | #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \ | |
27 | ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; }) | |
28 | ||
29 | #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \ | |
30 | ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; }) | |
31 | ||
32 | #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \ | |
33 | ({ \ | |
34 | uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \ | |
35 | mask ^= (p)->pkts_drop_mask; \ | |
36 | (counter) += __builtin_popcountll(mask); \ | |
37 | }) | |
38 | ||
39 | #else | |
40 | ||
41 | #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) | |
42 | #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) | |
43 | #define RTE_PIPELINE_STATS_TABLE_DROP0(p) | |
44 | #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) | |
45 | ||
46 | #endif | |
47 | ||
48 | struct rte_port_in { | |
49 | /* Input parameters */ | |
50 | struct rte_port_in_ops ops; | |
51 | rte_pipeline_port_in_action_handler f_action; | |
52 | void *arg_ah; | |
53 | uint32_t burst_size; | |
54 | ||
55 | /* The table to which this port is connected */ | |
56 | uint32_t table_id; | |
57 | ||
58 | /* Handle to low-level port */ | |
59 | void *h_port; | |
60 | ||
61 | /* List of enabled ports */ | |
62 | struct rte_port_in *next; | |
63 | ||
64 | /* Statistics */ | |
65 | uint64_t n_pkts_dropped_by_ah; | |
66 | }; | |
67 | ||
68 | struct rte_port_out { | |
69 | /* Input parameters */ | |
70 | struct rte_port_out_ops ops; | |
71 | rte_pipeline_port_out_action_handler f_action; | |
72 | void *arg_ah; | |
73 | ||
74 | /* Handle to low-level port */ | |
75 | void *h_port; | |
76 | ||
77 | /* Statistics */ | |
78 | uint64_t n_pkts_dropped_by_ah; | |
79 | }; | |
80 | ||
81 | struct rte_table { | |
82 | /* Input parameters */ | |
83 | struct rte_table_ops ops; | |
84 | rte_pipeline_table_action_handler_hit f_action_hit; | |
85 | rte_pipeline_table_action_handler_miss f_action_miss; | |
86 | void *arg_ah; | |
87 | struct rte_pipeline_table_entry *default_entry; | |
88 | uint32_t entry_size; | |
89 | ||
90 | uint32_t table_next_id; | |
91 | uint32_t table_next_id_valid; | |
92 | ||
93 | /* Handle to the low-level table object */ | |
94 | void *h_table; | |
95 | ||
96 | /* Statistics */ | |
97 | uint64_t n_pkts_dropped_by_lkp_hit_ah; | |
98 | uint64_t n_pkts_dropped_by_lkp_miss_ah; | |
99 | uint64_t n_pkts_dropped_lkp_hit; | |
100 | uint64_t n_pkts_dropped_lkp_miss; | |
101 | }; | |
102 | ||
103 | #define RTE_PIPELINE_MAX_NAME_SZ 124 | |
104 | ||
105 | struct rte_pipeline { | |
106 | /* Input parameters */ | |
107 | char name[RTE_PIPELINE_MAX_NAME_SZ]; | |
108 | int socket_id; | |
109 | uint32_t offset_port_id; | |
110 | ||
111 | /* Internal tables */ | |
112 | struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX]; | |
113 | struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX]; | |
114 | struct rte_table tables[RTE_PIPELINE_TABLE_MAX]; | |
115 | ||
116 | /* Occupancy of internal tables */ | |
117 | uint32_t num_ports_in; | |
118 | uint32_t num_ports_out; | |
119 | uint32_t num_tables; | |
120 | ||
121 | /* List of enabled ports */ | |
122 | uint64_t enabled_port_in_mask; | |
123 | struct rte_port_in *port_in_next; | |
124 | ||
125 | /* Pipeline run structures */ | |
126 | struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX]; | |
127 | struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX]; | |
128 | uint64_t action_mask0[RTE_PIPELINE_ACTIONS]; | |
129 | uint64_t action_mask1[RTE_PIPELINE_ACTIONS]; | |
130 | uint64_t pkts_mask; | |
131 | uint64_t n_pkts_ah_drop; | |
132 | uint64_t pkts_drop_mask; | |
133 | } __rte_cache_aligned; | |
134 | ||
135 | static inline uint32_t | |
136 | rte_mask_get_next(uint64_t mask, uint32_t pos) | |
137 | { | |
138 | uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) | | |
139 | (mask >> ((pos + 1) & 0x3F)); | |
140 | return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F; | |
141 | } | |
142 | ||
143 | static inline uint32_t | |
144 | rte_mask_get_prev(uint64_t mask, uint32_t pos) | |
145 | { | |
146 | uint64_t mask_rot = (mask >> (pos & 0x3F)) | | |
147 | (mask << ((64 - pos) & 0x3F)); | |
148 | return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F; | |
149 | } | |
150 | ||
151 | static void | |
152 | rte_pipeline_table_free(struct rte_table *table); | |
153 | ||
154 | static void | |
155 | rte_pipeline_port_in_free(struct rte_port_in *port); | |
156 | ||
157 | static void | |
158 | rte_pipeline_port_out_free(struct rte_port_out *port); | |
159 | ||
160 | /* | |
161 | * Pipeline | |
162 | * | |
163 | */ | |
164 | static int | |
165 | rte_pipeline_check_params(struct rte_pipeline_params *params) | |
166 | { | |
167 | if (params == NULL) { | |
168 | RTE_LOG(ERR, PIPELINE, | |
169 | "%s: Incorrect value for parameter params\n", __func__); | |
170 | return -EINVAL; | |
171 | } | |
172 | ||
173 | /* name */ | |
174 | if (params->name == NULL) { | |
175 | RTE_LOG(ERR, PIPELINE, | |
176 | "%s: Incorrect value for parameter name\n", __func__); | |
177 | return -EINVAL; | |
178 | } | |
179 | ||
180 | /* socket */ | |
9f95a23c | 181 | if (params->socket_id < 0) { |
7c673cae FG |
182 | RTE_LOG(ERR, PIPELINE, |
183 | "%s: Incorrect value for parameter socket_id\n", | |
184 | __func__); | |
185 | return -EINVAL; | |
186 | } | |
187 | ||
188 | return 0; | |
189 | } | |
190 | ||
191 | struct rte_pipeline * | |
192 | rte_pipeline_create(struct rte_pipeline_params *params) | |
193 | { | |
194 | struct rte_pipeline *p; | |
195 | int status; | |
196 | ||
197 | /* Check input parameters */ | |
198 | status = rte_pipeline_check_params(params); | |
199 | if (status != 0) { | |
200 | RTE_LOG(ERR, PIPELINE, | |
201 | "%s: Pipeline params check failed (%d)\n", | |
202 | __func__, status); | |
203 | return NULL; | |
204 | } | |
205 | ||
206 | /* Allocate memory for the pipeline on requested socket */ | |
207 | p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline), | |
208 | RTE_CACHE_LINE_SIZE, params->socket_id); | |
209 | ||
210 | if (p == NULL) { | |
211 | RTE_LOG(ERR, PIPELINE, | |
212 | "%s: Pipeline memory allocation failed\n", __func__); | |
213 | return NULL; | |
214 | } | |
215 | ||
216 | /* Save input parameters */ | |
9f95a23c | 217 | strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ); |
7c673cae FG |
218 | p->socket_id = params->socket_id; |
219 | p->offset_port_id = params->offset_port_id; | |
220 | ||
221 | /* Initialize pipeline internal data structure */ | |
222 | p->num_ports_in = 0; | |
223 | p->num_ports_out = 0; | |
224 | p->num_tables = 0; | |
225 | p->enabled_port_in_mask = 0; | |
226 | p->port_in_next = NULL; | |
227 | p->pkts_mask = 0; | |
228 | p->n_pkts_ah_drop = 0; | |
229 | ||
230 | return p; | |
231 | } | |
232 | ||
233 | int | |
234 | rte_pipeline_free(struct rte_pipeline *p) | |
235 | { | |
236 | uint32_t i; | |
237 | ||
238 | /* Check input parameters */ | |
239 | if (p == NULL) { | |
240 | RTE_LOG(ERR, PIPELINE, | |
241 | "%s: rte_pipeline parameter is NULL\n", __func__); | |
242 | return -EINVAL; | |
243 | } | |
244 | ||
245 | /* Free input ports */ | |
246 | for (i = 0; i < p->num_ports_in; i++) { | |
247 | struct rte_port_in *port = &p->ports_in[i]; | |
248 | ||
249 | rte_pipeline_port_in_free(port); | |
250 | } | |
251 | ||
252 | /* Free tables */ | |
253 | for (i = 0; i < p->num_tables; i++) { | |
254 | struct rte_table *table = &p->tables[i]; | |
255 | ||
256 | rte_pipeline_table_free(table); | |
257 | } | |
258 | ||
259 | /* Free output ports */ | |
260 | for (i = 0; i < p->num_ports_out; i++) { | |
261 | struct rte_port_out *port = &p->ports_out[i]; | |
262 | ||
263 | rte_pipeline_port_out_free(port); | |
264 | } | |
265 | ||
266 | /* Free pipeline memory */ | |
267 | rte_free(p); | |
268 | ||
269 | return 0; | |
270 | } | |
271 | ||
272 | /* | |
273 | * Table | |
274 | * | |
275 | */ | |
276 | static int | |
277 | rte_table_check_params(struct rte_pipeline *p, | |
278 | struct rte_pipeline_table_params *params, | |
279 | uint32_t *table_id) | |
280 | { | |
281 | if (p == NULL) { | |
282 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", | |
283 | __func__); | |
284 | return -EINVAL; | |
285 | } | |
286 | if (params == NULL) { | |
287 | RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n", | |
288 | __func__); | |
289 | return -EINVAL; | |
290 | } | |
291 | if (table_id == NULL) { | |
292 | RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n", | |
293 | __func__); | |
294 | return -EINVAL; | |
295 | } | |
296 | ||
297 | /* ops */ | |
298 | if (params->ops == NULL) { | |
299 | RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n", | |
300 | __func__); | |
301 | return -EINVAL; | |
302 | } | |
303 | ||
304 | if (params->ops->f_create == NULL) { | |
305 | RTE_LOG(ERR, PIPELINE, | |
306 | "%s: f_create function pointer is NULL\n", __func__); | |
307 | return -EINVAL; | |
308 | } | |
309 | ||
310 | if (params->ops->f_lookup == NULL) { | |
311 | RTE_LOG(ERR, PIPELINE, | |
312 | "%s: f_lookup function pointer is NULL\n", __func__); | |
313 | return -EINVAL; | |
314 | } | |
315 | ||
316 | /* De we have room for one more table? */ | |
317 | if (p->num_tables == RTE_PIPELINE_TABLE_MAX) { | |
318 | RTE_LOG(ERR, PIPELINE, | |
319 | "%s: Incorrect value for num_tables parameter\n", | |
320 | __func__); | |
321 | return -EINVAL; | |
322 | } | |
323 | ||
324 | return 0; | |
325 | } | |
326 | ||
327 | int | |
328 | rte_pipeline_table_create(struct rte_pipeline *p, | |
329 | struct rte_pipeline_table_params *params, | |
330 | uint32_t *table_id) | |
331 | { | |
332 | struct rte_table *table; | |
333 | struct rte_pipeline_table_entry *default_entry; | |
334 | void *h_table; | |
335 | uint32_t entry_size, id; | |
336 | int status; | |
337 | ||
338 | /* Check input arguments */ | |
339 | status = rte_table_check_params(p, params, table_id); | |
340 | if (status != 0) | |
341 | return status; | |
342 | ||
343 | id = p->num_tables; | |
344 | table = &p->tables[id]; | |
345 | ||
346 | /* Allocate space for the default table entry */ | |
347 | entry_size = sizeof(struct rte_pipeline_table_entry) + | |
348 | params->action_data_size; | |
9f95a23c | 349 | default_entry = rte_zmalloc_socket( |
7c673cae FG |
350 | "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id); |
351 | if (default_entry == NULL) { | |
352 | RTE_LOG(ERR, PIPELINE, | |
353 | "%s: Failed to allocate default entry\n", __func__); | |
354 | return -EINVAL; | |
355 | } | |
356 | ||
357 | /* Create the table */ | |
358 | h_table = params->ops->f_create(params->arg_create, p->socket_id, | |
359 | entry_size); | |
360 | if (h_table == NULL) { | |
361 | rte_free(default_entry); | |
362 | RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__); | |
363 | return -EINVAL; | |
364 | } | |
365 | ||
366 | /* Commit current table to the pipeline */ | |
367 | p->num_tables++; | |
368 | *table_id = id; | |
369 | ||
370 | /* Save input parameters */ | |
371 | memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops)); | |
372 | table->f_action_hit = params->f_action_hit; | |
373 | table->f_action_miss = params->f_action_miss; | |
374 | table->arg_ah = params->arg_ah; | |
375 | table->entry_size = entry_size; | |
376 | ||
377 | /* Clear the lookup miss actions (to be set later through API) */ | |
378 | table->default_entry = default_entry; | |
379 | table->default_entry->action = RTE_PIPELINE_ACTION_DROP; | |
380 | ||
381 | /* Initialize table internal data structure */ | |
382 | table->h_table = h_table; | |
383 | table->table_next_id = 0; | |
384 | table->table_next_id_valid = 0; | |
385 | ||
386 | return 0; | |
387 | } | |
388 | ||
389 | void | |
390 | rte_pipeline_table_free(struct rte_table *table) | |
391 | { | |
392 | if (table->ops.f_free != NULL) | |
393 | table->ops.f_free(table->h_table); | |
394 | ||
395 | rte_free(table->default_entry); | |
396 | } | |
397 | ||
398 | int | |
399 | rte_pipeline_table_default_entry_add(struct rte_pipeline *p, | |
400 | uint32_t table_id, | |
401 | struct rte_pipeline_table_entry *default_entry, | |
402 | struct rte_pipeline_table_entry **default_entry_ptr) | |
403 | { | |
404 | struct rte_table *table; | |
405 | ||
406 | /* Check input arguments */ | |
407 | if (p == NULL) { | |
408 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", | |
409 | __func__); | |
410 | return -EINVAL; | |
411 | } | |
412 | ||
413 | if (default_entry == NULL) { | |
414 | RTE_LOG(ERR, PIPELINE, | |
415 | "%s: default_entry parameter is NULL\n", __func__); | |
416 | return -EINVAL; | |
417 | } | |
418 | ||
419 | if (table_id >= p->num_tables) { | |
420 | RTE_LOG(ERR, PIPELINE, | |
421 | "%s: table_id %d out of range\n", __func__, table_id); | |
422 | return -EINVAL; | |
423 | } | |
424 | ||
425 | table = &p->tables[table_id]; | |
426 | ||
427 | if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && | |
428 | table->table_next_id_valid && | |
429 | (default_entry->table_id != table->table_next_id)) { | |
430 | RTE_LOG(ERR, PIPELINE, | |
431 | "%s: Tree-like topologies not allowed\n", __func__); | |
432 | return -EINVAL; | |
433 | } | |
434 | ||
435 | /* Set the lookup miss actions */ | |
436 | if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) && | |
437 | (table->table_next_id_valid == 0)) { | |
438 | table->table_next_id = default_entry->table_id; | |
439 | table->table_next_id_valid = 1; | |
440 | } | |
441 | ||
442 | memcpy(table->default_entry, default_entry, table->entry_size); | |
443 | ||
444 | *default_entry_ptr = table->default_entry; | |
445 | return 0; | |
446 | } | |
447 | ||
448 | int | |
449 | rte_pipeline_table_default_entry_delete(struct rte_pipeline *p, | |
450 | uint32_t table_id, | |
451 | struct rte_pipeline_table_entry *entry) | |
452 | { | |
453 | struct rte_table *table; | |
454 | ||
455 | /* Check input arguments */ | |
456 | if (p == NULL) { | |
457 | RTE_LOG(ERR, PIPELINE, | |
458 | "%s: pipeline parameter is NULL\n", __func__); | |
459 | return -EINVAL; | |
460 | } | |
461 | ||
462 | if (table_id >= p->num_tables) { | |
463 | RTE_LOG(ERR, PIPELINE, | |
464 | "%s: table_id %d out of range\n", __func__, table_id); | |
465 | return -EINVAL; | |
466 | } | |
467 | ||
468 | table = &p->tables[table_id]; | |
469 | ||
470 | /* Save the current contents of the default entry */ | |
471 | if (entry) | |
472 | memcpy(entry, table->default_entry, table->entry_size); | |
473 | ||
474 | /* Clear the lookup miss actions */ | |
475 | memset(table->default_entry, 0, table->entry_size); | |
476 | table->default_entry->action = RTE_PIPELINE_ACTION_DROP; | |
477 | ||
478 | return 0; | |
479 | } | |
480 | ||
481 | int | |
482 | rte_pipeline_table_entry_add(struct rte_pipeline *p, | |
483 | uint32_t table_id, | |
484 | void *key, | |
485 | struct rte_pipeline_table_entry *entry, | |
486 | int *key_found, | |
487 | struct rte_pipeline_table_entry **entry_ptr) | |
488 | { | |
489 | struct rte_table *table; | |
490 | ||
491 | /* Check input arguments */ | |
492 | if (p == NULL) { | |
493 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", | |
494 | __func__); | |
495 | return -EINVAL; | |
496 | } | |
497 | ||
498 | if (key == NULL) { | |
499 | RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__); | |
500 | return -EINVAL; | |
501 | } | |
502 | ||
503 | if (entry == NULL) { | |
504 | RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n", | |
505 | __func__); | |
506 | return -EINVAL; | |
507 | } | |
508 | ||
509 | if (table_id >= p->num_tables) { | |
510 | RTE_LOG(ERR, PIPELINE, | |
511 | "%s: table_id %d out of range\n", __func__, table_id); | |
512 | return -EINVAL; | |
513 | } | |
514 | ||
515 | table = &p->tables[table_id]; | |
516 | ||
517 | if (table->ops.f_add == NULL) { | |
518 | RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n", | |
519 | __func__); | |
520 | return -EINVAL; | |
521 | } | |
522 | ||
523 | if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && | |
524 | table->table_next_id_valid && | |
525 | (entry->table_id != table->table_next_id)) { | |
526 | RTE_LOG(ERR, PIPELINE, | |
527 | "%s: Tree-like topologies not allowed\n", __func__); | |
528 | return -EINVAL; | |
529 | } | |
530 | ||
531 | /* Add entry */ | |
532 | if ((entry->action == RTE_PIPELINE_ACTION_TABLE) && | |
533 | (table->table_next_id_valid == 0)) { | |
534 | table->table_next_id = entry->table_id; | |
535 | table->table_next_id_valid = 1; | |
536 | } | |
537 | ||
538 | return (table->ops.f_add)(table->h_table, key, (void *) entry, | |
539 | key_found, (void **) entry_ptr); | |
540 | } | |
541 | ||
542 | int | |
543 | rte_pipeline_table_entry_delete(struct rte_pipeline *p, | |
544 | uint32_t table_id, | |
545 | void *key, | |
546 | int *key_found, | |
547 | struct rte_pipeline_table_entry *entry) | |
548 | { | |
549 | struct rte_table *table; | |
550 | ||
551 | /* Check input arguments */ | |
552 | if (p == NULL) { | |
553 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
554 | __func__); | |
555 | return -EINVAL; | |
556 | } | |
557 | ||
558 | if (key == NULL) { | |
559 | RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", | |
560 | __func__); | |
561 | return -EINVAL; | |
562 | } | |
563 | ||
564 | if (table_id >= p->num_tables) { | |
565 | RTE_LOG(ERR, PIPELINE, | |
566 | "%s: table_id %d out of range\n", __func__, table_id); | |
567 | return -EINVAL; | |
568 | } | |
569 | ||
570 | table = &p->tables[table_id]; | |
571 | ||
572 | if (table->ops.f_delete == NULL) { | |
573 | RTE_LOG(ERR, PIPELINE, | |
574 | "%s: f_delete function pointer NULL\n", __func__); | |
575 | return -EINVAL; | |
576 | } | |
577 | ||
578 | return (table->ops.f_delete)(table->h_table, key, key_found, entry); | |
579 | } | |
580 | ||
581 | int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p, | |
582 | uint32_t table_id, | |
583 | void **keys, | |
584 | struct rte_pipeline_table_entry **entries, | |
585 | uint32_t n_keys, | |
586 | int *key_found, | |
587 | struct rte_pipeline_table_entry **entries_ptr) | |
588 | { | |
589 | struct rte_table *table; | |
590 | uint32_t i; | |
591 | ||
592 | /* Check input arguments */ | |
593 | if (p == NULL) { | |
594 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n", | |
595 | __func__); | |
596 | return -EINVAL; | |
597 | } | |
598 | ||
599 | if (keys == NULL) { | |
600 | RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__); | |
601 | return -EINVAL; | |
602 | } | |
603 | ||
604 | if (entries == NULL) { | |
605 | RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n", | |
606 | __func__); | |
607 | return -EINVAL; | |
608 | } | |
609 | ||
610 | if (table_id >= p->num_tables) { | |
611 | RTE_LOG(ERR, PIPELINE, | |
612 | "%s: table_id %d out of range\n", __func__, table_id); | |
613 | return -EINVAL; | |
614 | } | |
615 | ||
616 | table = &p->tables[table_id]; | |
617 | ||
618 | if (table->ops.f_add_bulk == NULL) { | |
619 | RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n", | |
620 | __func__); | |
621 | return -EINVAL; | |
622 | } | |
623 | ||
624 | for (i = 0; i < n_keys; i++) { | |
625 | if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && | |
626 | table->table_next_id_valid && | |
627 | (entries[i]->table_id != table->table_next_id)) { | |
628 | RTE_LOG(ERR, PIPELINE, | |
629 | "%s: Tree-like topologies not allowed\n", __func__); | |
630 | return -EINVAL; | |
631 | } | |
632 | } | |
633 | ||
634 | /* Add entry */ | |
635 | for (i = 0; i < n_keys; i++) { | |
636 | if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) && | |
637 | (table->table_next_id_valid == 0)) { | |
638 | table->table_next_id = entries[i]->table_id; | |
639 | table->table_next_id_valid = 1; | |
640 | } | |
641 | } | |
642 | ||
643 | return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries, | |
644 | n_keys, key_found, (void **) entries_ptr); | |
645 | } | |
646 | ||
647 | int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p, | |
648 | uint32_t table_id, | |
649 | void **keys, | |
650 | uint32_t n_keys, | |
651 | int *key_found, | |
652 | struct rte_pipeline_table_entry **entries) | |
653 | { | |
654 | struct rte_table *table; | |
655 | ||
656 | /* Check input arguments */ | |
657 | if (p == NULL) { | |
658 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
659 | __func__); | |
660 | return -EINVAL; | |
661 | } | |
662 | ||
663 | if (keys == NULL) { | |
664 | RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", | |
665 | __func__); | |
666 | return -EINVAL; | |
667 | } | |
668 | ||
669 | if (table_id >= p->num_tables) { | |
670 | RTE_LOG(ERR, PIPELINE, | |
671 | "%s: table_id %d out of range\n", __func__, table_id); | |
672 | return -EINVAL; | |
673 | } | |
674 | ||
675 | table = &p->tables[table_id]; | |
676 | ||
677 | if (table->ops.f_delete_bulk == NULL) { | |
678 | RTE_LOG(ERR, PIPELINE, | |
679 | "%s: f_delete function pointer NULL\n", __func__); | |
680 | return -EINVAL; | |
681 | } | |
682 | ||
683 | return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found, | |
684 | (void **) entries); | |
685 | } | |
686 | ||
687 | /* | |
688 | * Port | |
689 | * | |
690 | */ | |
691 | static int | |
692 | rte_pipeline_port_in_check_params(struct rte_pipeline *p, | |
693 | struct rte_pipeline_port_in_params *params, | |
694 | uint32_t *port_id) | |
695 | { | |
696 | if (p == NULL) { | |
697 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
698 | __func__); | |
699 | return -EINVAL; | |
700 | } | |
701 | if (params == NULL) { | |
702 | RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); | |
703 | return -EINVAL; | |
704 | } | |
705 | if (port_id == NULL) { | |
706 | RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", | |
707 | __func__); | |
708 | return -EINVAL; | |
709 | } | |
710 | ||
711 | /* ops */ | |
712 | if (params->ops == NULL) { | |
713 | RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", | |
714 | __func__); | |
715 | return -EINVAL; | |
716 | } | |
717 | ||
718 | if (params->ops->f_create == NULL) { | |
719 | RTE_LOG(ERR, PIPELINE, | |
720 | "%s: f_create function pointer NULL\n", __func__); | |
721 | return -EINVAL; | |
722 | } | |
723 | ||
724 | if (params->ops->f_rx == NULL) { | |
725 | RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n", | |
726 | __func__); | |
727 | return -EINVAL; | |
728 | } | |
729 | ||
730 | /* burst_size */ | |
731 | if ((params->burst_size == 0) || | |
732 | (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) { | |
733 | RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n", | |
734 | __func__); | |
735 | return -EINVAL; | |
736 | } | |
737 | ||
738 | /* Do we have room for one more port? */ | |
739 | if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) { | |
740 | RTE_LOG(ERR, PIPELINE, | |
741 | "%s: invalid value for num_ports_in\n", __func__); | |
742 | return -EINVAL; | |
743 | } | |
744 | ||
745 | return 0; | |
746 | } | |
747 | ||
748 | static int | |
749 | rte_pipeline_port_out_check_params(struct rte_pipeline *p, | |
750 | struct rte_pipeline_port_out_params *params, | |
751 | uint32_t *port_id) | |
752 | { | |
753 | if (p == NULL) { | |
754 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
755 | __func__); | |
756 | return -EINVAL; | |
757 | } | |
758 | ||
759 | if (params == NULL) { | |
760 | RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__); | |
761 | return -EINVAL; | |
762 | } | |
763 | ||
764 | if (port_id == NULL) { | |
765 | RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n", | |
766 | __func__); | |
767 | return -EINVAL; | |
768 | } | |
769 | ||
770 | /* ops */ | |
771 | if (params->ops == NULL) { | |
772 | RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n", | |
773 | __func__); | |
774 | return -EINVAL; | |
775 | } | |
776 | ||
777 | if (params->ops->f_create == NULL) { | |
778 | RTE_LOG(ERR, PIPELINE, | |
779 | "%s: f_create function pointer NULL\n", __func__); | |
780 | return -EINVAL; | |
781 | } | |
782 | ||
783 | if (params->ops->f_tx == NULL) { | |
784 | RTE_LOG(ERR, PIPELINE, | |
785 | "%s: f_tx function pointer NULL\n", __func__); | |
786 | return -EINVAL; | |
787 | } | |
788 | ||
789 | if (params->ops->f_tx_bulk == NULL) { | |
790 | RTE_LOG(ERR, PIPELINE, | |
791 | "%s: f_tx_bulk function pointer NULL\n", __func__); | |
792 | return -EINVAL; | |
793 | } | |
794 | ||
795 | /* Do we have room for one more port? */ | |
796 | if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) { | |
797 | RTE_LOG(ERR, PIPELINE, | |
798 | "%s: invalid value for num_ports_out\n", __func__); | |
799 | return -EINVAL; | |
800 | } | |
801 | ||
802 | return 0; | |
803 | } | |
804 | ||
805 | int | |
806 | rte_pipeline_port_in_create(struct rte_pipeline *p, | |
807 | struct rte_pipeline_port_in_params *params, | |
808 | uint32_t *port_id) | |
809 | { | |
810 | struct rte_port_in *port; | |
811 | void *h_port; | |
812 | uint32_t id; | |
813 | int status; | |
814 | ||
815 | /* Check input arguments */ | |
816 | status = rte_pipeline_port_in_check_params(p, params, port_id); | |
817 | if (status != 0) | |
818 | return status; | |
819 | ||
820 | id = p->num_ports_in; | |
821 | port = &p->ports_in[id]; | |
822 | ||
823 | /* Create the port */ | |
824 | h_port = params->ops->f_create(params->arg_create, p->socket_id); | |
825 | if (h_port == NULL) { | |
826 | RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); | |
827 | return -EINVAL; | |
828 | } | |
829 | ||
830 | /* Commit current table to the pipeline */ | |
831 | p->num_ports_in++; | |
832 | *port_id = id; | |
833 | ||
834 | /* Save input parameters */ | |
835 | memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops)); | |
836 | port->f_action = params->f_action; | |
837 | port->arg_ah = params->arg_ah; | |
838 | port->burst_size = params->burst_size; | |
839 | ||
840 | /* Initialize port internal data structure */ | |
841 | port->table_id = RTE_TABLE_INVALID; | |
842 | port->h_port = h_port; | |
843 | port->next = NULL; | |
844 | ||
845 | return 0; | |
846 | } | |
847 | ||
848 | void | |
849 | rte_pipeline_port_in_free(struct rte_port_in *port) | |
850 | { | |
851 | if (port->ops.f_free != NULL) | |
852 | port->ops.f_free(port->h_port); | |
853 | } | |
854 | ||
855 | int | |
856 | rte_pipeline_port_out_create(struct rte_pipeline *p, | |
857 | struct rte_pipeline_port_out_params *params, | |
858 | uint32_t *port_id) | |
859 | { | |
860 | struct rte_port_out *port; | |
861 | void *h_port; | |
862 | uint32_t id; | |
863 | int status; | |
864 | ||
865 | /* Check input arguments */ | |
866 | status = rte_pipeline_port_out_check_params(p, params, port_id); | |
867 | if (status != 0) | |
868 | return status; | |
869 | ||
870 | id = p->num_ports_out; | |
871 | port = &p->ports_out[id]; | |
872 | ||
873 | /* Create the port */ | |
874 | h_port = params->ops->f_create(params->arg_create, p->socket_id); | |
875 | if (h_port == NULL) { | |
876 | RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__); | |
877 | return -EINVAL; | |
878 | } | |
879 | ||
880 | /* Commit current table to the pipeline */ | |
881 | p->num_ports_out++; | |
882 | *port_id = id; | |
883 | ||
884 | /* Save input parameters */ | |
885 | memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops)); | |
886 | port->f_action = params->f_action; | |
887 | port->arg_ah = params->arg_ah; | |
888 | ||
889 | /* Initialize port internal data structure */ | |
890 | port->h_port = h_port; | |
891 | ||
892 | return 0; | |
893 | } | |
894 | ||
895 | void | |
896 | rte_pipeline_port_out_free(struct rte_port_out *port) | |
897 | { | |
898 | if (port->ops.f_free != NULL) | |
899 | port->ops.f_free(port->h_port); | |
900 | } | |
901 | ||
902 | int | |
903 | rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p, | |
904 | uint32_t port_id, | |
905 | uint32_t table_id) | |
906 | { | |
907 | struct rte_port_in *port; | |
908 | ||
909 | /* Check input arguments */ | |
910 | if (p == NULL) { | |
911 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
912 | __func__); | |
913 | return -EINVAL; | |
914 | } | |
915 | ||
916 | if (port_id >= p->num_ports_in) { | |
917 | RTE_LOG(ERR, PIPELINE, | |
918 | "%s: port IN ID %u is out of range\n", | |
919 | __func__, port_id); | |
920 | return -EINVAL; | |
921 | } | |
922 | ||
923 | if (table_id >= p->num_tables) { | |
924 | RTE_LOG(ERR, PIPELINE, | |
925 | "%s: Table ID %u is out of range\n", | |
926 | __func__, table_id); | |
927 | return -EINVAL; | |
928 | } | |
929 | ||
930 | port = &p->ports_in[port_id]; | |
931 | port->table_id = table_id; | |
932 | ||
933 | return 0; | |
934 | } | |
935 | ||
936 | int | |
937 | rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id) | |
938 | { | |
939 | struct rte_port_in *port, *port_prev, *port_next; | |
940 | uint64_t port_mask; | |
941 | uint32_t port_prev_id, port_next_id; | |
942 | ||
943 | /* Check input arguments */ | |
944 | if (p == NULL) { | |
945 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
946 | __func__); | |
947 | return -EINVAL; | |
948 | } | |
949 | ||
950 | if (port_id >= p->num_ports_in) { | |
951 | RTE_LOG(ERR, PIPELINE, | |
952 | "%s: port IN ID %u is out of range\n", | |
953 | __func__, port_id); | |
954 | return -EINVAL; | |
955 | } | |
956 | ||
957 | port = &p->ports_in[port_id]; | |
958 | ||
959 | /* Return if current input port is already enabled */ | |
960 | port_mask = 1LLU << port_id; | |
961 | if (p->enabled_port_in_mask & port_mask) | |
962 | return 0; | |
963 | ||
964 | p->enabled_port_in_mask |= port_mask; | |
965 | ||
966 | /* Add current input port to the pipeline chain of enabled ports */ | |
967 | port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); | |
968 | port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); | |
969 | ||
970 | port_prev = &p->ports_in[port_prev_id]; | |
971 | port_next = &p->ports_in[port_next_id]; | |
972 | ||
973 | port_prev->next = port; | |
974 | port->next = port_next; | |
975 | ||
976 | /* Check if list of enabled ports was previously empty */ | |
977 | if (p->enabled_port_in_mask == port_mask) | |
978 | p->port_in_next = port; | |
979 | ||
980 | return 0; | |
981 | } | |
982 | ||
983 | int | |
984 | rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id) | |
985 | { | |
986 | struct rte_port_in *port, *port_prev, *port_next; | |
987 | uint64_t port_mask; | |
988 | uint32_t port_prev_id, port_next_id; | |
989 | ||
990 | /* Check input arguments */ | |
991 | if (p == NULL) { | |
992 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
993 | __func__); | |
994 | return -EINVAL; | |
995 | } | |
996 | ||
997 | if (port_id >= p->num_ports_in) { | |
998 | RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n", | |
999 | __func__, port_id); | |
1000 | return -EINVAL; | |
1001 | } | |
1002 | ||
1003 | port = &p->ports_in[port_id]; | |
1004 | ||
1005 | /* Return if current input port is already disabled */ | |
1006 | port_mask = 1LLU << port_id; | |
1007 | if ((p->enabled_port_in_mask & port_mask) == 0) | |
1008 | return 0; | |
1009 | ||
1010 | p->enabled_port_in_mask &= ~port_mask; | |
1011 | ||
1012 | /* Return if no other enabled ports */ | |
1013 | if (p->enabled_port_in_mask == 0) { | |
1014 | p->port_in_next = NULL; | |
1015 | ||
1016 | return 0; | |
1017 | } | |
1018 | ||
1019 | /* Add current input port to the pipeline chain of enabled ports */ | |
1020 | port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id); | |
1021 | port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id); | |
1022 | ||
1023 | port_prev = &p->ports_in[port_prev_id]; | |
1024 | port_next = &p->ports_in[port_next_id]; | |
1025 | ||
1026 | port_prev->next = port_next; | |
1027 | ||
1028 | /* Check if the port which has just been disabled is next to serve */ | |
1029 | if (port == p->port_in_next) | |
1030 | p->port_in_next = port_next; | |
1031 | ||
1032 | return 0; | |
1033 | } | |
1034 | ||
1035 | /* | |
1036 | * Pipeline run-time | |
1037 | * | |
1038 | */ | |
1039 | int | |
1040 | rte_pipeline_check(struct rte_pipeline *p) | |
1041 | { | |
1042 | uint32_t port_in_id; | |
1043 | ||
1044 | /* Check input arguments */ | |
1045 | if (p == NULL) { | |
1046 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
1047 | __func__); | |
1048 | return -EINVAL; | |
1049 | } | |
1050 | ||
1051 | /* Check that pipeline has at least one input port, one table and one | |
1052 | output port */ | |
1053 | if (p->num_ports_in == 0) { | |
1054 | RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n", | |
1055 | __func__); | |
1056 | return -EINVAL; | |
1057 | } | |
1058 | if (p->num_tables == 0) { | |
1059 | RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n", | |
1060 | __func__); | |
1061 | return -EINVAL; | |
1062 | } | |
1063 | if (p->num_ports_out == 0) { | |
1064 | RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n", | |
1065 | __func__); | |
1066 | return -EINVAL; | |
1067 | } | |
1068 | ||
1069 | /* Check that all input ports are connected */ | |
1070 | for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) { | |
1071 | struct rte_port_in *port_in = &p->ports_in[port_in_id]; | |
1072 | ||
1073 | if (port_in->table_id == RTE_TABLE_INVALID) { | |
1074 | RTE_LOG(ERR, PIPELINE, | |
1075 | "%s: Port IN ID %u is not connected\n", | |
1076 | __func__, port_in_id); | |
1077 | return -EINVAL; | |
1078 | } | |
1079 | } | |
1080 | ||
1081 | return 0; | |
1082 | } | |
1083 | ||
1084 | static inline void | |
1085 | rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask) | |
1086 | { | |
1087 | p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0; | |
1088 | p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0; | |
1089 | p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0; | |
1090 | p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0; | |
1091 | ||
1092 | if ((pkts_mask & (pkts_mask + 1)) == 0) { | |
1093 | uint64_t n_pkts = __builtin_popcountll(pkts_mask); | |
1094 | uint32_t i; | |
1095 | ||
1096 | for (i = 0; i < n_pkts; i++) { | |
1097 | uint64_t pkt_mask = 1LLU << i; | |
1098 | uint32_t pos = p->entries[i]->action; | |
1099 | ||
1100 | p->action_mask1[pos] |= pkt_mask; | |
1101 | } | |
1102 | } else { | |
1103 | uint32_t i; | |
1104 | ||
1105 | for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { | |
1106 | uint64_t pkt_mask = 1LLU << i; | |
1107 | uint32_t pos; | |
1108 | ||
1109 | if ((pkt_mask & pkts_mask) == 0) | |
1110 | continue; | |
1111 | ||
1112 | pos = p->entries[i]->action; | |
1113 | p->action_mask1[pos] |= pkt_mask; | |
1114 | } | |
1115 | } | |
1116 | } | |
1117 | ||
1118 | static inline void | |
1119 | rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p, | |
1120 | uint64_t pkts_mask, uint32_t port_id) | |
1121 | { | |
1122 | struct rte_port_out *port_out = &p->ports_out[port_id]; | |
1123 | ||
1124 | p->pkts_mask = pkts_mask; | |
1125 | ||
1126 | /* Output port user actions */ | |
1127 | if (port_out->f_action != NULL) { | |
1128 | port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah); | |
1129 | ||
1130 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1131 | port_out->n_pkts_dropped_by_ah); | |
1132 | } | |
1133 | ||
1134 | /* Output port TX */ | |
1135 | if (p->pkts_mask != 0) | |
1136 | port_out->ops.f_tx_bulk(port_out->h_port, | |
1137 | p->pkts, | |
1138 | p->pkts_mask); | |
1139 | } | |
1140 | ||
1141 | static inline void | |
1142 | rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask) | |
1143 | { | |
1144 | p->pkts_mask = pkts_mask; | |
1145 | ||
1146 | if ((pkts_mask & (pkts_mask + 1)) == 0) { | |
1147 | uint64_t n_pkts = __builtin_popcountll(pkts_mask); | |
1148 | uint32_t i; | |
1149 | ||
1150 | for (i = 0; i < n_pkts; i++) { | |
1151 | struct rte_mbuf *pkt = p->pkts[i]; | |
1152 | uint32_t port_out_id = p->entries[i]->port_id; | |
1153 | struct rte_port_out *port_out = | |
1154 | &p->ports_out[port_out_id]; | |
1155 | ||
1156 | /* Output port user actions */ | |
1157 | if (port_out->f_action == NULL) /* Output port TX */ | |
1158 | port_out->ops.f_tx(port_out->h_port, pkt); | |
1159 | else { | |
1160 | uint64_t pkt_mask = 1LLU << i; | |
1161 | ||
1162 | port_out->f_action(p, | |
1163 | p->pkts, | |
1164 | pkt_mask, | |
1165 | port_out->arg_ah); | |
1166 | ||
1167 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1168 | port_out->n_pkts_dropped_by_ah); | |
1169 | ||
1170 | /* Output port TX */ | |
1171 | if (pkt_mask & p->pkts_mask) | |
1172 | port_out->ops.f_tx(port_out->h_port, | |
1173 | pkt); | |
1174 | } | |
1175 | } | |
1176 | } else { | |
1177 | uint32_t i; | |
1178 | ||
1179 | for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { | |
1180 | uint64_t pkt_mask = 1LLU << i; | |
1181 | struct rte_mbuf *pkt; | |
1182 | struct rte_port_out *port_out; | |
1183 | uint32_t port_out_id; | |
1184 | ||
1185 | if ((pkt_mask & pkts_mask) == 0) | |
1186 | continue; | |
1187 | ||
1188 | pkt = p->pkts[i]; | |
1189 | port_out_id = p->entries[i]->port_id; | |
1190 | port_out = &p->ports_out[port_out_id]; | |
1191 | ||
1192 | /* Output port user actions */ | |
1193 | if (port_out->f_action == NULL) /* Output port TX */ | |
1194 | port_out->ops.f_tx(port_out->h_port, pkt); | |
1195 | else { | |
1196 | port_out->f_action(p, | |
1197 | p->pkts, | |
1198 | pkt_mask, | |
1199 | port_out->arg_ah); | |
1200 | ||
1201 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1202 | port_out->n_pkts_dropped_by_ah); | |
1203 | ||
1204 | /* Output port TX */ | |
1205 | if (pkt_mask & p->pkts_mask) | |
1206 | port_out->ops.f_tx(port_out->h_port, | |
1207 | pkt); | |
1208 | } | |
1209 | } | |
1210 | } | |
1211 | } | |
1212 | ||
1213 | static inline void | |
1214 | rte_pipeline_action_handler_port_meta(struct rte_pipeline *p, | |
1215 | uint64_t pkts_mask) | |
1216 | { | |
1217 | p->pkts_mask = pkts_mask; | |
1218 | ||
1219 | if ((pkts_mask & (pkts_mask + 1)) == 0) { | |
1220 | uint64_t n_pkts = __builtin_popcountll(pkts_mask); | |
1221 | uint32_t i; | |
1222 | ||
1223 | for (i = 0; i < n_pkts; i++) { | |
1224 | struct rte_mbuf *pkt = p->pkts[i]; | |
1225 | uint32_t port_out_id = | |
1226 | RTE_MBUF_METADATA_UINT32(pkt, | |
1227 | p->offset_port_id); | |
1228 | struct rte_port_out *port_out = &p->ports_out[ | |
1229 | port_out_id]; | |
1230 | ||
1231 | /* Output port user actions */ | |
1232 | if (port_out->f_action == NULL) /* Output port TX */ | |
1233 | port_out->ops.f_tx(port_out->h_port, pkt); | |
1234 | else { | |
1235 | uint64_t pkt_mask = 1LLU << i; | |
1236 | ||
1237 | port_out->f_action(p, | |
1238 | p->pkts, | |
1239 | pkt_mask, | |
1240 | port_out->arg_ah); | |
1241 | ||
1242 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1243 | port_out->n_pkts_dropped_by_ah); | |
1244 | ||
1245 | /* Output port TX */ | |
1246 | if (pkt_mask & p->pkts_mask) | |
1247 | port_out->ops.f_tx(port_out->h_port, | |
1248 | pkt); | |
1249 | } | |
1250 | } | |
1251 | } else { | |
1252 | uint32_t i; | |
1253 | ||
1254 | for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { | |
1255 | uint64_t pkt_mask = 1LLU << i; | |
1256 | struct rte_mbuf *pkt; | |
1257 | struct rte_port_out *port_out; | |
1258 | uint32_t port_out_id; | |
1259 | ||
1260 | if ((pkt_mask & pkts_mask) == 0) | |
1261 | continue; | |
1262 | ||
1263 | pkt = p->pkts[i]; | |
1264 | port_out_id = RTE_MBUF_METADATA_UINT32(pkt, | |
1265 | p->offset_port_id); | |
1266 | port_out = &p->ports_out[port_out_id]; | |
1267 | ||
1268 | /* Output port user actions */ | |
1269 | if (port_out->f_action == NULL) /* Output port TX */ | |
1270 | port_out->ops.f_tx(port_out->h_port, pkt); | |
1271 | else { | |
1272 | port_out->f_action(p, | |
1273 | p->pkts, | |
1274 | pkt_mask, | |
1275 | port_out->arg_ah); | |
1276 | ||
1277 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1278 | port_out->n_pkts_dropped_by_ah); | |
1279 | ||
1280 | /* Output port TX */ | |
1281 | if (pkt_mask & p->pkts_mask) | |
1282 | port_out->ops.f_tx(port_out->h_port, | |
1283 | pkt); | |
1284 | } | |
1285 | } | |
1286 | } | |
1287 | } | |
1288 | ||
1289 | static inline void | |
1290 | rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask) | |
1291 | { | |
1292 | if ((pkts_mask & (pkts_mask + 1)) == 0) { | |
1293 | uint64_t n_pkts = __builtin_popcountll(pkts_mask); | |
1294 | uint32_t i; | |
1295 | ||
1296 | for (i = 0; i < n_pkts; i++) | |
1297 | rte_pktmbuf_free(p->pkts[i]); | |
1298 | } else { | |
1299 | uint32_t i; | |
1300 | ||
1301 | for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) { | |
1302 | uint64_t pkt_mask = 1LLU << i; | |
1303 | ||
1304 | if ((pkt_mask & pkts_mask) == 0) | |
1305 | continue; | |
1306 | ||
1307 | rte_pktmbuf_free(p->pkts[i]); | |
1308 | } | |
1309 | } | |
1310 | } | |
1311 | ||
1312 | int | |
1313 | rte_pipeline_run(struct rte_pipeline *p) | |
1314 | { | |
1315 | struct rte_port_in *port_in = p->port_in_next; | |
1316 | uint32_t n_pkts, table_id; | |
1317 | ||
1318 | if (port_in == NULL) | |
1319 | return 0; | |
1320 | ||
1321 | /* Input port RX */ | |
1322 | n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts, | |
1323 | port_in->burst_size); | |
1324 | if (n_pkts == 0) { | |
1325 | p->port_in_next = port_in->next; | |
1326 | return 0; | |
1327 | } | |
1328 | ||
1329 | p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t); | |
1330 | p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0; | |
1331 | p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0; | |
1332 | p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0; | |
1333 | p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; | |
1334 | ||
1335 | /* Input port user actions */ | |
1336 | if (port_in->f_action != NULL) { | |
1337 | port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah); | |
1338 | ||
1339 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1340 | port_in->n_pkts_dropped_by_ah); | |
1341 | } | |
1342 | ||
1343 | /* Table */ | |
1344 | for (table_id = port_in->table_id; p->pkts_mask != 0; ) { | |
1345 | struct rte_table *table; | |
1346 | uint64_t lookup_hit_mask, lookup_miss_mask; | |
1347 | ||
1348 | /* Lookup */ | |
1349 | table = &p->tables[table_id]; | |
1350 | table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask, | |
1351 | &lookup_hit_mask, (void **) p->entries); | |
1352 | lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask); | |
1353 | ||
1354 | /* Lookup miss */ | |
1355 | if (lookup_miss_mask != 0) { | |
1356 | struct rte_pipeline_table_entry *default_entry = | |
1357 | table->default_entry; | |
1358 | ||
1359 | p->pkts_mask = lookup_miss_mask; | |
1360 | ||
1361 | /* Table user actions */ | |
1362 | if (table->f_action_miss != NULL) { | |
1363 | table->f_action_miss(p, | |
1364 | p->pkts, | |
1365 | lookup_miss_mask, | |
1366 | default_entry, | |
1367 | table->arg_ah); | |
1368 | ||
1369 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1370 | table->n_pkts_dropped_by_lkp_miss_ah); | |
1371 | } | |
1372 | ||
1373 | /* Table reserved actions */ | |
1374 | if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) && | |
1375 | (p->pkts_mask != 0)) | |
1376 | rte_pipeline_action_handler_port_bulk(p, | |
1377 | p->pkts_mask, | |
1378 | default_entry->port_id); | |
1379 | else { | |
1380 | uint32_t pos = default_entry->action; | |
1381 | ||
1382 | RTE_PIPELINE_STATS_TABLE_DROP0(p); | |
1383 | ||
1384 | p->action_mask0[pos] |= p->pkts_mask; | |
1385 | ||
1386 | RTE_PIPELINE_STATS_TABLE_DROP1(p, | |
1387 | table->n_pkts_dropped_lkp_miss); | |
1388 | } | |
1389 | } | |
1390 | ||
1391 | /* Lookup hit */ | |
1392 | if (lookup_hit_mask != 0) { | |
1393 | p->pkts_mask = lookup_hit_mask; | |
1394 | ||
1395 | /* Table user actions */ | |
1396 | if (table->f_action_hit != NULL) { | |
1397 | table->f_action_hit(p, | |
1398 | p->pkts, | |
1399 | lookup_hit_mask, | |
1400 | p->entries, | |
1401 | table->arg_ah); | |
1402 | ||
1403 | RTE_PIPELINE_STATS_AH_DROP_READ(p, | |
1404 | table->n_pkts_dropped_by_lkp_hit_ah); | |
1405 | } | |
1406 | ||
1407 | /* Table reserved actions */ | |
1408 | RTE_PIPELINE_STATS_TABLE_DROP0(p); | |
1409 | rte_pipeline_compute_masks(p, p->pkts_mask); | |
1410 | p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= | |
1411 | p->action_mask1[ | |
1412 | RTE_PIPELINE_ACTION_DROP]; | |
1413 | p->action_mask0[RTE_PIPELINE_ACTION_PORT] |= | |
1414 | p->action_mask1[ | |
1415 | RTE_PIPELINE_ACTION_PORT]; | |
1416 | p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |= | |
1417 | p->action_mask1[ | |
1418 | RTE_PIPELINE_ACTION_PORT_META]; | |
1419 | p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |= | |
1420 | p->action_mask1[ | |
1421 | RTE_PIPELINE_ACTION_TABLE]; | |
1422 | ||
1423 | RTE_PIPELINE_STATS_TABLE_DROP1(p, | |
1424 | table->n_pkts_dropped_lkp_hit); | |
1425 | } | |
1426 | ||
1427 | /* Prepare for next iteration */ | |
1428 | p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE]; | |
1429 | table_id = table->table_next_id; | |
1430 | p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0; | |
1431 | } | |
1432 | ||
1433 | /* Table reserved action PORT */ | |
1434 | rte_pipeline_action_handler_port(p, | |
1435 | p->action_mask0[RTE_PIPELINE_ACTION_PORT]); | |
1436 | ||
1437 | /* Table reserved action PORT META */ | |
1438 | rte_pipeline_action_handler_port_meta(p, | |
1439 | p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]); | |
1440 | ||
1441 | /* Table reserved action DROP */ | |
1442 | rte_pipeline_action_handler_drop(p, | |
1443 | p->action_mask0[RTE_PIPELINE_ACTION_DROP]); | |
1444 | ||
1445 | /* Pick candidate for next port IN to serve */ | |
1446 | p->port_in_next = port_in->next; | |
1447 | ||
1448 | return (int) n_pkts; | |
1449 | } | |
1450 | ||
1451 | int | |
1452 | rte_pipeline_flush(struct rte_pipeline *p) | |
1453 | { | |
1454 | uint32_t port_id; | |
1455 | ||
1456 | /* Check input arguments */ | |
1457 | if (p == NULL) { | |
1458 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
1459 | __func__); | |
1460 | return -EINVAL; | |
1461 | } | |
1462 | ||
1463 | for (port_id = 0; port_id < p->num_ports_out; port_id++) { | |
1464 | struct rte_port_out *port = &p->ports_out[port_id]; | |
1465 | ||
1466 | if (port->ops.f_flush != NULL) | |
1467 | port->ops.f_flush(port->h_port); | |
1468 | } | |
1469 | ||
1470 | return 0; | |
1471 | } | |
1472 | ||
1473 | int | |
1474 | rte_pipeline_port_out_packet_insert(struct rte_pipeline *p, | |
1475 | uint32_t port_id, struct rte_mbuf *pkt) | |
1476 | { | |
1477 | struct rte_port_out *port_out = &p->ports_out[port_id]; | |
1478 | ||
1479 | port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */ | |
1480 | ||
1481 | return 0; | |
1482 | } | |
1483 | ||
1484 | int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p, | |
1485 | uint64_t pkts_mask) | |
1486 | { | |
1487 | pkts_mask &= p->pkts_mask; | |
1488 | p->pkts_mask &= ~pkts_mask; | |
1489 | ||
1490 | return 0; | |
1491 | } | |
1492 | ||
1493 | int rte_pipeline_ah_packet_drop(struct rte_pipeline *p, | |
1494 | uint64_t pkts_mask) | |
1495 | { | |
1496 | pkts_mask &= p->pkts_mask; | |
1497 | p->pkts_mask &= ~pkts_mask; | |
1498 | p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask; | |
1499 | ||
1500 | RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask); | |
1501 | return 0; | |
1502 | } | |
1503 | ||
1504 | int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id, | |
1505 | struct rte_pipeline_port_in_stats *stats, int clear) | |
1506 | { | |
1507 | struct rte_port_in *port; | |
1508 | int retval; | |
1509 | ||
1510 | if (p == NULL) { | |
1511 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
1512 | __func__); | |
1513 | return -EINVAL; | |
1514 | } | |
1515 | ||
1516 | if (port_id >= p->num_ports_in) { | |
1517 | RTE_LOG(ERR, PIPELINE, | |
1518 | "%s: port IN ID %u is out of range\n", | |
1519 | __func__, port_id); | |
1520 | return -EINVAL; | |
1521 | } | |
1522 | ||
1523 | port = &p->ports_in[port_id]; | |
1524 | ||
1525 | if (port->ops.f_stats != NULL) { | |
1526 | retval = port->ops.f_stats(port->h_port, &stats->stats, clear); | |
1527 | if (retval) | |
1528 | return retval; | |
1529 | } else if (stats != NULL) | |
1530 | memset(&stats->stats, 0, sizeof(stats->stats)); | |
1531 | ||
1532 | if (stats != NULL) | |
1533 | stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; | |
1534 | ||
1535 | if (clear != 0) | |
1536 | port->n_pkts_dropped_by_ah = 0; | |
1537 | ||
1538 | return 0; | |
1539 | } | |
1540 | ||
1541 | int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id, | |
1542 | struct rte_pipeline_port_out_stats *stats, int clear) | |
1543 | { | |
1544 | struct rte_port_out *port; | |
1545 | int retval; | |
1546 | ||
1547 | if (p == NULL) { | |
1548 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__); | |
1549 | return -EINVAL; | |
1550 | } | |
1551 | ||
1552 | if (port_id >= p->num_ports_out) { | |
1553 | RTE_LOG(ERR, PIPELINE, | |
1554 | "%s: port OUT ID %u is out of range\n", __func__, port_id); | |
1555 | return -EINVAL; | |
1556 | } | |
1557 | ||
1558 | port = &p->ports_out[port_id]; | |
1559 | if (port->ops.f_stats != NULL) { | |
1560 | retval = port->ops.f_stats(port->h_port, &stats->stats, clear); | |
1561 | if (retval != 0) | |
1562 | return retval; | |
1563 | } else if (stats != NULL) | |
1564 | memset(&stats->stats, 0, sizeof(stats->stats)); | |
1565 | ||
1566 | if (stats != NULL) | |
1567 | stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah; | |
1568 | ||
1569 | if (clear != 0) | |
1570 | port->n_pkts_dropped_by_ah = 0; | |
1571 | ||
1572 | return 0; | |
1573 | } | |
1574 | ||
1575 | int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id, | |
1576 | struct rte_pipeline_table_stats *stats, int clear) | |
1577 | { | |
1578 | struct rte_table *table; | |
1579 | int retval; | |
1580 | ||
1581 | if (p == NULL) { | |
1582 | RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", | |
1583 | __func__); | |
1584 | return -EINVAL; | |
1585 | } | |
1586 | ||
1587 | if (table_id >= p->num_tables) { | |
1588 | RTE_LOG(ERR, PIPELINE, | |
1589 | "%s: table %u is out of range\n", __func__, table_id); | |
1590 | return -EINVAL; | |
1591 | } | |
1592 | ||
1593 | table = &p->tables[table_id]; | |
1594 | if (table->ops.f_stats != NULL) { | |
1595 | retval = table->ops.f_stats(table->h_table, &stats->stats, clear); | |
1596 | if (retval != 0) | |
1597 | return retval; | |
1598 | } else if (stats != NULL) | |
1599 | memset(&stats->stats, 0, sizeof(stats->stats)); | |
1600 | ||
1601 | if (stats != NULL) { | |
1602 | stats->n_pkts_dropped_by_lkp_hit_ah = | |
1603 | table->n_pkts_dropped_by_lkp_hit_ah; | |
1604 | stats->n_pkts_dropped_by_lkp_miss_ah = | |
1605 | table->n_pkts_dropped_by_lkp_miss_ah; | |
1606 | stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit; | |
1607 | stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss; | |
1608 | } | |
1609 | ||
1610 | if (clear != 0) { | |
1611 | table->n_pkts_dropped_by_lkp_hit_ah = 0; | |
1612 | table->n_pkts_dropped_by_lkp_miss_ah = 0; | |
1613 | table->n_pkts_dropped_lkp_hit = 0; | |
1614 | table->n_pkts_dropped_lkp_miss = 0; | |
1615 | } | |
1616 | ||
1617 | return 0; | |
1618 | } |