]> git.proxmox.com Git - ceph.git/blob - ceph/src/seastar/dpdk/lib/librte_pipeline/rte_pipeline.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import 15.2.0 Octopus source
[ceph.git] / ceph / src / seastar / dpdk / lib / librte_pipeline / rte_pipeline.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
3 */
4
5 #include <string.h>
6 #include <stdio.h>
7
8 #include <rte_common.h>
9 #include <rte_memory.h>
10 #include <rte_cycles.h>
11 #include <rte_prefetch.h>
12 #include <rte_branch_prediction.h>
13 #include <rte_mbuf.h>
14 #include <rte_malloc.h>
15 #include <rte_string_fns.h>
16
17 #include "rte_pipeline.h"
18
19 #define RTE_TABLE_INVALID UINT32_MAX
20
21 #ifdef RTE_PIPELINE_STATS_COLLECT
22
23 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \
24 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
25
26 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \
27 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
28
29 #define RTE_PIPELINE_STATS_TABLE_DROP0(p) \
30 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
31
32 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \
33 ({ \
34 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \
35 mask ^= (p)->pkts_drop_mask; \
36 (counter) += __builtin_popcountll(mask); \
37 })
38
39 #else
40
41 #define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
42 #define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
43 #define RTE_PIPELINE_STATS_TABLE_DROP0(p)
44 #define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
45
46 #endif
47
48 struct rte_port_in {
49 /* Input parameters */
50 struct rte_port_in_ops ops;
51 rte_pipeline_port_in_action_handler f_action;
52 void *arg_ah;
53 uint32_t burst_size;
54
55 /* The table to which this port is connected */
56 uint32_t table_id;
57
58 /* Handle to low-level port */
59 void *h_port;
60
61 /* List of enabled ports */
62 struct rte_port_in *next;
63
64 /* Statistics */
65 uint64_t n_pkts_dropped_by_ah;
66 };
67
68 struct rte_port_out {
69 /* Input parameters */
70 struct rte_port_out_ops ops;
71 rte_pipeline_port_out_action_handler f_action;
72 void *arg_ah;
73
74 /* Handle to low-level port */
75 void *h_port;
76
77 /* Statistics */
78 uint64_t n_pkts_dropped_by_ah;
79 };
80
81 struct rte_table {
82 /* Input parameters */
83 struct rte_table_ops ops;
84 rte_pipeline_table_action_handler_hit f_action_hit;
85 rte_pipeline_table_action_handler_miss f_action_miss;
86 void *arg_ah;
87 struct rte_pipeline_table_entry *default_entry;
88 uint32_t entry_size;
89
90 uint32_t table_next_id;
91 uint32_t table_next_id_valid;
92
93 /* Handle to the low-level table object */
94 void *h_table;
95
96 /* Statistics */
97 uint64_t n_pkts_dropped_by_lkp_hit_ah;
98 uint64_t n_pkts_dropped_by_lkp_miss_ah;
99 uint64_t n_pkts_dropped_lkp_hit;
100 uint64_t n_pkts_dropped_lkp_miss;
101 };
102
103 #define RTE_PIPELINE_MAX_NAME_SZ 124
104
105 struct rte_pipeline {
106 /* Input parameters */
107 char name[RTE_PIPELINE_MAX_NAME_SZ];
108 int socket_id;
109 uint32_t offset_port_id;
110
111 /* Internal tables */
112 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
113 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
114 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
115
116 /* Occupancy of internal tables */
117 uint32_t num_ports_in;
118 uint32_t num_ports_out;
119 uint32_t num_tables;
120
121 /* List of enabled ports */
122 uint64_t enabled_port_in_mask;
123 struct rte_port_in *port_in_next;
124
125 /* Pipeline run structures */
126 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
127 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
128 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
129 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
130 uint64_t pkts_mask;
131 uint64_t n_pkts_ah_drop;
132 uint64_t pkts_drop_mask;
133 } __rte_cache_aligned;
134
135 static inline uint32_t
136 rte_mask_get_next(uint64_t mask, uint32_t pos)
137 {
138 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
139 (mask >> ((pos + 1) & 0x3F));
140 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
141 }
142
143 static inline uint32_t
144 rte_mask_get_prev(uint64_t mask, uint32_t pos)
145 {
146 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
147 (mask << ((64 - pos) & 0x3F));
148 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
149 }
150
151 static void
152 rte_pipeline_table_free(struct rte_table *table);
153
154 static void
155 rte_pipeline_port_in_free(struct rte_port_in *port);
156
157 static void
158 rte_pipeline_port_out_free(struct rte_port_out *port);
159
160 /*
161 * Pipeline
162 *
163 */
164 static int
165 rte_pipeline_check_params(struct rte_pipeline_params *params)
166 {
167 if (params == NULL) {
168 RTE_LOG(ERR, PIPELINE,
169 "%s: Incorrect value for parameter params\n", __func__);
170 return -EINVAL;
171 }
172
173 /* name */
174 if (params->name == NULL) {
175 RTE_LOG(ERR, PIPELINE,
176 "%s: Incorrect value for parameter name\n", __func__);
177 return -EINVAL;
178 }
179
180 /* socket */
181 if (params->socket_id < 0) {
182 RTE_LOG(ERR, PIPELINE,
183 "%s: Incorrect value for parameter socket_id\n",
184 __func__);
185 return -EINVAL;
186 }
187
188 return 0;
189 }
190
191 struct rte_pipeline *
192 rte_pipeline_create(struct rte_pipeline_params *params)
193 {
194 struct rte_pipeline *p;
195 int status;
196
197 /* Check input parameters */
198 status = rte_pipeline_check_params(params);
199 if (status != 0) {
200 RTE_LOG(ERR, PIPELINE,
201 "%s: Pipeline params check failed (%d)\n",
202 __func__, status);
203 return NULL;
204 }
205
206 /* Allocate memory for the pipeline on requested socket */
207 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
208 RTE_CACHE_LINE_SIZE, params->socket_id);
209
210 if (p == NULL) {
211 RTE_LOG(ERR, PIPELINE,
212 "%s: Pipeline memory allocation failed\n", __func__);
213 return NULL;
214 }
215
216 /* Save input parameters */
217 strlcpy(p->name, params->name, RTE_PIPELINE_MAX_NAME_SZ);
218 p->socket_id = params->socket_id;
219 p->offset_port_id = params->offset_port_id;
220
221 /* Initialize pipeline internal data structure */
222 p->num_ports_in = 0;
223 p->num_ports_out = 0;
224 p->num_tables = 0;
225 p->enabled_port_in_mask = 0;
226 p->port_in_next = NULL;
227 p->pkts_mask = 0;
228 p->n_pkts_ah_drop = 0;
229
230 return p;
231 }
232
233 int
234 rte_pipeline_free(struct rte_pipeline *p)
235 {
236 uint32_t i;
237
238 /* Check input parameters */
239 if (p == NULL) {
240 RTE_LOG(ERR, PIPELINE,
241 "%s: rte_pipeline parameter is NULL\n", __func__);
242 return -EINVAL;
243 }
244
245 /* Free input ports */
246 for (i = 0; i < p->num_ports_in; i++) {
247 struct rte_port_in *port = &p->ports_in[i];
248
249 rte_pipeline_port_in_free(port);
250 }
251
252 /* Free tables */
253 for (i = 0; i < p->num_tables; i++) {
254 struct rte_table *table = &p->tables[i];
255
256 rte_pipeline_table_free(table);
257 }
258
259 /* Free output ports */
260 for (i = 0; i < p->num_ports_out; i++) {
261 struct rte_port_out *port = &p->ports_out[i];
262
263 rte_pipeline_port_out_free(port);
264 }
265
266 /* Free pipeline memory */
267 rte_free(p);
268
269 return 0;
270 }
271
272 /*
273 * Table
274 *
275 */
276 static int
277 rte_table_check_params(struct rte_pipeline *p,
278 struct rte_pipeline_table_params *params,
279 uint32_t *table_id)
280 {
281 if (p == NULL) {
282 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
283 __func__);
284 return -EINVAL;
285 }
286 if (params == NULL) {
287 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
288 __func__);
289 return -EINVAL;
290 }
291 if (table_id == NULL) {
292 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
293 __func__);
294 return -EINVAL;
295 }
296
297 /* ops */
298 if (params->ops == NULL) {
299 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
300 __func__);
301 return -EINVAL;
302 }
303
304 if (params->ops->f_create == NULL) {
305 RTE_LOG(ERR, PIPELINE,
306 "%s: f_create function pointer is NULL\n", __func__);
307 return -EINVAL;
308 }
309
310 if (params->ops->f_lookup == NULL) {
311 RTE_LOG(ERR, PIPELINE,
312 "%s: f_lookup function pointer is NULL\n", __func__);
313 return -EINVAL;
314 }
315
316 /* De we have room for one more table? */
317 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
318 RTE_LOG(ERR, PIPELINE,
319 "%s: Incorrect value for num_tables parameter\n",
320 __func__);
321 return -EINVAL;
322 }
323
324 return 0;
325 }
326
327 int
328 rte_pipeline_table_create(struct rte_pipeline *p,
329 struct rte_pipeline_table_params *params,
330 uint32_t *table_id)
331 {
332 struct rte_table *table;
333 struct rte_pipeline_table_entry *default_entry;
334 void *h_table;
335 uint32_t entry_size, id;
336 int status;
337
338 /* Check input arguments */
339 status = rte_table_check_params(p, params, table_id);
340 if (status != 0)
341 return status;
342
343 id = p->num_tables;
344 table = &p->tables[id];
345
346 /* Allocate space for the default table entry */
347 entry_size = sizeof(struct rte_pipeline_table_entry) +
348 params->action_data_size;
349 default_entry = rte_zmalloc_socket(
350 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
351 if (default_entry == NULL) {
352 RTE_LOG(ERR, PIPELINE,
353 "%s: Failed to allocate default entry\n", __func__);
354 return -EINVAL;
355 }
356
357 /* Create the table */
358 h_table = params->ops->f_create(params->arg_create, p->socket_id,
359 entry_size);
360 if (h_table == NULL) {
361 rte_free(default_entry);
362 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
363 return -EINVAL;
364 }
365
366 /* Commit current table to the pipeline */
367 p->num_tables++;
368 *table_id = id;
369
370 /* Save input parameters */
371 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
372 table->f_action_hit = params->f_action_hit;
373 table->f_action_miss = params->f_action_miss;
374 table->arg_ah = params->arg_ah;
375 table->entry_size = entry_size;
376
377 /* Clear the lookup miss actions (to be set later through API) */
378 table->default_entry = default_entry;
379 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
380
381 /* Initialize table internal data structure */
382 table->h_table = h_table;
383 table->table_next_id = 0;
384 table->table_next_id_valid = 0;
385
386 return 0;
387 }
388
389 void
390 rte_pipeline_table_free(struct rte_table *table)
391 {
392 if (table->ops.f_free != NULL)
393 table->ops.f_free(table->h_table);
394
395 rte_free(table->default_entry);
396 }
397
398 int
399 rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
400 uint32_t table_id,
401 struct rte_pipeline_table_entry *default_entry,
402 struct rte_pipeline_table_entry **default_entry_ptr)
403 {
404 struct rte_table *table;
405
406 /* Check input arguments */
407 if (p == NULL) {
408 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
409 __func__);
410 return -EINVAL;
411 }
412
413 if (default_entry == NULL) {
414 RTE_LOG(ERR, PIPELINE,
415 "%s: default_entry parameter is NULL\n", __func__);
416 return -EINVAL;
417 }
418
419 if (table_id >= p->num_tables) {
420 RTE_LOG(ERR, PIPELINE,
421 "%s: table_id %d out of range\n", __func__, table_id);
422 return -EINVAL;
423 }
424
425 table = &p->tables[table_id];
426
427 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
428 table->table_next_id_valid &&
429 (default_entry->table_id != table->table_next_id)) {
430 RTE_LOG(ERR, PIPELINE,
431 "%s: Tree-like topologies not allowed\n", __func__);
432 return -EINVAL;
433 }
434
435 /* Set the lookup miss actions */
436 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
437 (table->table_next_id_valid == 0)) {
438 table->table_next_id = default_entry->table_id;
439 table->table_next_id_valid = 1;
440 }
441
442 memcpy(table->default_entry, default_entry, table->entry_size);
443
444 *default_entry_ptr = table->default_entry;
445 return 0;
446 }
447
448 int
449 rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
450 uint32_t table_id,
451 struct rte_pipeline_table_entry *entry)
452 {
453 struct rte_table *table;
454
455 /* Check input arguments */
456 if (p == NULL) {
457 RTE_LOG(ERR, PIPELINE,
458 "%s: pipeline parameter is NULL\n", __func__);
459 return -EINVAL;
460 }
461
462 if (table_id >= p->num_tables) {
463 RTE_LOG(ERR, PIPELINE,
464 "%s: table_id %d out of range\n", __func__, table_id);
465 return -EINVAL;
466 }
467
468 table = &p->tables[table_id];
469
470 /* Save the current contents of the default entry */
471 if (entry)
472 memcpy(entry, table->default_entry, table->entry_size);
473
474 /* Clear the lookup miss actions */
475 memset(table->default_entry, 0, table->entry_size);
476 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
477
478 return 0;
479 }
480
481 int
482 rte_pipeline_table_entry_add(struct rte_pipeline *p,
483 uint32_t table_id,
484 void *key,
485 struct rte_pipeline_table_entry *entry,
486 int *key_found,
487 struct rte_pipeline_table_entry **entry_ptr)
488 {
489 struct rte_table *table;
490
491 /* Check input arguments */
492 if (p == NULL) {
493 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
494 __func__);
495 return -EINVAL;
496 }
497
498 if (key == NULL) {
499 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
500 return -EINVAL;
501 }
502
503 if (entry == NULL) {
504 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
505 __func__);
506 return -EINVAL;
507 }
508
509 if (table_id >= p->num_tables) {
510 RTE_LOG(ERR, PIPELINE,
511 "%s: table_id %d out of range\n", __func__, table_id);
512 return -EINVAL;
513 }
514
515 table = &p->tables[table_id];
516
517 if (table->ops.f_add == NULL) {
518 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
519 __func__);
520 return -EINVAL;
521 }
522
523 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
524 table->table_next_id_valid &&
525 (entry->table_id != table->table_next_id)) {
526 RTE_LOG(ERR, PIPELINE,
527 "%s: Tree-like topologies not allowed\n", __func__);
528 return -EINVAL;
529 }
530
531 /* Add entry */
532 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
533 (table->table_next_id_valid == 0)) {
534 table->table_next_id = entry->table_id;
535 table->table_next_id_valid = 1;
536 }
537
538 return (table->ops.f_add)(table->h_table, key, (void *) entry,
539 key_found, (void **) entry_ptr);
540 }
541
542 int
543 rte_pipeline_table_entry_delete(struct rte_pipeline *p,
544 uint32_t table_id,
545 void *key,
546 int *key_found,
547 struct rte_pipeline_table_entry *entry)
548 {
549 struct rte_table *table;
550
551 /* Check input arguments */
552 if (p == NULL) {
553 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
554 __func__);
555 return -EINVAL;
556 }
557
558 if (key == NULL) {
559 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
560 __func__);
561 return -EINVAL;
562 }
563
564 if (table_id >= p->num_tables) {
565 RTE_LOG(ERR, PIPELINE,
566 "%s: table_id %d out of range\n", __func__, table_id);
567 return -EINVAL;
568 }
569
570 table = &p->tables[table_id];
571
572 if (table->ops.f_delete == NULL) {
573 RTE_LOG(ERR, PIPELINE,
574 "%s: f_delete function pointer NULL\n", __func__);
575 return -EINVAL;
576 }
577
578 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
579 }
580
581 int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
582 uint32_t table_id,
583 void **keys,
584 struct rte_pipeline_table_entry **entries,
585 uint32_t n_keys,
586 int *key_found,
587 struct rte_pipeline_table_entry **entries_ptr)
588 {
589 struct rte_table *table;
590 uint32_t i;
591
592 /* Check input arguments */
593 if (p == NULL) {
594 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
595 __func__);
596 return -EINVAL;
597 }
598
599 if (keys == NULL) {
600 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
601 return -EINVAL;
602 }
603
604 if (entries == NULL) {
605 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
606 __func__);
607 return -EINVAL;
608 }
609
610 if (table_id >= p->num_tables) {
611 RTE_LOG(ERR, PIPELINE,
612 "%s: table_id %d out of range\n", __func__, table_id);
613 return -EINVAL;
614 }
615
616 table = &p->tables[table_id];
617
618 if (table->ops.f_add_bulk == NULL) {
619 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
620 __func__);
621 return -EINVAL;
622 }
623
624 for (i = 0; i < n_keys; i++) {
625 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
626 table->table_next_id_valid &&
627 (entries[i]->table_id != table->table_next_id)) {
628 RTE_LOG(ERR, PIPELINE,
629 "%s: Tree-like topologies not allowed\n", __func__);
630 return -EINVAL;
631 }
632 }
633
634 /* Add entry */
635 for (i = 0; i < n_keys; i++) {
636 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
637 (table->table_next_id_valid == 0)) {
638 table->table_next_id = entries[i]->table_id;
639 table->table_next_id_valid = 1;
640 }
641 }
642
643 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
644 n_keys, key_found, (void **) entries_ptr);
645 }
646
647 int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
648 uint32_t table_id,
649 void **keys,
650 uint32_t n_keys,
651 int *key_found,
652 struct rte_pipeline_table_entry **entries)
653 {
654 struct rte_table *table;
655
656 /* Check input arguments */
657 if (p == NULL) {
658 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
659 __func__);
660 return -EINVAL;
661 }
662
663 if (keys == NULL) {
664 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
665 __func__);
666 return -EINVAL;
667 }
668
669 if (table_id >= p->num_tables) {
670 RTE_LOG(ERR, PIPELINE,
671 "%s: table_id %d out of range\n", __func__, table_id);
672 return -EINVAL;
673 }
674
675 table = &p->tables[table_id];
676
677 if (table->ops.f_delete_bulk == NULL) {
678 RTE_LOG(ERR, PIPELINE,
679 "%s: f_delete function pointer NULL\n", __func__);
680 return -EINVAL;
681 }
682
683 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
684 (void **) entries);
685 }
686
687 /*
688 * Port
689 *
690 */
691 static int
692 rte_pipeline_port_in_check_params(struct rte_pipeline *p,
693 struct rte_pipeline_port_in_params *params,
694 uint32_t *port_id)
695 {
696 if (p == NULL) {
697 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
698 __func__);
699 return -EINVAL;
700 }
701 if (params == NULL) {
702 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
703 return -EINVAL;
704 }
705 if (port_id == NULL) {
706 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
707 __func__);
708 return -EINVAL;
709 }
710
711 /* ops */
712 if (params->ops == NULL) {
713 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
714 __func__);
715 return -EINVAL;
716 }
717
718 if (params->ops->f_create == NULL) {
719 RTE_LOG(ERR, PIPELINE,
720 "%s: f_create function pointer NULL\n", __func__);
721 return -EINVAL;
722 }
723
724 if (params->ops->f_rx == NULL) {
725 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
726 __func__);
727 return -EINVAL;
728 }
729
730 /* burst_size */
731 if ((params->burst_size == 0) ||
732 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
733 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
734 __func__);
735 return -EINVAL;
736 }
737
738 /* Do we have room for one more port? */
739 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
740 RTE_LOG(ERR, PIPELINE,
741 "%s: invalid value for num_ports_in\n", __func__);
742 return -EINVAL;
743 }
744
745 return 0;
746 }
747
748 static int
749 rte_pipeline_port_out_check_params(struct rte_pipeline *p,
750 struct rte_pipeline_port_out_params *params,
751 uint32_t *port_id)
752 {
753 if (p == NULL) {
754 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
755 __func__);
756 return -EINVAL;
757 }
758
759 if (params == NULL) {
760 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
761 return -EINVAL;
762 }
763
764 if (port_id == NULL) {
765 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
766 __func__);
767 return -EINVAL;
768 }
769
770 /* ops */
771 if (params->ops == NULL) {
772 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
773 __func__);
774 return -EINVAL;
775 }
776
777 if (params->ops->f_create == NULL) {
778 RTE_LOG(ERR, PIPELINE,
779 "%s: f_create function pointer NULL\n", __func__);
780 return -EINVAL;
781 }
782
783 if (params->ops->f_tx == NULL) {
784 RTE_LOG(ERR, PIPELINE,
785 "%s: f_tx function pointer NULL\n", __func__);
786 return -EINVAL;
787 }
788
789 if (params->ops->f_tx_bulk == NULL) {
790 RTE_LOG(ERR, PIPELINE,
791 "%s: f_tx_bulk function pointer NULL\n", __func__);
792 return -EINVAL;
793 }
794
795 /* Do we have room for one more port? */
796 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
797 RTE_LOG(ERR, PIPELINE,
798 "%s: invalid value for num_ports_out\n", __func__);
799 return -EINVAL;
800 }
801
802 return 0;
803 }
804
805 int
806 rte_pipeline_port_in_create(struct rte_pipeline *p,
807 struct rte_pipeline_port_in_params *params,
808 uint32_t *port_id)
809 {
810 struct rte_port_in *port;
811 void *h_port;
812 uint32_t id;
813 int status;
814
815 /* Check input arguments */
816 status = rte_pipeline_port_in_check_params(p, params, port_id);
817 if (status != 0)
818 return status;
819
820 id = p->num_ports_in;
821 port = &p->ports_in[id];
822
823 /* Create the port */
824 h_port = params->ops->f_create(params->arg_create, p->socket_id);
825 if (h_port == NULL) {
826 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
827 return -EINVAL;
828 }
829
830 /* Commit current table to the pipeline */
831 p->num_ports_in++;
832 *port_id = id;
833
834 /* Save input parameters */
835 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
836 port->f_action = params->f_action;
837 port->arg_ah = params->arg_ah;
838 port->burst_size = params->burst_size;
839
840 /* Initialize port internal data structure */
841 port->table_id = RTE_TABLE_INVALID;
842 port->h_port = h_port;
843 port->next = NULL;
844
845 return 0;
846 }
847
848 void
849 rte_pipeline_port_in_free(struct rte_port_in *port)
850 {
851 if (port->ops.f_free != NULL)
852 port->ops.f_free(port->h_port);
853 }
854
855 int
856 rte_pipeline_port_out_create(struct rte_pipeline *p,
857 struct rte_pipeline_port_out_params *params,
858 uint32_t *port_id)
859 {
860 struct rte_port_out *port;
861 void *h_port;
862 uint32_t id;
863 int status;
864
865 /* Check input arguments */
866 status = rte_pipeline_port_out_check_params(p, params, port_id);
867 if (status != 0)
868 return status;
869
870 id = p->num_ports_out;
871 port = &p->ports_out[id];
872
873 /* Create the port */
874 h_port = params->ops->f_create(params->arg_create, p->socket_id);
875 if (h_port == NULL) {
876 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
877 return -EINVAL;
878 }
879
880 /* Commit current table to the pipeline */
881 p->num_ports_out++;
882 *port_id = id;
883
884 /* Save input parameters */
885 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
886 port->f_action = params->f_action;
887 port->arg_ah = params->arg_ah;
888
889 /* Initialize port internal data structure */
890 port->h_port = h_port;
891
892 return 0;
893 }
894
895 void
896 rte_pipeline_port_out_free(struct rte_port_out *port)
897 {
898 if (port->ops.f_free != NULL)
899 port->ops.f_free(port->h_port);
900 }
901
902 int
903 rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
904 uint32_t port_id,
905 uint32_t table_id)
906 {
907 struct rte_port_in *port;
908
909 /* Check input arguments */
910 if (p == NULL) {
911 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
912 __func__);
913 return -EINVAL;
914 }
915
916 if (port_id >= p->num_ports_in) {
917 RTE_LOG(ERR, PIPELINE,
918 "%s: port IN ID %u is out of range\n",
919 __func__, port_id);
920 return -EINVAL;
921 }
922
923 if (table_id >= p->num_tables) {
924 RTE_LOG(ERR, PIPELINE,
925 "%s: Table ID %u is out of range\n",
926 __func__, table_id);
927 return -EINVAL;
928 }
929
930 port = &p->ports_in[port_id];
931 port->table_id = table_id;
932
933 return 0;
934 }
935
936 int
937 rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
938 {
939 struct rte_port_in *port, *port_prev, *port_next;
940 uint64_t port_mask;
941 uint32_t port_prev_id, port_next_id;
942
943 /* Check input arguments */
944 if (p == NULL) {
945 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
946 __func__);
947 return -EINVAL;
948 }
949
950 if (port_id >= p->num_ports_in) {
951 RTE_LOG(ERR, PIPELINE,
952 "%s: port IN ID %u is out of range\n",
953 __func__, port_id);
954 return -EINVAL;
955 }
956
957 port = &p->ports_in[port_id];
958
959 /* Return if current input port is already enabled */
960 port_mask = 1LLU << port_id;
961 if (p->enabled_port_in_mask & port_mask)
962 return 0;
963
964 p->enabled_port_in_mask |= port_mask;
965
966 /* Add current input port to the pipeline chain of enabled ports */
967 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
968 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
969
970 port_prev = &p->ports_in[port_prev_id];
971 port_next = &p->ports_in[port_next_id];
972
973 port_prev->next = port;
974 port->next = port_next;
975
976 /* Check if list of enabled ports was previously empty */
977 if (p->enabled_port_in_mask == port_mask)
978 p->port_in_next = port;
979
980 return 0;
981 }
982
983 int
984 rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
985 {
986 struct rte_port_in *port, *port_prev, *port_next;
987 uint64_t port_mask;
988 uint32_t port_prev_id, port_next_id;
989
990 /* Check input arguments */
991 if (p == NULL) {
992 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
993 __func__);
994 return -EINVAL;
995 }
996
997 if (port_id >= p->num_ports_in) {
998 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
999 __func__, port_id);
1000 return -EINVAL;
1001 }
1002
1003 port = &p->ports_in[port_id];
1004
1005 /* Return if current input port is already disabled */
1006 port_mask = 1LLU << port_id;
1007 if ((p->enabled_port_in_mask & port_mask) == 0)
1008 return 0;
1009
1010 p->enabled_port_in_mask &= ~port_mask;
1011
1012 /* Return if no other enabled ports */
1013 if (p->enabled_port_in_mask == 0) {
1014 p->port_in_next = NULL;
1015
1016 return 0;
1017 }
1018
1019 /* Add current input port to the pipeline chain of enabled ports */
1020 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1021 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1022
1023 port_prev = &p->ports_in[port_prev_id];
1024 port_next = &p->ports_in[port_next_id];
1025
1026 port_prev->next = port_next;
1027
1028 /* Check if the port which has just been disabled is next to serve */
1029 if (port == p->port_in_next)
1030 p->port_in_next = port_next;
1031
1032 return 0;
1033 }
1034
1035 /*
1036 * Pipeline run-time
1037 *
1038 */
1039 int
1040 rte_pipeline_check(struct rte_pipeline *p)
1041 {
1042 uint32_t port_in_id;
1043
1044 /* Check input arguments */
1045 if (p == NULL) {
1046 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1047 __func__);
1048 return -EINVAL;
1049 }
1050
1051 /* Check that pipeline has at least one input port, one table and one
1052 output port */
1053 if (p->num_ports_in == 0) {
1054 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1055 __func__);
1056 return -EINVAL;
1057 }
1058 if (p->num_tables == 0) {
1059 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1060 __func__);
1061 return -EINVAL;
1062 }
1063 if (p->num_ports_out == 0) {
1064 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1065 __func__);
1066 return -EINVAL;
1067 }
1068
1069 /* Check that all input ports are connected */
1070 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1071 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1072
1073 if (port_in->table_id == RTE_TABLE_INVALID) {
1074 RTE_LOG(ERR, PIPELINE,
1075 "%s: Port IN ID %u is not connected\n",
1076 __func__, port_in_id);
1077 return -EINVAL;
1078 }
1079 }
1080
1081 return 0;
1082 }
1083
1084 static inline void
1085 rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1086 {
1087 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1088 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1089 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1090 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1091
1092 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1093 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1094 uint32_t i;
1095
1096 for (i = 0; i < n_pkts; i++) {
1097 uint64_t pkt_mask = 1LLU << i;
1098 uint32_t pos = p->entries[i]->action;
1099
1100 p->action_mask1[pos] |= pkt_mask;
1101 }
1102 } else {
1103 uint32_t i;
1104
1105 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1106 uint64_t pkt_mask = 1LLU << i;
1107 uint32_t pos;
1108
1109 if ((pkt_mask & pkts_mask) == 0)
1110 continue;
1111
1112 pos = p->entries[i]->action;
1113 p->action_mask1[pos] |= pkt_mask;
1114 }
1115 }
1116 }
1117
1118 static inline void
1119 rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1120 uint64_t pkts_mask, uint32_t port_id)
1121 {
1122 struct rte_port_out *port_out = &p->ports_out[port_id];
1123
1124 p->pkts_mask = pkts_mask;
1125
1126 /* Output port user actions */
1127 if (port_out->f_action != NULL) {
1128 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1129
1130 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1131 port_out->n_pkts_dropped_by_ah);
1132 }
1133
1134 /* Output port TX */
1135 if (p->pkts_mask != 0)
1136 port_out->ops.f_tx_bulk(port_out->h_port,
1137 p->pkts,
1138 p->pkts_mask);
1139 }
1140
1141 static inline void
1142 rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1143 {
1144 p->pkts_mask = pkts_mask;
1145
1146 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1147 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1148 uint32_t i;
1149
1150 for (i = 0; i < n_pkts; i++) {
1151 struct rte_mbuf *pkt = p->pkts[i];
1152 uint32_t port_out_id = p->entries[i]->port_id;
1153 struct rte_port_out *port_out =
1154 &p->ports_out[port_out_id];
1155
1156 /* Output port user actions */
1157 if (port_out->f_action == NULL) /* Output port TX */
1158 port_out->ops.f_tx(port_out->h_port, pkt);
1159 else {
1160 uint64_t pkt_mask = 1LLU << i;
1161
1162 port_out->f_action(p,
1163 p->pkts,
1164 pkt_mask,
1165 port_out->arg_ah);
1166
1167 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1168 port_out->n_pkts_dropped_by_ah);
1169
1170 /* Output port TX */
1171 if (pkt_mask & p->pkts_mask)
1172 port_out->ops.f_tx(port_out->h_port,
1173 pkt);
1174 }
1175 }
1176 } else {
1177 uint32_t i;
1178
1179 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1180 uint64_t pkt_mask = 1LLU << i;
1181 struct rte_mbuf *pkt;
1182 struct rte_port_out *port_out;
1183 uint32_t port_out_id;
1184
1185 if ((pkt_mask & pkts_mask) == 0)
1186 continue;
1187
1188 pkt = p->pkts[i];
1189 port_out_id = p->entries[i]->port_id;
1190 port_out = &p->ports_out[port_out_id];
1191
1192 /* Output port user actions */
1193 if (port_out->f_action == NULL) /* Output port TX */
1194 port_out->ops.f_tx(port_out->h_port, pkt);
1195 else {
1196 port_out->f_action(p,
1197 p->pkts,
1198 pkt_mask,
1199 port_out->arg_ah);
1200
1201 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1202 port_out->n_pkts_dropped_by_ah);
1203
1204 /* Output port TX */
1205 if (pkt_mask & p->pkts_mask)
1206 port_out->ops.f_tx(port_out->h_port,
1207 pkt);
1208 }
1209 }
1210 }
1211 }
1212
1213 static inline void
1214 rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1215 uint64_t pkts_mask)
1216 {
1217 p->pkts_mask = pkts_mask;
1218
1219 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1220 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1221 uint32_t i;
1222
1223 for (i = 0; i < n_pkts; i++) {
1224 struct rte_mbuf *pkt = p->pkts[i];
1225 uint32_t port_out_id =
1226 RTE_MBUF_METADATA_UINT32(pkt,
1227 p->offset_port_id);
1228 struct rte_port_out *port_out = &p->ports_out[
1229 port_out_id];
1230
1231 /* Output port user actions */
1232 if (port_out->f_action == NULL) /* Output port TX */
1233 port_out->ops.f_tx(port_out->h_port, pkt);
1234 else {
1235 uint64_t pkt_mask = 1LLU << i;
1236
1237 port_out->f_action(p,
1238 p->pkts,
1239 pkt_mask,
1240 port_out->arg_ah);
1241
1242 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1243 port_out->n_pkts_dropped_by_ah);
1244
1245 /* Output port TX */
1246 if (pkt_mask & p->pkts_mask)
1247 port_out->ops.f_tx(port_out->h_port,
1248 pkt);
1249 }
1250 }
1251 } else {
1252 uint32_t i;
1253
1254 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1255 uint64_t pkt_mask = 1LLU << i;
1256 struct rte_mbuf *pkt;
1257 struct rte_port_out *port_out;
1258 uint32_t port_out_id;
1259
1260 if ((pkt_mask & pkts_mask) == 0)
1261 continue;
1262
1263 pkt = p->pkts[i];
1264 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1265 p->offset_port_id);
1266 port_out = &p->ports_out[port_out_id];
1267
1268 /* Output port user actions */
1269 if (port_out->f_action == NULL) /* Output port TX */
1270 port_out->ops.f_tx(port_out->h_port, pkt);
1271 else {
1272 port_out->f_action(p,
1273 p->pkts,
1274 pkt_mask,
1275 port_out->arg_ah);
1276
1277 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1278 port_out->n_pkts_dropped_by_ah);
1279
1280 /* Output port TX */
1281 if (pkt_mask & p->pkts_mask)
1282 port_out->ops.f_tx(port_out->h_port,
1283 pkt);
1284 }
1285 }
1286 }
1287 }
1288
1289 static inline void
1290 rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1291 {
1292 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1293 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1294 uint32_t i;
1295
1296 for (i = 0; i < n_pkts; i++)
1297 rte_pktmbuf_free(p->pkts[i]);
1298 } else {
1299 uint32_t i;
1300
1301 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1302 uint64_t pkt_mask = 1LLU << i;
1303
1304 if ((pkt_mask & pkts_mask) == 0)
1305 continue;
1306
1307 rte_pktmbuf_free(p->pkts[i]);
1308 }
1309 }
1310 }
1311
1312 int
1313 rte_pipeline_run(struct rte_pipeline *p)
1314 {
1315 struct rte_port_in *port_in = p->port_in_next;
1316 uint32_t n_pkts, table_id;
1317
1318 if (port_in == NULL)
1319 return 0;
1320
1321 /* Input port RX */
1322 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1323 port_in->burst_size);
1324 if (n_pkts == 0) {
1325 p->port_in_next = port_in->next;
1326 return 0;
1327 }
1328
1329 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1330 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1331 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1332 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1333 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1334
1335 /* Input port user actions */
1336 if (port_in->f_action != NULL) {
1337 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1338
1339 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1340 port_in->n_pkts_dropped_by_ah);
1341 }
1342
1343 /* Table */
1344 for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1345 struct rte_table *table;
1346 uint64_t lookup_hit_mask, lookup_miss_mask;
1347
1348 /* Lookup */
1349 table = &p->tables[table_id];
1350 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1351 &lookup_hit_mask, (void **) p->entries);
1352 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1353
1354 /* Lookup miss */
1355 if (lookup_miss_mask != 0) {
1356 struct rte_pipeline_table_entry *default_entry =
1357 table->default_entry;
1358
1359 p->pkts_mask = lookup_miss_mask;
1360
1361 /* Table user actions */
1362 if (table->f_action_miss != NULL) {
1363 table->f_action_miss(p,
1364 p->pkts,
1365 lookup_miss_mask,
1366 default_entry,
1367 table->arg_ah);
1368
1369 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1370 table->n_pkts_dropped_by_lkp_miss_ah);
1371 }
1372
1373 /* Table reserved actions */
1374 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1375 (p->pkts_mask != 0))
1376 rte_pipeline_action_handler_port_bulk(p,
1377 p->pkts_mask,
1378 default_entry->port_id);
1379 else {
1380 uint32_t pos = default_entry->action;
1381
1382 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1383
1384 p->action_mask0[pos] |= p->pkts_mask;
1385
1386 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1387 table->n_pkts_dropped_lkp_miss);
1388 }
1389 }
1390
1391 /* Lookup hit */
1392 if (lookup_hit_mask != 0) {
1393 p->pkts_mask = lookup_hit_mask;
1394
1395 /* Table user actions */
1396 if (table->f_action_hit != NULL) {
1397 table->f_action_hit(p,
1398 p->pkts,
1399 lookup_hit_mask,
1400 p->entries,
1401 table->arg_ah);
1402
1403 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1404 table->n_pkts_dropped_by_lkp_hit_ah);
1405 }
1406
1407 /* Table reserved actions */
1408 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1409 rte_pipeline_compute_masks(p, p->pkts_mask);
1410 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1411 p->action_mask1[
1412 RTE_PIPELINE_ACTION_DROP];
1413 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1414 p->action_mask1[
1415 RTE_PIPELINE_ACTION_PORT];
1416 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1417 p->action_mask1[
1418 RTE_PIPELINE_ACTION_PORT_META];
1419 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1420 p->action_mask1[
1421 RTE_PIPELINE_ACTION_TABLE];
1422
1423 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1424 table->n_pkts_dropped_lkp_hit);
1425 }
1426
1427 /* Prepare for next iteration */
1428 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1429 table_id = table->table_next_id;
1430 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1431 }
1432
1433 /* Table reserved action PORT */
1434 rte_pipeline_action_handler_port(p,
1435 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1436
1437 /* Table reserved action PORT META */
1438 rte_pipeline_action_handler_port_meta(p,
1439 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1440
1441 /* Table reserved action DROP */
1442 rte_pipeline_action_handler_drop(p,
1443 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1444
1445 /* Pick candidate for next port IN to serve */
1446 p->port_in_next = port_in->next;
1447
1448 return (int) n_pkts;
1449 }
1450
1451 int
1452 rte_pipeline_flush(struct rte_pipeline *p)
1453 {
1454 uint32_t port_id;
1455
1456 /* Check input arguments */
1457 if (p == NULL) {
1458 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1459 __func__);
1460 return -EINVAL;
1461 }
1462
1463 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1464 struct rte_port_out *port = &p->ports_out[port_id];
1465
1466 if (port->ops.f_flush != NULL)
1467 port->ops.f_flush(port->h_port);
1468 }
1469
1470 return 0;
1471 }
1472
1473 int
1474 rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1475 uint32_t port_id, struct rte_mbuf *pkt)
1476 {
1477 struct rte_port_out *port_out = &p->ports_out[port_id];
1478
1479 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1480
1481 return 0;
1482 }
1483
1484 int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1485 uint64_t pkts_mask)
1486 {
1487 pkts_mask &= p->pkts_mask;
1488 p->pkts_mask &= ~pkts_mask;
1489
1490 return 0;
1491 }
1492
1493 int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1494 uint64_t pkts_mask)
1495 {
1496 pkts_mask &= p->pkts_mask;
1497 p->pkts_mask &= ~pkts_mask;
1498 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1499
1500 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1501 return 0;
1502 }
1503
1504 int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1505 struct rte_pipeline_port_in_stats *stats, int clear)
1506 {
1507 struct rte_port_in *port;
1508 int retval;
1509
1510 if (p == NULL) {
1511 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1512 __func__);
1513 return -EINVAL;
1514 }
1515
1516 if (port_id >= p->num_ports_in) {
1517 RTE_LOG(ERR, PIPELINE,
1518 "%s: port IN ID %u is out of range\n",
1519 __func__, port_id);
1520 return -EINVAL;
1521 }
1522
1523 port = &p->ports_in[port_id];
1524
1525 if (port->ops.f_stats != NULL) {
1526 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1527 if (retval)
1528 return retval;
1529 } else if (stats != NULL)
1530 memset(&stats->stats, 0, sizeof(stats->stats));
1531
1532 if (stats != NULL)
1533 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1534
1535 if (clear != 0)
1536 port->n_pkts_dropped_by_ah = 0;
1537
1538 return 0;
1539 }
1540
1541 int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1542 struct rte_pipeline_port_out_stats *stats, int clear)
1543 {
1544 struct rte_port_out *port;
1545 int retval;
1546
1547 if (p == NULL) {
1548 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1549 return -EINVAL;
1550 }
1551
1552 if (port_id >= p->num_ports_out) {
1553 RTE_LOG(ERR, PIPELINE,
1554 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1555 return -EINVAL;
1556 }
1557
1558 port = &p->ports_out[port_id];
1559 if (port->ops.f_stats != NULL) {
1560 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1561 if (retval != 0)
1562 return retval;
1563 } else if (stats != NULL)
1564 memset(&stats->stats, 0, sizeof(stats->stats));
1565
1566 if (stats != NULL)
1567 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1568
1569 if (clear != 0)
1570 port->n_pkts_dropped_by_ah = 0;
1571
1572 return 0;
1573 }
1574
1575 int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1576 struct rte_pipeline_table_stats *stats, int clear)
1577 {
1578 struct rte_table *table;
1579 int retval;
1580
1581 if (p == NULL) {
1582 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1583 __func__);
1584 return -EINVAL;
1585 }
1586
1587 if (table_id >= p->num_tables) {
1588 RTE_LOG(ERR, PIPELINE,
1589 "%s: table %u is out of range\n", __func__, table_id);
1590 return -EINVAL;
1591 }
1592
1593 table = &p->tables[table_id];
1594 if (table->ops.f_stats != NULL) {
1595 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1596 if (retval != 0)
1597 return retval;
1598 } else if (stats != NULL)
1599 memset(&stats->stats, 0, sizeof(stats->stats));
1600
1601 if (stats != NULL) {
1602 stats->n_pkts_dropped_by_lkp_hit_ah =
1603 table->n_pkts_dropped_by_lkp_hit_ah;
1604 stats->n_pkts_dropped_by_lkp_miss_ah =
1605 table->n_pkts_dropped_by_lkp_miss_ah;
1606 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1607 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1608 }
1609
1610 if (clear != 0) {
1611 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1612 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1613 table->n_pkts_dropped_lkp_hit = 0;
1614 table->n_pkts_dropped_lkp_miss = 0;
1615 }
1616
1617 return 0;
1618 }
Ceph