]> git.proxmox.com Git - ceph.git/blame - ceph/src/dpdk/lib/librte_pipeline/rte_pipeline.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bump version to 12.2.12-pve1
[ceph.git] / ceph / src / dpdk / lib / librte_pipeline / rte_pipeline.c
CommitLineData
7c673cae
FG		 1/*-
2 * BSD LICENSE
3 *
4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <string.h>
35#include <stdio.h>
36
37#include <rte_common.h>
38#include <rte_memory.h>
39#include <rte_memzone.h>
40#include <rte_cycles.h>
41#include <rte_prefetch.h>
42#include <rte_branch_prediction.h>
43#include <rte_mbuf.h>
44#include <rte_malloc.h>
45#include <rte_string_fns.h>
46
47#include "rte_pipeline.h"
48
49#define RTE_TABLE_INVALID UINT32_MAX
50
51#ifdef RTE_PIPELINE_STATS_COLLECT
52
53#define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask) \
54 ({ (p)->n_pkts_ah_drop = __builtin_popcountll(mask); })
55
56#define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter) \
57 ({ (counter) += (p)->n_pkts_ah_drop; (p)->n_pkts_ah_drop = 0; })
58
59#define RTE_PIPELINE_STATS_TABLE_DROP0(p) \
60 ({ (p)->pkts_drop_mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; })
61
62#define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter) \
63({ \
64 uint64_t mask = (p)->action_mask0[RTE_PIPELINE_ACTION_DROP]; \
65 mask ^= (p)->pkts_drop_mask; \
66 (counter) += __builtin_popcountll(mask); \
67})
68
69#else
70
71#define RTE_PIPELINE_STATS_AH_DROP_WRITE(p, mask)
72#define RTE_PIPELINE_STATS_AH_DROP_READ(p, counter)
73#define RTE_PIPELINE_STATS_TABLE_DROP0(p)
74#define RTE_PIPELINE_STATS_TABLE_DROP1(p, counter)
75
76#endif
77
78struct rte_port_in {
79 /* Input parameters */
80 struct rte_port_in_ops ops;
81 rte_pipeline_port_in_action_handler f_action;
82 void *arg_ah;
83 uint32_t burst_size;
84
85 /* The table to which this port is connected */
86 uint32_t table_id;
87
88 /* Handle to low-level port */
89 void *h_port;
90
91 /* List of enabled ports */
92 struct rte_port_in *next;
93
94 /* Statistics */
95 uint64_t n_pkts_dropped_by_ah;
96};
97
98struct rte_port_out {
99 /* Input parameters */
100 struct rte_port_out_ops ops;
101 rte_pipeline_port_out_action_handler f_action;
102 void *arg_ah;
103
104 /* Handle to low-level port */
105 void *h_port;
106
107 /* Statistics */
108 uint64_t n_pkts_dropped_by_ah;
109};
110
111struct rte_table {
112 /* Input parameters */
113 struct rte_table_ops ops;
114 rte_pipeline_table_action_handler_hit f_action_hit;
115 rte_pipeline_table_action_handler_miss f_action_miss;
116 void *arg_ah;
117 struct rte_pipeline_table_entry *default_entry;
118 uint32_t entry_size;
119
120 uint32_t table_next_id;
121 uint32_t table_next_id_valid;
122
123 /* Handle to the low-level table object */
124 void *h_table;
125
126 /* Statistics */
127 uint64_t n_pkts_dropped_by_lkp_hit_ah;
128 uint64_t n_pkts_dropped_by_lkp_miss_ah;
129 uint64_t n_pkts_dropped_lkp_hit;
130 uint64_t n_pkts_dropped_lkp_miss;
131};
132
133#define RTE_PIPELINE_MAX_NAME_SZ 124
134
135struct rte_pipeline {
136 /* Input parameters */
137 char name[RTE_PIPELINE_MAX_NAME_SZ];
138 int socket_id;
139 uint32_t offset_port_id;
140
141 /* Internal tables */
142 struct rte_port_in ports_in[RTE_PIPELINE_PORT_IN_MAX];
143 struct rte_port_out ports_out[RTE_PIPELINE_PORT_OUT_MAX];
144 struct rte_table tables[RTE_PIPELINE_TABLE_MAX];
145
146 /* Occupancy of internal tables */
147 uint32_t num_ports_in;
148 uint32_t num_ports_out;
149 uint32_t num_tables;
150
151 /* List of enabled ports */
152 uint64_t enabled_port_in_mask;
153 struct rte_port_in *port_in_next;
154
155 /* Pipeline run structures */
156 struct rte_mbuf *pkts[RTE_PORT_IN_BURST_SIZE_MAX];
157 struct rte_pipeline_table_entry *entries[RTE_PORT_IN_BURST_SIZE_MAX];
158 uint64_t action_mask0[RTE_PIPELINE_ACTIONS];
159 uint64_t action_mask1[RTE_PIPELINE_ACTIONS];
160 uint64_t pkts_mask;
161 uint64_t n_pkts_ah_drop;
162 uint64_t pkts_drop_mask;
163} __rte_cache_aligned;
164
165static inline uint32_t
166rte_mask_get_next(uint64_t mask, uint32_t pos)
167{
168 uint64_t mask_rot = (mask << ((63 - pos) & 0x3F)) |
169 (mask >> ((pos + 1) & 0x3F));
170 return (__builtin_ctzll(mask_rot) - (63 - pos)) & 0x3F;
171}
172
173static inline uint32_t
174rte_mask_get_prev(uint64_t mask, uint32_t pos)
175{
176 uint64_t mask_rot = (mask >> (pos & 0x3F)) |
177 (mask << ((64 - pos) & 0x3F));
178 return ((63 - __builtin_clzll(mask_rot)) + pos) & 0x3F;
179}
180
181static void
182rte_pipeline_table_free(struct rte_table *table);
183
184static void
185rte_pipeline_port_in_free(struct rte_port_in *port);
186
187static void
188rte_pipeline_port_out_free(struct rte_port_out *port);
189
190/*
191 * Pipeline
192 *
193 */
194static int
195rte_pipeline_check_params(struct rte_pipeline_params *params)
196{
197 if (params == NULL) {
198 RTE_LOG(ERR, PIPELINE,
199 "%s: Incorrect value for parameter params\n", __func__);
200 return -EINVAL;
201 }
202
203 /* name */
204 if (params->name == NULL) {
205 RTE_LOG(ERR, PIPELINE,
206 "%s: Incorrect value for parameter name\n", __func__);
207 return -EINVAL;
208 }
209
210 /* socket */
211 if ((params->socket_id < 0) ||
212 (params->socket_id >= RTE_MAX_NUMA_NODES)) {
213 RTE_LOG(ERR, PIPELINE,
214 "%s: Incorrect value for parameter socket_id\n",
215 __func__);
216 return -EINVAL;
217 }
218
219 return 0;
220}
221
222struct rte_pipeline *
223rte_pipeline_create(struct rte_pipeline_params *params)
224{
225 struct rte_pipeline *p;
226 int status;
227
228 /* Check input parameters */
229 status = rte_pipeline_check_params(params);
230 if (status != 0) {
231 RTE_LOG(ERR, PIPELINE,
232 "%s: Pipeline params check failed (%d)\n",
233 __func__, status);
234 return NULL;
235 }
236
237 /* Allocate memory for the pipeline on requested socket */
238 p = rte_zmalloc_socket("PIPELINE", sizeof(struct rte_pipeline),
239 RTE_CACHE_LINE_SIZE, params->socket_id);
240
241 if (p == NULL) {
242 RTE_LOG(ERR, PIPELINE,
243 "%s: Pipeline memory allocation failed\n", __func__);
244 return NULL;
245 }
246
247 /* Save input parameters */
248 snprintf(p->name, RTE_PIPELINE_MAX_NAME_SZ, "%s", params->name);
249 p->socket_id = params->socket_id;
250 p->offset_port_id = params->offset_port_id;
251
252 /* Initialize pipeline internal data structure */
253 p->num_ports_in = 0;
254 p->num_ports_out = 0;
255 p->num_tables = 0;
256 p->enabled_port_in_mask = 0;
257 p->port_in_next = NULL;
258 p->pkts_mask = 0;
259 p->n_pkts_ah_drop = 0;
260
261 return p;
262}
263
264int
265rte_pipeline_free(struct rte_pipeline *p)
266{
267 uint32_t i;
268
269 /* Check input parameters */
270 if (p == NULL) {
271 RTE_LOG(ERR, PIPELINE,
272 "%s: rte_pipeline parameter is NULL\n", __func__);
273 return -EINVAL;
274 }
275
276 /* Free input ports */
277 for (i = 0; i < p->num_ports_in; i++) {
278 struct rte_port_in *port = &p->ports_in[i];
279
280 rte_pipeline_port_in_free(port);
281 }
282
283 /* Free tables */
284 for (i = 0; i < p->num_tables; i++) {
285 struct rte_table *table = &p->tables[i];
286
287 rte_pipeline_table_free(table);
288 }
289
290 /* Free output ports */
291 for (i = 0; i < p->num_ports_out; i++) {
292 struct rte_port_out *port = &p->ports_out[i];
293
294 rte_pipeline_port_out_free(port);
295 }
296
297 /* Free pipeline memory */
298 rte_free(p);
299
300 return 0;
301}
302
303/*
304 * Table
305 *
306 */
307static int
308rte_table_check_params(struct rte_pipeline *p,
309 struct rte_pipeline_table_params *params,
310 uint32_t *table_id)
311{
312 if (p == NULL) {
313 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
314 __func__);
315 return -EINVAL;
316 }
317 if (params == NULL) {
318 RTE_LOG(ERR, PIPELINE, "%s: params parameter is NULL\n",
319 __func__);
320 return -EINVAL;
321 }
322 if (table_id == NULL) {
323 RTE_LOG(ERR, PIPELINE, "%s: table_id parameter is NULL\n",
324 __func__);
325 return -EINVAL;
326 }
327
328 /* ops */
329 if (params->ops == NULL) {
330 RTE_LOG(ERR, PIPELINE, "%s: params->ops is NULL\n",
331 __func__);
332 return -EINVAL;
333 }
334
335 if (params->ops->f_create == NULL) {
336 RTE_LOG(ERR, PIPELINE,
337 "%s: f_create function pointer is NULL\n", __func__);
338 return -EINVAL;
339 }
340
341 if (params->ops->f_lookup == NULL) {
342 RTE_LOG(ERR, PIPELINE,
343 "%s: f_lookup function pointer is NULL\n", __func__);
344 return -EINVAL;
345 }
346
347 /* De we have room for one more table? */
348 if (p->num_tables == RTE_PIPELINE_TABLE_MAX) {
349 RTE_LOG(ERR, PIPELINE,
350 "%s: Incorrect value for num_tables parameter\n",
351 __func__);
352 return -EINVAL;
353 }
354
355 return 0;
356}
357
358int
359rte_pipeline_table_create(struct rte_pipeline *p,
360 struct rte_pipeline_table_params *params,
361 uint32_t *table_id)
362{
363 struct rte_table *table;
364 struct rte_pipeline_table_entry *default_entry;
365 void *h_table;
366 uint32_t entry_size, id;
367 int status;
368
369 /* Check input arguments */
370 status = rte_table_check_params(p, params, table_id);
371 if (status != 0)
372 return status;
373
374 id = p->num_tables;
375 table = &p->tables[id];
376
377 /* Allocate space for the default table entry */
378 entry_size = sizeof(struct rte_pipeline_table_entry) +
379 params->action_data_size;
380 default_entry = (struct rte_pipeline_table_entry *) rte_zmalloc_socket(
381 "PIPELINE", entry_size, RTE_CACHE_LINE_SIZE, p->socket_id);
382 if (default_entry == NULL) {
383 RTE_LOG(ERR, PIPELINE,
384 "%s: Failed to allocate default entry\n", __func__);
385 return -EINVAL;
386 }
387
388 /* Create the table */
389 h_table = params->ops->f_create(params->arg_create, p->socket_id,
390 entry_size);
391 if (h_table == NULL) {
392 rte_free(default_entry);
393 RTE_LOG(ERR, PIPELINE, "%s: Table creation failed\n", __func__);
394 return -EINVAL;
395 }
396
397 /* Commit current table to the pipeline */
398 p->num_tables++;
399 *table_id = id;
400
401 /* Save input parameters */
402 memcpy(&table->ops, params->ops, sizeof(struct rte_table_ops));
403 table->f_action_hit = params->f_action_hit;
404 table->f_action_miss = params->f_action_miss;
405 table->arg_ah = params->arg_ah;
406 table->entry_size = entry_size;
407
408 /* Clear the lookup miss actions (to be set later through API) */
409 table->default_entry = default_entry;
410 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
411
412 /* Initialize table internal data structure */
413 table->h_table = h_table;
414 table->table_next_id = 0;
415 table->table_next_id_valid = 0;
416
417 return 0;
418}
419
420void
421rte_pipeline_table_free(struct rte_table *table)
422{
423 if (table->ops.f_free != NULL)
424 table->ops.f_free(table->h_table);
425
426 rte_free(table->default_entry);
427}
428
429int
430rte_pipeline_table_default_entry_add(struct rte_pipeline *p,
431 uint32_t table_id,
432 struct rte_pipeline_table_entry *default_entry,
433 struct rte_pipeline_table_entry **default_entry_ptr)
434{
435 struct rte_table *table;
436
437 /* Check input arguments */
438 if (p == NULL) {
439 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
440 __func__);
441 return -EINVAL;
442 }
443
444 if (default_entry == NULL) {
445 RTE_LOG(ERR, PIPELINE,
446 "%s: default_entry parameter is NULL\n", __func__);
447 return -EINVAL;
448 }
449
450 if (table_id >= p->num_tables) {
451 RTE_LOG(ERR, PIPELINE,
452 "%s: table_id %d out of range\n", __func__, table_id);
453 return -EINVAL;
454 }
455
456 table = &p->tables[table_id];
457
458 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
459 table->table_next_id_valid &&
460 (default_entry->table_id != table->table_next_id)) {
461 RTE_LOG(ERR, PIPELINE,
462 "%s: Tree-like topologies not allowed\n", __func__);
463 return -EINVAL;
464 }
465
466 /* Set the lookup miss actions */
467 if ((default_entry->action == RTE_PIPELINE_ACTION_TABLE) &&
468 (table->table_next_id_valid == 0)) {
469 table->table_next_id = default_entry->table_id;
470 table->table_next_id_valid = 1;
471 }
472
473 memcpy(table->default_entry, default_entry, table->entry_size);
474
475 *default_entry_ptr = table->default_entry;
476 return 0;
477}
478
479int
480rte_pipeline_table_default_entry_delete(struct rte_pipeline *p,
481 uint32_t table_id,
482 struct rte_pipeline_table_entry *entry)
483{
484 struct rte_table *table;
485
486 /* Check input arguments */
487 if (p == NULL) {
488 RTE_LOG(ERR, PIPELINE,
489 "%s: pipeline parameter is NULL\n", __func__);
490 return -EINVAL;
491 }
492
493 if (table_id >= p->num_tables) {
494 RTE_LOG(ERR, PIPELINE,
495 "%s: table_id %d out of range\n", __func__, table_id);
496 return -EINVAL;
497 }
498
499 table = &p->tables[table_id];
500
501 /* Save the current contents of the default entry */
502 if (entry)
503 memcpy(entry, table->default_entry, table->entry_size);
504
505 /* Clear the lookup miss actions */
506 memset(table->default_entry, 0, table->entry_size);
507 table->default_entry->action = RTE_PIPELINE_ACTION_DROP;
508
509 return 0;
510}
511
512int
513rte_pipeline_table_entry_add(struct rte_pipeline *p,
514 uint32_t table_id,
515 void *key,
516 struct rte_pipeline_table_entry *entry,
517 int *key_found,
518 struct rte_pipeline_table_entry **entry_ptr)
519{
520 struct rte_table *table;
521
522 /* Check input arguments */
523 if (p == NULL) {
524 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
525 __func__);
526 return -EINVAL;
527 }
528
529 if (key == NULL) {
530 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n", __func__);
531 return -EINVAL;
532 }
533
534 if (entry == NULL) {
535 RTE_LOG(ERR, PIPELINE, "%s: entry parameter is NULL\n",
536 __func__);
537 return -EINVAL;
538 }
539
540 if (table_id >= p->num_tables) {
541 RTE_LOG(ERR, PIPELINE,
542 "%s: table_id %d out of range\n", __func__, table_id);
543 return -EINVAL;
544 }
545
546 table = &p->tables[table_id];
547
548 if (table->ops.f_add == NULL) {
549 RTE_LOG(ERR, PIPELINE, "%s: f_add function pointer NULL\n",
550 __func__);
551 return -EINVAL;
552 }
553
554 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
555 table->table_next_id_valid &&
556 (entry->table_id != table->table_next_id)) {
557 RTE_LOG(ERR, PIPELINE,
558 "%s: Tree-like topologies not allowed\n", __func__);
559 return -EINVAL;
560 }
561
562 /* Add entry */
563 if ((entry->action == RTE_PIPELINE_ACTION_TABLE) &&
564 (table->table_next_id_valid == 0)) {
565 table->table_next_id = entry->table_id;
566 table->table_next_id_valid = 1;
567 }
568
569 return (table->ops.f_add)(table->h_table, key, (void *) entry,
570 key_found, (void **) entry_ptr);
571}
572
573int
574rte_pipeline_table_entry_delete(struct rte_pipeline *p,
575 uint32_t table_id,
576 void *key,
577 int *key_found,
578 struct rte_pipeline_table_entry *entry)
579{
580 struct rte_table *table;
581
582 /* Check input arguments */
583 if (p == NULL) {
584 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
585 __func__);
586 return -EINVAL;
587 }
588
589 if (key == NULL) {
590 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
591 __func__);
592 return -EINVAL;
593 }
594
595 if (table_id >= p->num_tables) {
596 RTE_LOG(ERR, PIPELINE,
597 "%s: table_id %d out of range\n", __func__, table_id);
598 return -EINVAL;
599 }
600
601 table = &p->tables[table_id];
602
603 if (table->ops.f_delete == NULL) {
604 RTE_LOG(ERR, PIPELINE,
605 "%s: f_delete function pointer NULL\n", __func__);
606 return -EINVAL;
607 }
608
609 return (table->ops.f_delete)(table->h_table, key, key_found, entry);
610}
611
612int rte_pipeline_table_entry_add_bulk(struct rte_pipeline *p,
613 uint32_t table_id,
614 void **keys,
615 struct rte_pipeline_table_entry **entries,
616 uint32_t n_keys,
617 int *key_found,
618 struct rte_pipeline_table_entry **entries_ptr)
619{
620 struct rte_table *table;
621 uint32_t i;
622
623 /* Check input arguments */
624 if (p == NULL) {
625 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter is NULL\n",
626 __func__);
627 return -EINVAL;
628 }
629
630 if (keys == NULL) {
631 RTE_LOG(ERR, PIPELINE, "%s: keys parameter is NULL\n", __func__);
632 return -EINVAL;
633 }
634
635 if (entries == NULL) {
636 RTE_LOG(ERR, PIPELINE, "%s: entries parameter is NULL\n",
637 __func__);
638 return -EINVAL;
639 }
640
641 if (table_id >= p->num_tables) {
642 RTE_LOG(ERR, PIPELINE,
643 "%s: table_id %d out of range\n", __func__, table_id);
644 return -EINVAL;
645 }
646
647 table = &p->tables[table_id];
648
649 if (table->ops.f_add_bulk == NULL) {
650 RTE_LOG(ERR, PIPELINE, "%s: f_add_bulk function pointer NULL\n",
651 __func__);
652 return -EINVAL;
653 }
654
655 for (i = 0; i < n_keys; i++) {
656 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
657 table->table_next_id_valid &&
658 (entries[i]->table_id != table->table_next_id)) {
659 RTE_LOG(ERR, PIPELINE,
660 "%s: Tree-like topologies not allowed\n", __func__);
661 return -EINVAL;
662 }
663 }
664
665 /* Add entry */
666 for (i = 0; i < n_keys; i++) {
667 if ((entries[i]->action == RTE_PIPELINE_ACTION_TABLE) &&
668 (table->table_next_id_valid == 0)) {
669 table->table_next_id = entries[i]->table_id;
670 table->table_next_id_valid = 1;
671 }
672 }
673
674 return (table->ops.f_add_bulk)(table->h_table, keys, (void **) entries,
675 n_keys, key_found, (void **) entries_ptr);
676}
677
678int rte_pipeline_table_entry_delete_bulk(struct rte_pipeline *p,
679 uint32_t table_id,
680 void **keys,
681 uint32_t n_keys,
682 int *key_found,
683 struct rte_pipeline_table_entry **entries)
684{
685 struct rte_table *table;
686
687 /* Check input arguments */
688 if (p == NULL) {
689 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
690 __func__);
691 return -EINVAL;
692 }
693
694 if (keys == NULL) {
695 RTE_LOG(ERR, PIPELINE, "%s: key parameter is NULL\n",
696 __func__);
697 return -EINVAL;
698 }
699
700 if (table_id >= p->num_tables) {
701 RTE_LOG(ERR, PIPELINE,
702 "%s: table_id %d out of range\n", __func__, table_id);
703 return -EINVAL;
704 }
705
706 table = &p->tables[table_id];
707
708 if (table->ops.f_delete_bulk == NULL) {
709 RTE_LOG(ERR, PIPELINE,
710 "%s: f_delete function pointer NULL\n", __func__);
711 return -EINVAL;
712 }
713
714 return (table->ops.f_delete_bulk)(table->h_table, keys, n_keys, key_found,
715 (void **) entries);
716}
717
718/*
719 * Port
720 *
721 */
722static int
723rte_pipeline_port_in_check_params(struct rte_pipeline *p,
724 struct rte_pipeline_port_in_params *params,
725 uint32_t *port_id)
726{
727 if (p == NULL) {
728 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
729 __func__);
730 return -EINVAL;
731 }
732 if (params == NULL) {
733 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
734 return -EINVAL;
735 }
736 if (port_id == NULL) {
737 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
738 __func__);
739 return -EINVAL;
740 }
741
742 /* ops */
743 if (params->ops == NULL) {
744 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
745 __func__);
746 return -EINVAL;
747 }
748
749 if (params->ops->f_create == NULL) {
750 RTE_LOG(ERR, PIPELINE,
751 "%s: f_create function pointer NULL\n", __func__);
752 return -EINVAL;
753 }
754
755 if (params->ops->f_rx == NULL) {
756 RTE_LOG(ERR, PIPELINE, "%s: f_rx function pointer NULL\n",
757 __func__);
758 return -EINVAL;
759 }
760
761 /* burst_size */
762 if ((params->burst_size == 0) ||
763 (params->burst_size > RTE_PORT_IN_BURST_SIZE_MAX)) {
764 RTE_LOG(ERR, PIPELINE, "%s: invalid value for burst_size\n",
765 __func__);
766 return -EINVAL;
767 }
768
769 /* Do we have room for one more port? */
770 if (p->num_ports_in == RTE_PIPELINE_PORT_IN_MAX) {
771 RTE_LOG(ERR, PIPELINE,
772 "%s: invalid value for num_ports_in\n", __func__);
773 return -EINVAL;
774 }
775
776 return 0;
777}
778
779static int
780rte_pipeline_port_out_check_params(struct rte_pipeline *p,
781 struct rte_pipeline_port_out_params *params,
782 uint32_t *port_id)
783{
784 if (p == NULL) {
785 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
786 __func__);
787 return -EINVAL;
788 }
789
790 if (params == NULL) {
791 RTE_LOG(ERR, PIPELINE, "%s: params parameter NULL\n", __func__);
792 return -EINVAL;
793 }
794
795 if (port_id == NULL) {
796 RTE_LOG(ERR, PIPELINE, "%s: port_id parameter NULL\n",
797 __func__);
798 return -EINVAL;
799 }
800
801 /* ops */
802 if (params->ops == NULL) {
803 RTE_LOG(ERR, PIPELINE, "%s: params->ops parameter NULL\n",
804 __func__);
805 return -EINVAL;
806 }
807
808 if (params->ops->f_create == NULL) {
809 RTE_LOG(ERR, PIPELINE,
810 "%s: f_create function pointer NULL\n", __func__);
811 return -EINVAL;
812 }
813
814 if (params->ops->f_tx == NULL) {
815 RTE_LOG(ERR, PIPELINE,
816 "%s: f_tx function pointer NULL\n", __func__);
817 return -EINVAL;
818 }
819
820 if (params->ops->f_tx_bulk == NULL) {
821 RTE_LOG(ERR, PIPELINE,
822 "%s: f_tx_bulk function pointer NULL\n", __func__);
823 return -EINVAL;
824 }
825
826 /* Do we have room for one more port? */
827 if (p->num_ports_out == RTE_PIPELINE_PORT_OUT_MAX) {
828 RTE_LOG(ERR, PIPELINE,
829 "%s: invalid value for num_ports_out\n", __func__);
830 return -EINVAL;
831 }
832
833 return 0;
834}
835
836int
837rte_pipeline_port_in_create(struct rte_pipeline *p,
838 struct rte_pipeline_port_in_params *params,
839 uint32_t *port_id)
840{
841 struct rte_port_in *port;
842 void *h_port;
843 uint32_t id;
844 int status;
845
846 /* Check input arguments */
847 status = rte_pipeline_port_in_check_params(p, params, port_id);
848 if (status != 0)
849 return status;
850
851 id = p->num_ports_in;
852 port = &p->ports_in[id];
853
854 /* Create the port */
855 h_port = params->ops->f_create(params->arg_create, p->socket_id);
856 if (h_port == NULL) {
857 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
858 return -EINVAL;
859 }
860
861 /* Commit current table to the pipeline */
862 p->num_ports_in++;
863 *port_id = id;
864
865 /* Save input parameters */
866 memcpy(&port->ops, params->ops, sizeof(struct rte_port_in_ops));
867 port->f_action = params->f_action;
868 port->arg_ah = params->arg_ah;
869 port->burst_size = params->burst_size;
870
871 /* Initialize port internal data structure */
872 port->table_id = RTE_TABLE_INVALID;
873 port->h_port = h_port;
874 port->next = NULL;
875
876 return 0;
877}
878
879void
880rte_pipeline_port_in_free(struct rte_port_in *port)
881{
882 if (port->ops.f_free != NULL)
883 port->ops.f_free(port->h_port);
884}
885
886int
887rte_pipeline_port_out_create(struct rte_pipeline *p,
888 struct rte_pipeline_port_out_params *params,
889 uint32_t *port_id)
890{
891 struct rte_port_out *port;
892 void *h_port;
893 uint32_t id;
894 int status;
895
896 /* Check input arguments */
897 status = rte_pipeline_port_out_check_params(p, params, port_id);
898 if (status != 0)
899 return status;
900
901 id = p->num_ports_out;
902 port = &p->ports_out[id];
903
904 /* Create the port */
905 h_port = params->ops->f_create(params->arg_create, p->socket_id);
906 if (h_port == NULL) {
907 RTE_LOG(ERR, PIPELINE, "%s: Port creation failed\n", __func__);
908 return -EINVAL;
909 }
910
911 /* Commit current table to the pipeline */
912 p->num_ports_out++;
913 *port_id = id;
914
915 /* Save input parameters */
916 memcpy(&port->ops, params->ops, sizeof(struct rte_port_out_ops));
917 port->f_action = params->f_action;
918 port->arg_ah = params->arg_ah;
919
920 /* Initialize port internal data structure */
921 port->h_port = h_port;
922
923 return 0;
924}
925
926void
927rte_pipeline_port_out_free(struct rte_port_out *port)
928{
929 if (port->ops.f_free != NULL)
930 port->ops.f_free(port->h_port);
931}
932
933int
934rte_pipeline_port_in_connect_to_table(struct rte_pipeline *p,
935 uint32_t port_id,
936 uint32_t table_id)
937{
938 struct rte_port_in *port;
939
940 /* Check input arguments */
941 if (p == NULL) {
942 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
943 __func__);
944 return -EINVAL;
945 }
946
947 if (port_id >= p->num_ports_in) {
948 RTE_LOG(ERR, PIPELINE,
949 "%s: port IN ID %u is out of range\n",
950 __func__, port_id);
951 return -EINVAL;
952 }
953
954 if (table_id >= p->num_tables) {
955 RTE_LOG(ERR, PIPELINE,
956 "%s: Table ID %u is out of range\n",
957 __func__, table_id);
958 return -EINVAL;
959 }
960
961 port = &p->ports_in[port_id];
962 port->table_id = table_id;
963
964 return 0;
965}
966
967int
968rte_pipeline_port_in_enable(struct rte_pipeline *p, uint32_t port_id)
969{
970 struct rte_port_in *port, *port_prev, *port_next;
971 uint64_t port_mask;
972 uint32_t port_prev_id, port_next_id;
973
974 /* Check input arguments */
975 if (p == NULL) {
976 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
977 __func__);
978 return -EINVAL;
979 }
980
981 if (port_id >= p->num_ports_in) {
982 RTE_LOG(ERR, PIPELINE,
983 "%s: port IN ID %u is out of range\n",
984 __func__, port_id);
985 return -EINVAL;
986 }
987
988 port = &p->ports_in[port_id];
989
990 /* Return if current input port is already enabled */
991 port_mask = 1LLU << port_id;
992 if (p->enabled_port_in_mask & port_mask)
993 return 0;
994
995 p->enabled_port_in_mask |= port_mask;
996
997 /* Add current input port to the pipeline chain of enabled ports */
998 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
999 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1000
1001 port_prev = &p->ports_in[port_prev_id];
1002 port_next = &p->ports_in[port_next_id];
1003
1004 port_prev->next = port;
1005 port->next = port_next;
1006
1007 /* Check if list of enabled ports was previously empty */
1008 if (p->enabled_port_in_mask == port_mask)
1009 p->port_in_next = port;
1010
1011 return 0;
1012}
1013
1014int
1015rte_pipeline_port_in_disable(struct rte_pipeline *p, uint32_t port_id)
1016{
1017 struct rte_port_in *port, *port_prev, *port_next;
1018 uint64_t port_mask;
1019 uint32_t port_prev_id, port_next_id;
1020
1021 /* Check input arguments */
1022 if (p == NULL) {
1023 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1024 __func__);
1025 return -EINVAL;
1026 }
1027
1028 if (port_id >= p->num_ports_in) {
1029 RTE_LOG(ERR, PIPELINE, "%s: port IN ID %u is out of range\n",
1030 __func__, port_id);
1031 return -EINVAL;
1032 }
1033
1034 port = &p->ports_in[port_id];
1035
1036 /* Return if current input port is already disabled */
1037 port_mask = 1LLU << port_id;
1038 if ((p->enabled_port_in_mask & port_mask) == 0)
1039 return 0;
1040
1041 p->enabled_port_in_mask &= ~port_mask;
1042
1043 /* Return if no other enabled ports */
1044 if (p->enabled_port_in_mask == 0) {
1045 p->port_in_next = NULL;
1046
1047 return 0;
1048 }
1049
1050 /* Add current input port to the pipeline chain of enabled ports */
1051 port_prev_id = rte_mask_get_prev(p->enabled_port_in_mask, port_id);
1052 port_next_id = rte_mask_get_next(p->enabled_port_in_mask, port_id);
1053
1054 port_prev = &p->ports_in[port_prev_id];
1055 port_next = &p->ports_in[port_next_id];
1056
1057 port_prev->next = port_next;
1058
1059 /* Check if the port which has just been disabled is next to serve */
1060 if (port == p->port_in_next)
1061 p->port_in_next = port_next;
1062
1063 return 0;
1064}
1065
1066/*
1067 * Pipeline run-time
1068 *
1069 */
1070int
1071rte_pipeline_check(struct rte_pipeline *p)
1072{
1073 uint32_t port_in_id;
1074
1075 /* Check input arguments */
1076 if (p == NULL) {
1077 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1078 __func__);
1079 return -EINVAL;
1080 }
1081
1082 /* Check that pipeline has at least one input port, one table and one
1083 output port */
1084 if (p->num_ports_in == 0) {
1085 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 input port\n",
1086 __func__);
1087 return -EINVAL;
1088 }
1089 if (p->num_tables == 0) {
1090 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 table\n",
1091 __func__);
1092 return -EINVAL;
1093 }
1094 if (p->num_ports_out == 0) {
1095 RTE_LOG(ERR, PIPELINE, "%s: must have at least 1 output port\n",
1096 __func__);
1097 return -EINVAL;
1098 }
1099
1100 /* Check that all input ports are connected */
1101 for (port_in_id = 0; port_in_id < p->num_ports_in; port_in_id++) {
1102 struct rte_port_in *port_in = &p->ports_in[port_in_id];
1103
1104 if (port_in->table_id == RTE_TABLE_INVALID) {
1105 RTE_LOG(ERR, PIPELINE,
1106 "%s: Port IN ID %u is not connected\n",
1107 __func__, port_in_id);
1108 return -EINVAL;
1109 }
1110 }
1111
1112 return 0;
1113}
1114
1115static inline void
1116rte_pipeline_compute_masks(struct rte_pipeline *p, uint64_t pkts_mask)
1117{
1118 p->action_mask1[RTE_PIPELINE_ACTION_DROP] = 0;
1119 p->action_mask1[RTE_PIPELINE_ACTION_PORT] = 0;
1120 p->action_mask1[RTE_PIPELINE_ACTION_PORT_META] = 0;
1121 p->action_mask1[RTE_PIPELINE_ACTION_TABLE] = 0;
1122
1123 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1124 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1125 uint32_t i;
1126
1127 for (i = 0; i < n_pkts; i++) {
1128 uint64_t pkt_mask = 1LLU << i;
1129 uint32_t pos = p->entries[i]->action;
1130
1131 p->action_mask1[pos] |= pkt_mask;
1132 }
1133 } else {
1134 uint32_t i;
1135
1136 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1137 uint64_t pkt_mask = 1LLU << i;
1138 uint32_t pos;
1139
1140 if ((pkt_mask & pkts_mask) == 0)
1141 continue;
1142
1143 pos = p->entries[i]->action;
1144 p->action_mask1[pos] |= pkt_mask;
1145 }
1146 }
1147}
1148
1149static inline void
1150rte_pipeline_action_handler_port_bulk(struct rte_pipeline *p,
1151 uint64_t pkts_mask, uint32_t port_id)
1152{
1153 struct rte_port_out *port_out = &p->ports_out[port_id];
1154
1155 p->pkts_mask = pkts_mask;
1156
1157 /* Output port user actions */
1158 if (port_out->f_action != NULL) {
1159 port_out->f_action(p, p->pkts, pkts_mask, port_out->arg_ah);
1160
1161 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1162 port_out->n_pkts_dropped_by_ah);
1163 }
1164
1165 /* Output port TX */
1166 if (p->pkts_mask != 0)
1167 port_out->ops.f_tx_bulk(port_out->h_port,
1168 p->pkts,
1169 p->pkts_mask);
1170}
1171
1172static inline void
1173rte_pipeline_action_handler_port(struct rte_pipeline *p, uint64_t pkts_mask)
1174{
1175 p->pkts_mask = pkts_mask;
1176
1177 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1178 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1179 uint32_t i;
1180
1181 for (i = 0; i < n_pkts; i++) {
1182 struct rte_mbuf *pkt = p->pkts[i];
1183 uint32_t port_out_id = p->entries[i]->port_id;
1184 struct rte_port_out *port_out =
1185 &p->ports_out[port_out_id];
1186
1187 /* Output port user actions */
1188 if (port_out->f_action == NULL) /* Output port TX */
1189 port_out->ops.f_tx(port_out->h_port, pkt);
1190 else {
1191 uint64_t pkt_mask = 1LLU << i;
1192
1193 port_out->f_action(p,
1194 p->pkts,
1195 pkt_mask,
1196 port_out->arg_ah);
1197
1198 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1199 port_out->n_pkts_dropped_by_ah);
1200
1201 /* Output port TX */
1202 if (pkt_mask & p->pkts_mask)
1203 port_out->ops.f_tx(port_out->h_port,
1204 pkt);
1205 }
1206 }
1207 } else {
1208 uint32_t i;
1209
1210 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1211 uint64_t pkt_mask = 1LLU << i;
1212 struct rte_mbuf *pkt;
1213 struct rte_port_out *port_out;
1214 uint32_t port_out_id;
1215
1216 if ((pkt_mask & pkts_mask) == 0)
1217 continue;
1218
1219 pkt = p->pkts[i];
1220 port_out_id = p->entries[i]->port_id;
1221 port_out = &p->ports_out[port_out_id];
1222
1223 /* Output port user actions */
1224 if (port_out->f_action == NULL) /* Output port TX */
1225 port_out->ops.f_tx(port_out->h_port, pkt);
1226 else {
1227 port_out->f_action(p,
1228 p->pkts,
1229 pkt_mask,
1230 port_out->arg_ah);
1231
1232 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1233 port_out->n_pkts_dropped_by_ah);
1234
1235 /* Output port TX */
1236 if (pkt_mask & p->pkts_mask)
1237 port_out->ops.f_tx(port_out->h_port,
1238 pkt);
1239 }
1240 }
1241 }
1242}
1243
1244static inline void
1245rte_pipeline_action_handler_port_meta(struct rte_pipeline *p,
1246 uint64_t pkts_mask)
1247{
1248 p->pkts_mask = pkts_mask;
1249
1250 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1251 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1252 uint32_t i;
1253
1254 for (i = 0; i < n_pkts; i++) {
1255 struct rte_mbuf *pkt = p->pkts[i];
1256 uint32_t port_out_id =
1257 RTE_MBUF_METADATA_UINT32(pkt,
1258 p->offset_port_id);
1259 struct rte_port_out *port_out = &p->ports_out[
1260 port_out_id];
1261
1262 /* Output port user actions */
1263 if (port_out->f_action == NULL) /* Output port TX */
1264 port_out->ops.f_tx(port_out->h_port, pkt);
1265 else {
1266 uint64_t pkt_mask = 1LLU << i;
1267
1268 port_out->f_action(p,
1269 p->pkts,
1270 pkt_mask,
1271 port_out->arg_ah);
1272
1273 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1274 port_out->n_pkts_dropped_by_ah);
1275
1276 /* Output port TX */
1277 if (pkt_mask & p->pkts_mask)
1278 port_out->ops.f_tx(port_out->h_port,
1279 pkt);
1280 }
1281 }
1282 } else {
1283 uint32_t i;
1284
1285 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1286 uint64_t pkt_mask = 1LLU << i;
1287 struct rte_mbuf *pkt;
1288 struct rte_port_out *port_out;
1289 uint32_t port_out_id;
1290
1291 if ((pkt_mask & pkts_mask) == 0)
1292 continue;
1293
1294 pkt = p->pkts[i];
1295 port_out_id = RTE_MBUF_METADATA_UINT32(pkt,
1296 p->offset_port_id);
1297 port_out = &p->ports_out[port_out_id];
1298
1299 /* Output port user actions */
1300 if (port_out->f_action == NULL) /* Output port TX */
1301 port_out->ops.f_tx(port_out->h_port, pkt);
1302 else {
1303 port_out->f_action(p,
1304 p->pkts,
1305 pkt_mask,
1306 port_out->arg_ah);
1307
1308 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1309 port_out->n_pkts_dropped_by_ah);
1310
1311 /* Output port TX */
1312 if (pkt_mask & p->pkts_mask)
1313 port_out->ops.f_tx(port_out->h_port,
1314 pkt);
1315 }
1316 }
1317 }
1318}
1319
1320static inline void
1321rte_pipeline_action_handler_drop(struct rte_pipeline *p, uint64_t pkts_mask)
1322{
1323 if ((pkts_mask & (pkts_mask + 1)) == 0) {
1324 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1325 uint32_t i;
1326
1327 for (i = 0; i < n_pkts; i++)
1328 rte_pktmbuf_free(p->pkts[i]);
1329 } else {
1330 uint32_t i;
1331
1332 for (i = 0; i < RTE_PORT_IN_BURST_SIZE_MAX; i++) {
1333 uint64_t pkt_mask = 1LLU << i;
1334
1335 if ((pkt_mask & pkts_mask) == 0)
1336 continue;
1337
1338 rte_pktmbuf_free(p->pkts[i]);
1339 }
1340 }
1341}
1342
1343int
1344rte_pipeline_run(struct rte_pipeline *p)
1345{
1346 struct rte_port_in *port_in = p->port_in_next;
1347 uint32_t n_pkts, table_id;
1348
1349 if (port_in == NULL)
1350 return 0;
1351
1352 /* Input port RX */
1353 n_pkts = port_in->ops.f_rx(port_in->h_port, p->pkts,
1354 port_in->burst_size);
1355 if (n_pkts == 0) {
1356 p->port_in_next = port_in->next;
1357 return 0;
1358 }
1359
1360 p->pkts_mask = RTE_LEN2MASK(n_pkts, uint64_t);
1361 p->action_mask0[RTE_PIPELINE_ACTION_DROP] = 0;
1362 p->action_mask0[RTE_PIPELINE_ACTION_PORT] = 0;
1363 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] = 0;
1364 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1365
1366 /* Input port user actions */
1367 if (port_in->f_action != NULL) {
1368 port_in->f_action(p, p->pkts, n_pkts, port_in->arg_ah);
1369
1370 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1371 port_in->n_pkts_dropped_by_ah);
1372 }
1373
1374 /* Table */
1375 for (table_id = port_in->table_id; p->pkts_mask != 0; ) {
1376 struct rte_table *table;
1377 uint64_t lookup_hit_mask, lookup_miss_mask;
1378
1379 /* Lookup */
1380 table = &p->tables[table_id];
1381 table->ops.f_lookup(table->h_table, p->pkts, p->pkts_mask,
1382 &lookup_hit_mask, (void **) p->entries);
1383 lookup_miss_mask = p->pkts_mask & (~lookup_hit_mask);
1384
1385 /* Lookup miss */
1386 if (lookup_miss_mask != 0) {
1387 struct rte_pipeline_table_entry *default_entry =
1388 table->default_entry;
1389
1390 p->pkts_mask = lookup_miss_mask;
1391
1392 /* Table user actions */
1393 if (table->f_action_miss != NULL) {
1394 table->f_action_miss(p,
1395 p->pkts,
1396 lookup_miss_mask,
1397 default_entry,
1398 table->arg_ah);
1399
1400 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1401 table->n_pkts_dropped_by_lkp_miss_ah);
1402 }
1403
1404 /* Table reserved actions */
1405 if ((default_entry->action == RTE_PIPELINE_ACTION_PORT) &&
1406 (p->pkts_mask != 0))
1407 rte_pipeline_action_handler_port_bulk(p,
1408 p->pkts_mask,
1409 default_entry->port_id);
1410 else {
1411 uint32_t pos = default_entry->action;
1412
1413 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1414
1415 p->action_mask0[pos] |= p->pkts_mask;
1416
1417 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1418 table->n_pkts_dropped_lkp_miss);
1419 }
1420 }
1421
1422 /* Lookup hit */
1423 if (lookup_hit_mask != 0) {
1424 p->pkts_mask = lookup_hit_mask;
1425
1426 /* Table user actions */
1427 if (table->f_action_hit != NULL) {
1428 table->f_action_hit(p,
1429 p->pkts,
1430 lookup_hit_mask,
1431 p->entries,
1432 table->arg_ah);
1433
1434 RTE_PIPELINE_STATS_AH_DROP_READ(p,
1435 table->n_pkts_dropped_by_lkp_hit_ah);
1436 }
1437
1438 /* Table reserved actions */
1439 RTE_PIPELINE_STATS_TABLE_DROP0(p);
1440 rte_pipeline_compute_masks(p, p->pkts_mask);
1441 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |=
1442 p->action_mask1[
1443 RTE_PIPELINE_ACTION_DROP];
1444 p->action_mask0[RTE_PIPELINE_ACTION_PORT] |=
1445 p->action_mask1[
1446 RTE_PIPELINE_ACTION_PORT];
1447 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META] |=
1448 p->action_mask1[
1449 RTE_PIPELINE_ACTION_PORT_META];
1450 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] |=
1451 p->action_mask1[
1452 RTE_PIPELINE_ACTION_TABLE];
1453
1454 RTE_PIPELINE_STATS_TABLE_DROP1(p,
1455 table->n_pkts_dropped_lkp_hit);
1456 }
1457
1458 /* Prepare for next iteration */
1459 p->pkts_mask = p->action_mask0[RTE_PIPELINE_ACTION_TABLE];
1460 table_id = table->table_next_id;
1461 p->action_mask0[RTE_PIPELINE_ACTION_TABLE] = 0;
1462 }
1463
1464 /* Table reserved action PORT */
1465 rte_pipeline_action_handler_port(p,
1466 p->action_mask0[RTE_PIPELINE_ACTION_PORT]);
1467
1468 /* Table reserved action PORT META */
1469 rte_pipeline_action_handler_port_meta(p,
1470 p->action_mask0[RTE_PIPELINE_ACTION_PORT_META]);
1471
1472 /* Table reserved action DROP */
1473 rte_pipeline_action_handler_drop(p,
1474 p->action_mask0[RTE_PIPELINE_ACTION_DROP]);
1475
1476 /* Pick candidate for next port IN to serve */
1477 p->port_in_next = port_in->next;
1478
1479 return (int) n_pkts;
1480}
1481
1482int
1483rte_pipeline_flush(struct rte_pipeline *p)
1484{
1485 uint32_t port_id;
1486
1487 /* Check input arguments */
1488 if (p == NULL) {
1489 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1490 __func__);
1491 return -EINVAL;
1492 }
1493
1494 for (port_id = 0; port_id < p->num_ports_out; port_id++) {
1495 struct rte_port_out *port = &p->ports_out[port_id];
1496
1497 if (port->ops.f_flush != NULL)
1498 port->ops.f_flush(port->h_port);
1499 }
1500
1501 return 0;
1502}
1503
1504int
1505rte_pipeline_port_out_packet_insert(struct rte_pipeline *p,
1506 uint32_t port_id, struct rte_mbuf *pkt)
1507{
1508 struct rte_port_out *port_out = &p->ports_out[port_id];
1509
1510 port_out->ops.f_tx(port_out->h_port, pkt); /* Output port TX */
1511
1512 return 0;
1513}
1514
1515int rte_pipeline_ah_packet_hijack(struct rte_pipeline *p,
1516 uint64_t pkts_mask)
1517{
1518 pkts_mask &= p->pkts_mask;
1519 p->pkts_mask &= ~pkts_mask;
1520
1521 return 0;
1522}
1523
1524int rte_pipeline_ah_packet_drop(struct rte_pipeline *p,
1525 uint64_t pkts_mask)
1526{
1527 pkts_mask &= p->pkts_mask;
1528 p->pkts_mask &= ~pkts_mask;
1529 p->action_mask0[RTE_PIPELINE_ACTION_DROP] |= pkts_mask;
1530
1531 RTE_PIPELINE_STATS_AH_DROP_WRITE(p, pkts_mask);
1532 return 0;
1533}
1534
1535int rte_pipeline_port_in_stats_read(struct rte_pipeline *p, uint32_t port_id,
1536 struct rte_pipeline_port_in_stats *stats, int clear)
1537{
1538 struct rte_port_in *port;
1539 int retval;
1540
1541 if (p == NULL) {
1542 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1543 __func__);
1544 return -EINVAL;
1545 }
1546
1547 if (port_id >= p->num_ports_in) {
1548 RTE_LOG(ERR, PIPELINE,
1549 "%s: port IN ID %u is out of range\n",
1550 __func__, port_id);
1551 return -EINVAL;
1552 }
1553
1554 port = &p->ports_in[port_id];
1555
1556 if (port->ops.f_stats != NULL) {
1557 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1558 if (retval)
1559 return retval;
1560 } else if (stats != NULL)
1561 memset(&stats->stats, 0, sizeof(stats->stats));
1562
1563 if (stats != NULL)
1564 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1565
1566 if (clear != 0)
1567 port->n_pkts_dropped_by_ah = 0;
1568
1569 return 0;
1570}
1571
1572int rte_pipeline_port_out_stats_read(struct rte_pipeline *p, uint32_t port_id,
1573 struct rte_pipeline_port_out_stats *stats, int clear)
1574{
1575 struct rte_port_out *port;
1576 int retval;
1577
1578 if (p == NULL) {
1579 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n", __func__);
1580 return -EINVAL;
1581 }
1582
1583 if (port_id >= p->num_ports_out) {
1584 RTE_LOG(ERR, PIPELINE,
1585 "%s: port OUT ID %u is out of range\n", __func__, port_id);
1586 return -EINVAL;
1587 }
1588
1589 port = &p->ports_out[port_id];
1590 if (port->ops.f_stats != NULL) {
1591 retval = port->ops.f_stats(port->h_port, &stats->stats, clear);
1592 if (retval != 0)
1593 return retval;
1594 } else if (stats != NULL)
1595 memset(&stats->stats, 0, sizeof(stats->stats));
1596
1597 if (stats != NULL)
1598 stats->n_pkts_dropped_by_ah = port->n_pkts_dropped_by_ah;
1599
1600 if (clear != 0)
1601 port->n_pkts_dropped_by_ah = 0;
1602
1603 return 0;
1604}
1605
1606int rte_pipeline_table_stats_read(struct rte_pipeline *p, uint32_t table_id,
1607 struct rte_pipeline_table_stats *stats, int clear)
1608{
1609 struct rte_table *table;
1610 int retval;
1611
1612 if (p == NULL) {
1613 RTE_LOG(ERR, PIPELINE, "%s: pipeline parameter NULL\n",
1614 __func__);
1615 return -EINVAL;
1616 }
1617
1618 if (table_id >= p->num_tables) {
1619 RTE_LOG(ERR, PIPELINE,
1620 "%s: table %u is out of range\n", __func__, table_id);
1621 return -EINVAL;
1622 }
1623
1624 table = &p->tables[table_id];
1625 if (table->ops.f_stats != NULL) {
1626 retval = table->ops.f_stats(table->h_table, &stats->stats, clear);
1627 if (retval != 0)
1628 return retval;
1629 } else if (stats != NULL)
1630 memset(&stats->stats, 0, sizeof(stats->stats));
1631
1632 if (stats != NULL) {
1633 stats->n_pkts_dropped_by_lkp_hit_ah =
1634 table->n_pkts_dropped_by_lkp_hit_ah;
1635 stats->n_pkts_dropped_by_lkp_miss_ah =
1636 table->n_pkts_dropped_by_lkp_miss_ah;
1637 stats->n_pkts_dropped_lkp_hit = table->n_pkts_dropped_lkp_hit;
1638 stats->n_pkts_dropped_lkp_miss = table->n_pkts_dropped_lkp_miss;
1639 }
1640
1641 if (clear != 0) {
1642 table->n_pkts_dropped_by_lkp_hit_ah = 0;
1643 table->n_pkts_dropped_by_lkp_miss_ah = 0;
1644 table->n_pkts_dropped_lkp_hit = 0;
1645 table->n_pkts_dropped_lkp_miss = 0;
1646 }
1647
1648 return 0;
1649}
Ceph