4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <rte_common.h>
37 #include <rte_malloc.h>
38 #include <rte_table_hash.h>
39 #include <rte_byteorder.h>
42 #include "pipeline_flow_classification_be.h"
43 #include "pipeline_actions_common.h"
45 #include "hash_func.h"
47 struct pipeline_flow_classification
{
49 pipeline_msg_req_handler custom_handlers
[PIPELINE_FC_MSG_REQS
];
57 uint8_t key_mask
[PIPELINE_FC_FLOW_KEY_MAX_SIZE
];
58 uint32_t key_mask_present
;
59 uint32_t flow_id_offset
;
61 } __rte_cache_aligned
;
64 pipeline_fc_msg_req_custom_handler(struct pipeline
*p
, void *msg
);
66 static pipeline_msg_req_handler handlers
[] = {
67 [PIPELINE_MSG_REQ_PING
] =
68 pipeline_msg_req_ping_handler
,
69 [PIPELINE_MSG_REQ_STATS_PORT_IN
] =
70 pipeline_msg_req_stats_port_in_handler
,
71 [PIPELINE_MSG_REQ_STATS_PORT_OUT
] =
72 pipeline_msg_req_stats_port_out_handler
,
73 [PIPELINE_MSG_REQ_STATS_TABLE
] =
74 pipeline_msg_req_stats_table_handler
,
75 [PIPELINE_MSG_REQ_PORT_IN_ENABLE
] =
76 pipeline_msg_req_port_in_enable_handler
,
77 [PIPELINE_MSG_REQ_PORT_IN_DISABLE
] =
78 pipeline_msg_req_port_in_disable_handler
,
79 [PIPELINE_MSG_REQ_CUSTOM
] =
80 pipeline_fc_msg_req_custom_handler
,
84 pipeline_fc_msg_req_add_handler(struct pipeline
*p
, void *msg
);
87 pipeline_fc_msg_req_add_bulk_handler(struct pipeline
*p
, void *msg
);
90 pipeline_fc_msg_req_del_handler(struct pipeline
*p
, void *msg
);
93 pipeline_fc_msg_req_add_default_handler(struct pipeline
*p
, void *msg
);
96 pipeline_fc_msg_req_del_default_handler(struct pipeline
*p
, void *msg
);
98 static pipeline_msg_req_handler custom_handlers
[] = {
99 [PIPELINE_FC_MSG_REQ_FLOW_ADD
] =
100 pipeline_fc_msg_req_add_handler
,
101 [PIPELINE_FC_MSG_REQ_FLOW_ADD_BULK
] =
102 pipeline_fc_msg_req_add_bulk_handler
,
103 [PIPELINE_FC_MSG_REQ_FLOW_DEL
] =
104 pipeline_fc_msg_req_del_handler
,
105 [PIPELINE_FC_MSG_REQ_FLOW_ADD_DEFAULT
] =
106 pipeline_fc_msg_req_add_default_handler
,
107 [PIPELINE_FC_MSG_REQ_FLOW_DEL_DEFAULT
] =
108 pipeline_fc_msg_req_del_default_handler
,
114 struct flow_table_entry
{
115 struct rte_pipeline_table_entry head
;
121 rte_table_hash_op_hash hash_func
[] = {
133 * Flow table AH - Write flow_id to packet meta-data
137 struct rte_mbuf
*pkt
,
138 struct rte_pipeline_table_entry
*table_entry
,
141 struct pipeline_flow_classification
*p_fc
= arg
;
142 uint32_t *flow_id_ptr
=
143 RTE_MBUF_METADATA_UINT32_PTR(pkt
, p_fc
->flow_id_offset
);
144 struct flow_table_entry
*entry
=
145 (struct flow_table_entry
*) table_entry
;
148 uint32_t flow_id
= entry
->flow_id
;
153 *flow_id_ptr
= flow_id
;
158 struct rte_mbuf
**pkts
,
159 struct rte_pipeline_table_entry
**table_entries
,
162 struct pipeline_flow_classification
*p_fc
= arg
;
164 uint32_t *flow_id_ptr0
=
165 RTE_MBUF_METADATA_UINT32_PTR(pkts
[0], p_fc
->flow_id_offset
);
166 uint32_t *flow_id_ptr1
=
167 RTE_MBUF_METADATA_UINT32_PTR(pkts
[1], p_fc
->flow_id_offset
);
168 uint32_t *flow_id_ptr2
=
169 RTE_MBUF_METADATA_UINT32_PTR(pkts
[2], p_fc
->flow_id_offset
);
170 uint32_t *flow_id_ptr3
=
171 RTE_MBUF_METADATA_UINT32_PTR(pkts
[3], p_fc
->flow_id_offset
);
173 struct flow_table_entry
*entry0
=
174 (struct flow_table_entry
*) table_entries
[0];
175 struct flow_table_entry
*entry1
=
176 (struct flow_table_entry
*) table_entries
[1];
177 struct flow_table_entry
*entry2
=
178 (struct flow_table_entry
*) table_entries
[2];
179 struct flow_table_entry
*entry3
=
180 (struct flow_table_entry
*) table_entries
[3];
183 uint32_t flow_id0
= entry0
->flow_id
;
184 uint32_t flow_id1
= entry1
->flow_id
;
185 uint32_t flow_id2
= entry2
->flow_id
;
186 uint32_t flow_id3
= entry3
->flow_id
;
191 *flow_id_ptr0
= flow_id0
;
192 *flow_id_ptr1
= flow_id1
;
193 *flow_id_ptr2
= flow_id2
;
194 *flow_id_ptr3
= flow_id3
;
197 PIPELINE_TABLE_AH_HIT(fc_table_ah_hit
,
198 pkt_work_flow_id
, pkt4_work_flow_id
);
200 static rte_pipeline_table_action_handler_hit
201 get_fc_table_ah_hit(struct pipeline_flow_classification
*p
)
204 return fc_table_ah_hit
;
213 pipeline_fc_parse_args(struct pipeline_flow_classification
*p
,
214 struct pipeline_params
*params
)
216 uint32_t n_flows_present
= 0;
217 uint32_t key_offset_present
= 0;
218 uint32_t key_size_present
= 0;
219 uint32_t hash_offset_present
= 0;
220 uint32_t key_mask_present
= 0;
221 uint32_t flow_id_offset_present
= 0;
224 char key_mask_str
[PIPELINE_FC_FLOW_KEY_MAX_SIZE
* 2 + 1];
231 for (i
= 0; i
< params
->n_args
; i
++) {
232 char *arg_name
= params
->args_name
[i
];
233 char *arg_value
= params
->args_value
[i
];
236 if (strcmp(arg_name
, "n_flows") == 0) {
239 PIPELINE_PARSE_ERR_DUPLICATE(
240 n_flows_present
== 0, params
->name
,
244 status
= parser_read_uint32(&p
->n_flows
,
246 PIPELINE_PARSE_ERR_INV_VAL(((status
!= -EINVAL
) &&
247 (p
->n_flows
!= 0)), params
->name
,
248 arg_name
, arg_value
);
249 PIPELINE_PARSE_ERR_OUT_RNG((status
!= -ERANGE
),
250 params
->name
, arg_name
, arg_value
);
256 if (strcmp(arg_name
, "key_offset") == 0) {
259 PIPELINE_PARSE_ERR_DUPLICATE(
260 key_offset_present
== 0, params
->name
,
262 key_offset_present
= 1;
264 status
= parser_read_uint32(&p
->key_offset
,
266 PIPELINE_PARSE_ERR_INV_VAL((status
!= -EINVAL
),
267 params
->name
, arg_name
, arg_value
);
268 PIPELINE_PARSE_ERR_OUT_RNG((status
!= -ERANGE
),
269 params
->name
, arg_name
, arg_value
);
275 if (strcmp(arg_name
, "key_size") == 0) {
278 PIPELINE_PARSE_ERR_DUPLICATE(
279 key_size_present
== 0, params
->name
,
281 key_size_present
= 1;
283 status
= parser_read_uint32(&p
->key_size
,
285 PIPELINE_PARSE_ERR_INV_VAL(((status
!= -EINVAL
) &&
286 (p
->key_size
!= 0) &&
287 (p
->key_size
% 8 == 0)),
288 params
->name
, arg_name
, arg_value
);
289 PIPELINE_PARSE_ERR_OUT_RNG(((status
!= -ERANGE
) &&
291 PIPELINE_FC_FLOW_KEY_MAX_SIZE
)),
292 params
->name
, arg_name
, arg_value
);
298 if (strcmp(arg_name
, "key_mask") == 0) {
299 int mask_str_len
= strlen(arg_value
);
301 PIPELINE_PARSE_ERR_DUPLICATE(
302 key_mask_present
== 0,
303 params
->name
, arg_name
);
304 key_mask_present
= 1;
306 PIPELINE_ARG_CHECK((mask_str_len
<=
307 (PIPELINE_FC_FLOW_KEY_MAX_SIZE
* 2)),
308 "Parse error in section \"%s\": entry "
309 "\"%s\" is too long", params
->name
,
312 snprintf(key_mask_str
, mask_str_len
+ 1, "%s",
319 if (strcmp(arg_name
, "hash_offset") == 0) {
322 PIPELINE_PARSE_ERR_DUPLICATE(
323 hash_offset_present
== 0, params
->name
,
325 hash_offset_present
= 1;
327 status
= parser_read_uint32(&p
->hash_offset
,
329 PIPELINE_PARSE_ERR_INV_VAL((status
!= -EINVAL
),
330 params
->name
, arg_name
, arg_value
);
331 PIPELINE_PARSE_ERR_OUT_RNG((status
!= -ERANGE
),
332 params
->name
, arg_name
, arg_value
);
338 if (strcmp(arg_name
, "flowid_offset") == 0) {
341 PIPELINE_PARSE_ERR_DUPLICATE(
342 flow_id_offset_present
== 0, params
->name
,
344 flow_id_offset_present
= 1;
346 status
= parser_read_uint32(&p
->flow_id_offset
,
348 PIPELINE_PARSE_ERR_INV_VAL((status
!= -EINVAL
),
349 params
->name
, arg_name
, arg_value
);
350 PIPELINE_PARSE_ERR_OUT_RNG((status
!= -ERANGE
),
351 params
->name
, arg_name
, arg_value
);
358 /* Unknown argument */
359 PIPELINE_PARSE_ERR_INV_ENT(0, params
->name
, arg_name
);
362 /* Check that mandatory arguments are present */
363 PIPELINE_PARSE_ERR_MANDATORY((n_flows_present
), params
->name
,
365 PIPELINE_PARSE_ERR_MANDATORY((key_offset_present
), params
->name
,
367 PIPELINE_PARSE_ERR_MANDATORY((key_size_present
), params
->name
,
370 if (key_mask_present
) {
371 uint32_t key_size
= p
->key_size
;
374 PIPELINE_ARG_CHECK(((key_size
== 8) || (key_size
== 16)),
375 "Parse error in section \"%s\": entry key_mask "
376 "only allowed for key_size of 8 or 16 bytes",
379 PIPELINE_ARG_CHECK((strlen(key_mask_str
) ==
380 (key_size
* 2)), "Parse error in section "
381 "\"%s\": key_mask should have exactly %u hex "
382 "digits", params
->name
, (key_size
* 2));
384 PIPELINE_ARG_CHECK((hash_offset_present
== 0), "Parse "
385 "error in section \"%s\": entry hash_offset only "
386 "allowed when key_mask is not present",
389 status
= parse_hex_string(key_mask_str
, p
->key_mask
,
392 PIPELINE_PARSE_ERR_INV_VAL(((status
== 0) &&
393 (key_size
== p
->key_size
)), params
->name
,
394 "key_mask", key_mask_str
);
397 p
->key_mask_present
= key_mask_present
;
402 static void *pipeline_fc_init(struct pipeline_params
*params
,
403 __rte_unused
void *arg
)
406 struct pipeline_flow_classification
*p_fc
;
409 /* Check input arguments */
413 /* Memory allocation */
414 size
= RTE_CACHE_LINE_ROUNDUP(
415 sizeof(struct pipeline_flow_classification
));
416 p
= rte_zmalloc(NULL
, size
, RTE_CACHE_LINE_SIZE
);
419 p_fc
= (struct pipeline_flow_classification
*) p
;
421 strcpy(p
->name
, params
->name
);
422 p
->log_level
= params
->log_level
;
424 PLOG(p
, HIGH
, "Flow classification");
426 /* Parse arguments */
427 if (pipeline_fc_parse_args(p_fc
, params
))
432 struct rte_pipeline_params pipeline_params
= {
433 .name
= params
->name
,
434 .socket_id
= params
->socket_id
,
438 p
->p
= rte_pipeline_create(&pipeline_params
);
446 p
->n_ports_in
= params
->n_ports_in
;
447 for (i
= 0; i
< p
->n_ports_in
; i
++) {
448 struct rte_pipeline_port_in_params port_params
= {
449 .ops
= pipeline_port_in_params_get_ops(
450 ¶ms
->port_in
[i
]),
451 .arg_create
= pipeline_port_in_params_convert(
452 ¶ms
->port_in
[i
]),
455 .burst_size
= params
->port_in
[i
].burst_size
,
458 int status
= rte_pipeline_port_in_create(p
->p
,
463 rte_pipeline_free(p
->p
);
470 p
->n_ports_out
= params
->n_ports_out
;
471 for (i
= 0; i
< p
->n_ports_out
; i
++) {
472 struct rte_pipeline_port_out_params port_params
= {
473 .ops
= pipeline_port_out_params_get_ops(
474 ¶ms
->port_out
[i
]),
475 .arg_create
= pipeline_port_out_params_convert(
476 ¶ms
->port_out
[i
]),
481 int status
= rte_pipeline_port_out_create(p
->p
,
486 rte_pipeline_free(p
->p
);
495 struct rte_table_hash_key8_ext_params
496 table_hash_key8_params
= {
497 .n_entries
= p_fc
->n_flows
,
498 .n_entries_ext
= p_fc
->n_flows
,
499 .signature_offset
= p_fc
->hash_offset
,
500 .key_offset
= p_fc
->key_offset
,
501 .f_hash
= hash_func
[(p_fc
->key_size
/ 8) - 1],
502 .key_mask
= (p_fc
->key_mask_present
) ?
503 p_fc
->key_mask
: NULL
,
507 struct rte_table_hash_key16_ext_params
508 table_hash_key16_params
= {
509 .n_entries
= p_fc
->n_flows
,
510 .n_entries_ext
= p_fc
->n_flows
,
511 .signature_offset
= p_fc
->hash_offset
,
512 .key_offset
= p_fc
->key_offset
,
513 .f_hash
= hash_func
[(p_fc
->key_size
/ 8) - 1],
514 .key_mask
= (p_fc
->key_mask_present
) ?
515 p_fc
->key_mask
: NULL
,
519 struct rte_table_hash_ext_params
520 table_hash_params
= {
521 .key_size
= p_fc
->key_size
,
522 .n_keys
= p_fc
->n_flows
,
523 .n_buckets
= p_fc
->n_flows
/ 4,
524 .n_buckets_ext
= p_fc
->n_flows
/ 4,
525 .f_hash
= hash_func
[(p_fc
->key_size
/ 8) - 1],
527 .signature_offset
= p_fc
->hash_offset
,
528 .key_offset
= p_fc
->key_offset
,
531 struct rte_pipeline_table_params table_params
= {
532 .ops
= NULL
, /* set below */
533 .arg_create
= NULL
, /* set below */
534 .f_action_hit
= get_fc_table_ah_hit(p_fc
),
535 .f_action_miss
= NULL
,
537 .action_data_size
= sizeof(struct flow_table_entry
) -
538 sizeof(struct rte_pipeline_table_entry
),
543 switch (p_fc
->key_size
) {
545 if (p_fc
->hash_offset
!= 0) {
547 &rte_table_hash_key8_ext_ops
;
550 &rte_table_hash_key8_ext_dosig_ops
;
552 table_params
.arg_create
= &table_hash_key8_params
;
556 if (p_fc
->hash_offset
!= 0) {
558 &rte_table_hash_key16_ext_ops
;
561 &rte_table_hash_key16_ext_dosig_ops
;
563 table_params
.arg_create
= &table_hash_key16_params
;
567 table_params
.ops
= &rte_table_hash_ext_ops
;
568 table_params
.arg_create
= &table_hash_params
;
571 status
= rte_pipeline_table_create(p
->p
,
576 rte_pipeline_free(p
->p
);
582 /* Connecting input ports to tables */
583 for (i
= 0; i
< p
->n_ports_in
; i
++) {
584 int status
= rte_pipeline_port_in_connect_to_table(p
->p
,
589 rte_pipeline_free(p
->p
);
595 /* Enable input ports */
596 for (i
= 0; i
< p
->n_ports_in
; i
++) {
597 int status
= rte_pipeline_port_in_enable(p
->p
,
601 rte_pipeline_free(p
->p
);
607 /* Check pipeline consistency */
608 if (rte_pipeline_check(p
->p
) < 0) {
609 rte_pipeline_free(p
->p
);
615 p
->n_msgq
= params
->n_msgq
;
616 for (i
= 0; i
< p
->n_msgq
; i
++)
617 p
->msgq_in
[i
] = params
->msgq_in
[i
];
618 for (i
= 0; i
< p
->n_msgq
; i
++)
619 p
->msgq_out
[i
] = params
->msgq_out
[i
];
621 /* Message handlers */
622 memcpy(p
->handlers
, handlers
, sizeof(p
->handlers
));
623 memcpy(p_fc
->custom_handlers
,
625 sizeof(p_fc
->custom_handlers
));
631 pipeline_fc_free(void *pipeline
)
633 struct pipeline
*p
= (struct pipeline
*) pipeline
;
635 /* Check input arguments */
640 rte_pipeline_free(p
->p
);
646 pipeline_fc_timer(void *pipeline
)
648 struct pipeline
*p
= (struct pipeline
*) pipeline
;
650 pipeline_msg_req_handle(p
);
651 rte_pipeline_flush(p
->p
);
657 pipeline_fc_msg_req_custom_handler(struct pipeline
*p
, void *msg
)
659 struct pipeline_flow_classification
*p_fc
=
660 (struct pipeline_flow_classification
*) p
;
661 struct pipeline_custom_msg_req
*req
= msg
;
662 pipeline_msg_req_handler f_handle
;
664 f_handle
= (req
->subtype
< PIPELINE_FC_MSG_REQS
) ?
665 p_fc
->custom_handlers
[req
->subtype
] :
666 pipeline_msg_req_invalid_handler
;
668 if (f_handle
== NULL
)
669 f_handle
= pipeline_msg_req_invalid_handler
;
671 return f_handle(p
, req
);
675 pipeline_fc_msg_req_add_handler(struct pipeline
*p
, void *msg
)
677 struct pipeline_fc_add_msg_req
*req
= msg
;
678 struct pipeline_fc_add_msg_rsp
*rsp
= msg
;
680 struct flow_table_entry entry
= {
682 .action
= RTE_PIPELINE_ACTION_PORT
,
683 {.port_id
= p
->port_out_id
[req
->port_id
]},
685 .flow_id
= req
->flow_id
,
688 rsp
->status
= rte_pipeline_table_entry_add(p
->p
,
691 (struct rte_pipeline_table_entry
*) &entry
,
693 (struct rte_pipeline_table_entry
**) &rsp
->entry_ptr
);
699 pipeline_fc_msg_req_add_bulk_handler(struct pipeline
*p
, void *msg
)
701 struct pipeline_fc_add_bulk_msg_req
*req
= msg
;
702 struct pipeline_fc_add_bulk_msg_rsp
*rsp
= msg
;
705 for (i
= 0; i
< req
->n_keys
; i
++) {
706 struct pipeline_fc_add_bulk_flow_req
*flow_req
= &req
->req
[i
];
707 struct pipeline_fc_add_bulk_flow_rsp
*flow_rsp
= &req
->rsp
[i
];
709 struct flow_table_entry entry
= {
711 .action
= RTE_PIPELINE_ACTION_PORT
,
712 {.port_id
= p
->port_out_id
[flow_req
->port_id
]},
714 .flow_id
= flow_req
->flow_id
,
717 int status
= rte_pipeline_table_entry_add(p
->p
,
720 (struct rte_pipeline_table_entry
*) &entry
,
721 &flow_rsp
->key_found
,
722 (struct rte_pipeline_table_entry
**)
723 &flow_rsp
->entry_ptr
);
735 pipeline_fc_msg_req_del_handler(struct pipeline
*p
, void *msg
)
737 struct pipeline_fc_del_msg_req
*req
= msg
;
738 struct pipeline_fc_del_msg_rsp
*rsp
= msg
;
740 rsp
->status
= rte_pipeline_table_entry_delete(p
->p
,
750 pipeline_fc_msg_req_add_default_handler(struct pipeline
*p
, void *msg
)
752 struct pipeline_fc_add_default_msg_req
*req
= msg
;
753 struct pipeline_fc_add_default_msg_rsp
*rsp
= msg
;
755 struct flow_table_entry default_entry
= {
757 .action
= RTE_PIPELINE_ACTION_PORT
,
758 {.port_id
= p
->port_out_id
[req
->port_id
]},
764 rsp
->status
= rte_pipeline_table_default_entry_add(p
->p
,
766 (struct rte_pipeline_table_entry
*) &default_entry
,
767 (struct rte_pipeline_table_entry
**) &rsp
->entry_ptr
);
773 pipeline_fc_msg_req_del_default_handler(struct pipeline
*p
, void *msg
)
775 struct pipeline_fc_del_default_msg_rsp
*rsp
= msg
;
777 rsp
->status
= rte_pipeline_table_default_entry_delete(p
->p
,
784 struct pipeline_be_ops pipeline_flow_classification_be_ops
= {
785 .f_init
= pipeline_fc_init
,
786 .f_free
= pipeline_fc_free
,
788 .f_timer
= pipeline_fc_timer
,