1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 Mellanox Technologies, Ltd
5 #include <netinet/in.h>
12 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
14 #pragma GCC diagnostic ignored "-Wpedantic"
16 #include <infiniband/verbs.h>
18 #pragma GCC diagnostic error "-Wpedantic"
21 #include <rte_common.h>
22 #include <rte_ether.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
30 #include "mlx5_defs.h"
31 #include "mlx5_flow.h"
32 #include "mlx5_glue.h"
34 #include "mlx5_rxtx.h"
36 #define VERBS_SPEC_INNER(item_flags) \
37 (!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
40 * Create Verbs flow counter with Verbs library.
43 * Pointer to the Ethernet device structure.
44 * @param[in, out] counter
45 * mlx5 flow counter object, contains the counter id,
46 * handle of created Verbs flow counter is returned
47 * in cs field (if counters are supported).
50 * 0 On success else a negative errno value is returned
51 * and rte_errno is set.
54 flow_verbs_counter_create(struct rte_eth_dev
*dev
,
55 struct mlx5_flow_counter
*counter
)
57 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
58 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
59 struct ibv_context
*ctx
= priv
->sh
->ctx
;
60 struct ibv_counter_set_init_attr init
= {
61 .counter_set_id
= counter
->id
};
63 counter
->cs
= mlx5_glue
->create_counter_set(ctx
, &init
);
69 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
70 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
71 struct ibv_context
*ctx
= priv
->sh
->ctx
;
72 struct ibv_counters_init_attr init
= {0};
73 struct ibv_counter_attach_attr attach
;
76 memset(&attach
, 0, sizeof(attach
));
77 counter
->cs
= mlx5_glue
->create_counters(ctx
, &init
);
82 attach
.counter_desc
= IBV_COUNTER_PACKETS
;
84 ret
= mlx5_glue
->attach_counters(counter
->cs
, &attach
, NULL
);
86 attach
.counter_desc
= IBV_COUNTER_BYTES
;
88 ret
= mlx5_glue
->attach_counters
89 (counter
->cs
, &attach
, NULL
);
92 claim_zero(mlx5_glue
->destroy_counters(counter
->cs
));
107 * Get a flow counter.
110 * Pointer to the Ethernet device structure.
112 * Indicate if this counter is shared with other flows.
114 * Counter identifier.
117 * A pointer to the counter, NULL otherwise and rte_errno is set.
119 static struct mlx5_flow_counter
*
120 flow_verbs_counter_new(struct rte_eth_dev
*dev
, uint32_t shared
, uint32_t id
)
122 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
123 struct mlx5_flow_counter
*cnt
;
127 LIST_FOREACH(cnt
, &priv
->flow_counters
, next
) {
128 if (cnt
->shared
&& cnt
->id
== id
) {
134 cnt
= rte_calloc(__func__
, 1, sizeof(*cnt
), 0);
140 cnt
->shared
= shared
;
144 /* Create counter with Verbs. */
145 ret
= flow_verbs_counter_create(dev
, cnt
);
147 LIST_INSERT_HEAD(&priv
->flow_counters
, cnt
, next
);
150 /* Some error occurred in Verbs library. */
157 * Release a flow counter.
160 * Pointer to the counter handler.
163 flow_verbs_counter_release(struct mlx5_flow_counter
*counter
)
165 if (--counter
->ref_cnt
== 0) {
166 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
167 claim_zero(mlx5_glue
->destroy_counter_set(counter
->cs
));
168 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
169 claim_zero(mlx5_glue
->destroy_counters(counter
->cs
));
171 LIST_REMOVE(counter
, next
);
177 * Query a flow counter via Verbs library call.
179 * @see rte_flow_query()
183 flow_verbs_counter_query(struct rte_eth_dev
*dev __rte_unused
,
184 struct rte_flow
*flow
, void *data
,
185 struct rte_flow_error
*error
)
187 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
188 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
189 if (flow
->actions
& MLX5_FLOW_ACTION_COUNT
) {
190 struct rte_flow_query_count
*qc
= data
;
191 uint64_t counters
[2] = {0, 0};
192 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
193 struct ibv_query_counter_set_attr query_cs_attr
= {
194 .cs
= flow
->counter
->cs
,
195 .query_flags
= IBV_COUNTER_SET_FORCE_UPDATE
,
197 struct ibv_counter_set_data query_out
= {
199 .outlen
= 2 * sizeof(uint64_t),
201 int err
= mlx5_glue
->query_counter_set(&query_cs_attr
,
203 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
204 int err
= mlx5_glue
->query_counters
205 (flow
->counter
->cs
, counters
,
207 IBV_READ_COUNTERS_ATTR_PREFER_CACHED
);
210 return rte_flow_error_set
212 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
,
214 "cannot read counter");
217 qc
->hits
= counters
[0] - flow
->counter
->hits
;
218 qc
->bytes
= counters
[1] - flow
->counter
->bytes
;
220 flow
->counter
->hits
= counters
[0];
221 flow
->counter
->bytes
= counters
[1];
225 return rte_flow_error_set(error
, EINVAL
,
226 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
,
228 "flow does not have counter");
232 return rte_flow_error_set(error
, ENOTSUP
,
233 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
,
235 "counters are not available");
240 * Add a verbs item specification into @p verbs.
243 * Pointer to verbs structure.
245 * Create specification.
247 * Size in bytes of the specification to copy.
250 flow_verbs_spec_add(struct mlx5_flow_verbs
*verbs
, void *src
, unsigned int size
)
256 assert(verbs
->specs
);
257 dst
= (void *)(verbs
->specs
+ verbs
->size
);
258 memcpy(dst
, src
, size
);
259 ++verbs
->attr
->num_of_specs
;
264 * Convert the @p item into a Verbs specification. This function assumes that
265 * the input is valid and that there is space to insert the requested item
268 * @param[in, out] dev_flow
269 * Pointer to dev_flow structure.
271 * Item specification.
272 * @param[in] item_flags
276 flow_verbs_translate_item_eth(struct mlx5_flow
*dev_flow
,
277 const struct rte_flow_item
*item
,
280 const struct rte_flow_item_eth
*spec
= item
->spec
;
281 const struct rte_flow_item_eth
*mask
= item
->mask
;
282 const unsigned int size
= sizeof(struct ibv_flow_spec_eth
);
283 struct ibv_flow_spec_eth eth
= {
284 .type
= IBV_FLOW_SPEC_ETH
| VERBS_SPEC_INNER(item_flags
),
289 mask
= &rte_flow_item_eth_mask
;
293 memcpy(ð
.val
.dst_mac
, spec
->dst
.addr_bytes
, ETHER_ADDR_LEN
);
294 memcpy(ð
.val
.src_mac
, spec
->src
.addr_bytes
, ETHER_ADDR_LEN
);
295 eth
.val
.ether_type
= spec
->type
;
296 memcpy(ð
.mask
.dst_mac
, mask
->dst
.addr_bytes
, ETHER_ADDR_LEN
);
297 memcpy(ð
.mask
.src_mac
, mask
->src
.addr_bytes
, ETHER_ADDR_LEN
);
298 eth
.mask
.ether_type
= mask
->type
;
299 /* Remove unwanted bits from values. */
300 for (i
= 0; i
< ETHER_ADDR_LEN
; ++i
) {
301 eth
.val
.dst_mac
[i
] &= eth
.mask
.dst_mac
[i
];
302 eth
.val
.src_mac
[i
] &= eth
.mask
.src_mac
[i
];
304 eth
.val
.ether_type
&= eth
.mask
.ether_type
;
306 flow_verbs_spec_add(&dev_flow
->verbs
, ð
, size
);
310 * Update the VLAN tag in the Verbs Ethernet specification.
311 * This function assumes that the input is valid and there is space to add
312 * the requested item.
314 * @param[in, out] attr
315 * Pointer to Verbs attributes structure.
317 * Verbs structure containing the VLAN information to copy.
320 flow_verbs_item_vlan_update(struct ibv_flow_attr
*attr
,
321 struct ibv_flow_spec_eth
*eth
)
324 const enum ibv_flow_spec_type search
= eth
->type
;
325 struct ibv_spec_header
*hdr
= (struct ibv_spec_header
*)
326 ((uint8_t *)attr
+ sizeof(struct ibv_flow_attr
));
328 for (i
= 0; i
!= attr
->num_of_specs
; ++i
) {
329 if (hdr
->type
== search
) {
330 struct ibv_flow_spec_eth
*e
=
331 (struct ibv_flow_spec_eth
*)hdr
;
333 e
->val
.vlan_tag
= eth
->val
.vlan_tag
;
334 e
->mask
.vlan_tag
= eth
->mask
.vlan_tag
;
335 e
->val
.ether_type
= eth
->val
.ether_type
;
336 e
->mask
.ether_type
= eth
->mask
.ether_type
;
339 hdr
= (struct ibv_spec_header
*)((uint8_t *)hdr
+ hdr
->size
);
344 * Convert the @p item into a Verbs specification. This function assumes that
345 * the input is valid and that there is space to insert the requested item
348 * @param[in, out] dev_flow
349 * Pointer to dev_flow structure.
351 * Item specification.
352 * @param[in] item_flags
356 flow_verbs_translate_item_vlan(struct mlx5_flow
*dev_flow
,
357 const struct rte_flow_item
*item
,
360 const struct rte_flow_item_vlan
*spec
= item
->spec
;
361 const struct rte_flow_item_vlan
*mask
= item
->mask
;
362 unsigned int size
= sizeof(struct ibv_flow_spec_eth
);
363 const int tunnel
= !!(item_flags
& MLX5_FLOW_LAYER_TUNNEL
);
364 struct ibv_flow_spec_eth eth
= {
365 .type
= IBV_FLOW_SPEC_ETH
| VERBS_SPEC_INNER(item_flags
),
368 const uint32_t l2m
= tunnel
? MLX5_FLOW_LAYER_INNER_L2
:
369 MLX5_FLOW_LAYER_OUTER_L2
;
372 mask
= &rte_flow_item_vlan_mask
;
374 eth
.val
.vlan_tag
= spec
->tci
;
375 eth
.mask
.vlan_tag
= mask
->tci
;
376 eth
.val
.vlan_tag
&= eth
.mask
.vlan_tag
;
377 eth
.val
.ether_type
= spec
->inner_type
;
378 eth
.mask
.ether_type
= mask
->inner_type
;
379 eth
.val
.ether_type
&= eth
.mask
.ether_type
;
381 if (!(item_flags
& l2m
))
382 flow_verbs_spec_add(&dev_flow
->verbs
, ð
, size
);
384 flow_verbs_item_vlan_update(dev_flow
->verbs
.attr
, ð
);
388 * Convert the @p item into a Verbs specification. This function assumes that
389 * the input is valid and that there is space to insert the requested item
392 * @param[in, out] dev_flow
393 * Pointer to dev_flow structure.
395 * Item specification.
396 * @param[in] item_flags
400 flow_verbs_translate_item_ipv4(struct mlx5_flow
*dev_flow
,
401 const struct rte_flow_item
*item
,
404 const struct rte_flow_item_ipv4
*spec
= item
->spec
;
405 const struct rte_flow_item_ipv4
*mask
= item
->mask
;
406 unsigned int size
= sizeof(struct ibv_flow_spec_ipv4_ext
);
407 struct ibv_flow_spec_ipv4_ext ipv4
= {
408 .type
= IBV_FLOW_SPEC_IPV4_EXT
| VERBS_SPEC_INNER(item_flags
),
413 mask
= &rte_flow_item_ipv4_mask
;
415 ipv4
.val
= (struct ibv_flow_ipv4_ext_filter
){
416 .src_ip
= spec
->hdr
.src_addr
,
417 .dst_ip
= spec
->hdr
.dst_addr
,
418 .proto
= spec
->hdr
.next_proto_id
,
419 .tos
= spec
->hdr
.type_of_service
,
421 ipv4
.mask
= (struct ibv_flow_ipv4_ext_filter
){
422 .src_ip
= mask
->hdr
.src_addr
,
423 .dst_ip
= mask
->hdr
.dst_addr
,
424 .proto
= mask
->hdr
.next_proto_id
,
425 .tos
= mask
->hdr
.type_of_service
,
427 /* Remove unwanted bits from values. */
428 ipv4
.val
.src_ip
&= ipv4
.mask
.src_ip
;
429 ipv4
.val
.dst_ip
&= ipv4
.mask
.dst_ip
;
430 ipv4
.val
.proto
&= ipv4
.mask
.proto
;
431 ipv4
.val
.tos
&= ipv4
.mask
.tos
;
433 flow_verbs_spec_add(&dev_flow
->verbs
, &ipv4
, size
);
437 * Convert the @p item into a Verbs specification. This function assumes that
438 * the input is valid and that there is space to insert the requested item
441 * @param[in, out] dev_flow
442 * Pointer to dev_flow structure.
444 * Item specification.
445 * @param[in] item_flags
449 flow_verbs_translate_item_ipv6(struct mlx5_flow
*dev_flow
,
450 const struct rte_flow_item
*item
,
453 const struct rte_flow_item_ipv6
*spec
= item
->spec
;
454 const struct rte_flow_item_ipv6
*mask
= item
->mask
;
455 unsigned int size
= sizeof(struct ibv_flow_spec_ipv6
);
456 struct ibv_flow_spec_ipv6 ipv6
= {
457 .type
= IBV_FLOW_SPEC_IPV6
| VERBS_SPEC_INNER(item_flags
),
462 mask
= &rte_flow_item_ipv6_mask
;
465 uint32_t vtc_flow_val
;
466 uint32_t vtc_flow_mask
;
468 memcpy(&ipv6
.val
.src_ip
, spec
->hdr
.src_addr
,
469 RTE_DIM(ipv6
.val
.src_ip
));
470 memcpy(&ipv6
.val
.dst_ip
, spec
->hdr
.dst_addr
,
471 RTE_DIM(ipv6
.val
.dst_ip
));
472 memcpy(&ipv6
.mask
.src_ip
, mask
->hdr
.src_addr
,
473 RTE_DIM(ipv6
.mask
.src_ip
));
474 memcpy(&ipv6
.mask
.dst_ip
, mask
->hdr
.dst_addr
,
475 RTE_DIM(ipv6
.mask
.dst_ip
));
476 vtc_flow_val
= rte_be_to_cpu_32(spec
->hdr
.vtc_flow
);
477 vtc_flow_mask
= rte_be_to_cpu_32(mask
->hdr
.vtc_flow
);
478 ipv6
.val
.flow_label
=
479 rte_cpu_to_be_32((vtc_flow_val
& IPV6_HDR_FL_MASK
) >>
481 ipv6
.val
.traffic_class
= (vtc_flow_val
& IPV6_HDR_TC_MASK
) >>
483 ipv6
.val
.next_hdr
= spec
->hdr
.proto
;
484 ipv6
.val
.hop_limit
= spec
->hdr
.hop_limits
;
485 ipv6
.mask
.flow_label
=
486 rte_cpu_to_be_32((vtc_flow_mask
& IPV6_HDR_FL_MASK
) >>
488 ipv6
.mask
.traffic_class
= (vtc_flow_mask
& IPV6_HDR_TC_MASK
) >>
490 ipv6
.mask
.next_hdr
= mask
->hdr
.proto
;
491 ipv6
.mask
.hop_limit
= mask
->hdr
.hop_limits
;
492 /* Remove unwanted bits from values. */
493 for (i
= 0; i
< RTE_DIM(ipv6
.val
.src_ip
); ++i
) {
494 ipv6
.val
.src_ip
[i
] &= ipv6
.mask
.src_ip
[i
];
495 ipv6
.val
.dst_ip
[i
] &= ipv6
.mask
.dst_ip
[i
];
497 ipv6
.val
.flow_label
&= ipv6
.mask
.flow_label
;
498 ipv6
.val
.traffic_class
&= ipv6
.mask
.traffic_class
;
499 ipv6
.val
.next_hdr
&= ipv6
.mask
.next_hdr
;
500 ipv6
.val
.hop_limit
&= ipv6
.mask
.hop_limit
;
502 flow_verbs_spec_add(&dev_flow
->verbs
, &ipv6
, size
);
506 * Convert the @p item into a Verbs specification. This function assumes that
507 * the input is valid and that there is space to insert the requested item
510 * @param[in, out] dev_flow
511 * Pointer to dev_flow structure.
513 * Item specification.
514 * @param[in] item_flags
518 flow_verbs_translate_item_tcp(struct mlx5_flow
*dev_flow
,
519 const struct rte_flow_item
*item
,
520 uint64_t item_flags __rte_unused
)
522 const struct rte_flow_item_tcp
*spec
= item
->spec
;
523 const struct rte_flow_item_tcp
*mask
= item
->mask
;
524 unsigned int size
= sizeof(struct ibv_flow_spec_tcp_udp
);
525 struct ibv_flow_spec_tcp_udp tcp
= {
526 .type
= IBV_FLOW_SPEC_TCP
| VERBS_SPEC_INNER(item_flags
),
531 mask
= &rte_flow_item_tcp_mask
;
533 tcp
.val
.dst_port
= spec
->hdr
.dst_port
;
534 tcp
.val
.src_port
= spec
->hdr
.src_port
;
535 tcp
.mask
.dst_port
= mask
->hdr
.dst_port
;
536 tcp
.mask
.src_port
= mask
->hdr
.src_port
;
537 /* Remove unwanted bits from values. */
538 tcp
.val
.src_port
&= tcp
.mask
.src_port
;
539 tcp
.val
.dst_port
&= tcp
.mask
.dst_port
;
541 flow_verbs_spec_add(&dev_flow
->verbs
, &tcp
, size
);
545 * Convert the @p item into a Verbs specification. This function assumes that
546 * the input is valid and that there is space to insert the requested item
549 * @param[in, out] dev_flow
550 * Pointer to dev_flow structure.
552 * Item specification.
553 * @param[in] item_flags
557 flow_verbs_translate_item_udp(struct mlx5_flow
*dev_flow
,
558 const struct rte_flow_item
*item
,
559 uint64_t item_flags __rte_unused
)
561 const struct rte_flow_item_udp
*spec
= item
->spec
;
562 const struct rte_flow_item_udp
*mask
= item
->mask
;
563 unsigned int size
= sizeof(struct ibv_flow_spec_tcp_udp
);
564 struct ibv_flow_spec_tcp_udp udp
= {
565 .type
= IBV_FLOW_SPEC_UDP
| VERBS_SPEC_INNER(item_flags
),
570 mask
= &rte_flow_item_udp_mask
;
572 udp
.val
.dst_port
= spec
->hdr
.dst_port
;
573 udp
.val
.src_port
= spec
->hdr
.src_port
;
574 udp
.mask
.dst_port
= mask
->hdr
.dst_port
;
575 udp
.mask
.src_port
= mask
->hdr
.src_port
;
576 /* Remove unwanted bits from values. */
577 udp
.val
.src_port
&= udp
.mask
.src_port
;
578 udp
.val
.dst_port
&= udp
.mask
.dst_port
;
580 flow_verbs_spec_add(&dev_flow
->verbs
, &udp
, size
);
584 * Convert the @p item into a Verbs specification. This function assumes that
585 * the input is valid and that there is space to insert the requested item
588 * @param[in, out] dev_flow
589 * Pointer to dev_flow structure.
591 * Item specification.
592 * @param[in] item_flags
596 flow_verbs_translate_item_vxlan(struct mlx5_flow
*dev_flow
,
597 const struct rte_flow_item
*item
,
598 uint64_t item_flags __rte_unused
)
600 const struct rte_flow_item_vxlan
*spec
= item
->spec
;
601 const struct rte_flow_item_vxlan
*mask
= item
->mask
;
602 unsigned int size
= sizeof(struct ibv_flow_spec_tunnel
);
603 struct ibv_flow_spec_tunnel vxlan
= {
604 .type
= IBV_FLOW_SPEC_VXLAN_TUNNEL
,
610 } id
= { .vlan_id
= 0, };
613 mask
= &rte_flow_item_vxlan_mask
;
615 memcpy(&id
.vni
[1], spec
->vni
, 3);
616 vxlan
.val
.tunnel_id
= id
.vlan_id
;
617 memcpy(&id
.vni
[1], mask
->vni
, 3);
618 vxlan
.mask
.tunnel_id
= id
.vlan_id
;
619 /* Remove unwanted bits from values. */
620 vxlan
.val
.tunnel_id
&= vxlan
.mask
.tunnel_id
;
622 flow_verbs_spec_add(&dev_flow
->verbs
, &vxlan
, size
);
626 * Convert the @p item into a Verbs specification. This function assumes that
627 * the input is valid and that there is space to insert the requested item
630 * @param[in, out] dev_flow
631 * Pointer to dev_flow structure.
633 * Item specification.
634 * @param[in] item_flags
638 flow_verbs_translate_item_vxlan_gpe(struct mlx5_flow
*dev_flow
,
639 const struct rte_flow_item
*item
,
640 uint64_t item_flags __rte_unused
)
642 const struct rte_flow_item_vxlan_gpe
*spec
= item
->spec
;
643 const struct rte_flow_item_vxlan_gpe
*mask
= item
->mask
;
644 unsigned int size
= sizeof(struct ibv_flow_spec_tunnel
);
645 struct ibv_flow_spec_tunnel vxlan_gpe
= {
646 .type
= IBV_FLOW_SPEC_VXLAN_TUNNEL
,
652 } id
= { .vlan_id
= 0, };
655 mask
= &rte_flow_item_vxlan_gpe_mask
;
657 memcpy(&id
.vni
[1], spec
->vni
, 3);
658 vxlan_gpe
.val
.tunnel_id
= id
.vlan_id
;
659 memcpy(&id
.vni
[1], mask
->vni
, 3);
660 vxlan_gpe
.mask
.tunnel_id
= id
.vlan_id
;
661 /* Remove unwanted bits from values. */
662 vxlan_gpe
.val
.tunnel_id
&= vxlan_gpe
.mask
.tunnel_id
;
664 flow_verbs_spec_add(&dev_flow
->verbs
, &vxlan_gpe
, size
);
668 * Update the protocol in Verbs IPv4/IPv6 spec.
670 * @param[in, out] attr
671 * Pointer to Verbs attributes structure.
673 * Specification type to search in order to update the IP protocol.
674 * @param[in] protocol
675 * Protocol value to set if none is present in the specification.
678 flow_verbs_item_gre_ip_protocol_update(struct ibv_flow_attr
*attr
,
679 enum ibv_flow_spec_type search
,
683 struct ibv_spec_header
*hdr
= (struct ibv_spec_header
*)
684 ((uint8_t *)attr
+ sizeof(struct ibv_flow_attr
));
688 for (i
= 0; i
!= attr
->num_of_specs
; ++i
) {
689 if (hdr
->type
== search
) {
691 struct ibv_flow_spec_ipv4_ext
*ipv4
;
692 struct ibv_flow_spec_ipv6
*ipv6
;
696 case IBV_FLOW_SPEC_IPV4_EXT
:
697 ip
.ipv4
= (struct ibv_flow_spec_ipv4_ext
*)hdr
;
698 if (!ip
.ipv4
->val
.proto
) {
699 ip
.ipv4
->val
.proto
= protocol
;
700 ip
.ipv4
->mask
.proto
= 0xff;
703 case IBV_FLOW_SPEC_IPV6
:
704 ip
.ipv6
= (struct ibv_flow_spec_ipv6
*)hdr
;
705 if (!ip
.ipv6
->val
.next_hdr
) {
706 ip
.ipv6
->val
.next_hdr
= protocol
;
707 ip
.ipv6
->mask
.next_hdr
= 0xff;
715 hdr
= (struct ibv_spec_header
*)((uint8_t *)hdr
+ hdr
->size
);
720 * Convert the @p item into a Verbs specification. This function assumes that
721 * the input is valid and that there is space to insert the requested item
724 * @param[in, out] dev_flow
725 * Pointer to dev_flow structure.
727 * Item specification.
728 * @param[in] item_flags
732 flow_verbs_translate_item_gre(struct mlx5_flow
*dev_flow
,
733 const struct rte_flow_item
*item __rte_unused
,
736 struct mlx5_flow_verbs
*verbs
= &dev_flow
->verbs
;
737 #ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
738 unsigned int size
= sizeof(struct ibv_flow_spec_tunnel
);
739 struct ibv_flow_spec_tunnel tunnel
= {
740 .type
= IBV_FLOW_SPEC_VXLAN_TUNNEL
,
744 const struct rte_flow_item_gre
*spec
= item
->spec
;
745 const struct rte_flow_item_gre
*mask
= item
->mask
;
746 unsigned int size
= sizeof(struct ibv_flow_spec_gre
);
747 struct ibv_flow_spec_gre tunnel
= {
748 .type
= IBV_FLOW_SPEC_GRE
,
753 mask
= &rte_flow_item_gre_mask
;
755 tunnel
.val
.c_ks_res0_ver
= spec
->c_rsvd0_ver
;
756 tunnel
.val
.protocol
= spec
->protocol
;
757 tunnel
.mask
.c_ks_res0_ver
= mask
->c_rsvd0_ver
;
758 tunnel
.mask
.protocol
= mask
->protocol
;
759 /* Remove unwanted bits from values. */
760 tunnel
.val
.c_ks_res0_ver
&= tunnel
.mask
.c_ks_res0_ver
;
761 tunnel
.val
.protocol
&= tunnel
.mask
.protocol
;
762 tunnel
.val
.key
&= tunnel
.mask
.key
;
765 if (item_flags
& MLX5_FLOW_LAYER_OUTER_L3_IPV4
)
766 flow_verbs_item_gre_ip_protocol_update(verbs
->attr
,
767 IBV_FLOW_SPEC_IPV4_EXT
,
770 flow_verbs_item_gre_ip_protocol_update(verbs
->attr
,
773 flow_verbs_spec_add(verbs
, &tunnel
, size
);
777 * Convert the @p action into a Verbs specification. This function assumes that
778 * the input is valid and that there is space to insert the requested action
779 * into the flow. This function also return the action that was added.
781 * @param[in, out] dev_flow
782 * Pointer to dev_flow structure.
784 * Item specification.
785 * @param[in] item_flags
789 flow_verbs_translate_item_mpls(struct mlx5_flow
*dev_flow __rte_unused
,
790 const struct rte_flow_item
*item __rte_unused
,
791 uint64_t item_flags __rte_unused
)
793 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
794 const struct rte_flow_item_mpls
*spec
= item
->spec
;
795 const struct rte_flow_item_mpls
*mask
= item
->mask
;
796 unsigned int size
= sizeof(struct ibv_flow_spec_mpls
);
797 struct ibv_flow_spec_mpls mpls
= {
798 .type
= IBV_FLOW_SPEC_MPLS
,
803 mask
= &rte_flow_item_mpls_mask
;
805 memcpy(&mpls
.val
.label
, spec
, sizeof(mpls
.val
.label
));
806 memcpy(&mpls
.mask
.label
, mask
, sizeof(mpls
.mask
.label
));
807 /* Remove unwanted bits from values. */
808 mpls
.val
.label
&= mpls
.mask
.label
;
810 flow_verbs_spec_add(&dev_flow
->verbs
, &mpls
, size
);
815 * Convert the @p action into a Verbs specification. This function assumes that
816 * the input is valid and that there is space to insert the requested action
819 * @param[in] dev_flow
820 * Pointer to mlx5_flow.
822 * Action configuration.
825 flow_verbs_translate_action_drop
826 (struct mlx5_flow
*dev_flow
,
827 const struct rte_flow_action
*action __rte_unused
)
829 unsigned int size
= sizeof(struct ibv_flow_spec_action_drop
);
830 struct ibv_flow_spec_action_drop drop
= {
831 .type
= IBV_FLOW_SPEC_ACTION_DROP
,
835 flow_verbs_spec_add(&dev_flow
->verbs
, &drop
, size
);
839 * Convert the @p action into a Verbs specification. This function assumes that
840 * the input is valid and that there is space to insert the requested action
843 * @param[in] dev_flow
844 * Pointer to mlx5_flow.
846 * Action configuration.
849 flow_verbs_translate_action_queue(struct mlx5_flow
*dev_flow
,
850 const struct rte_flow_action
*action
)
852 const struct rte_flow_action_queue
*queue
= action
->conf
;
853 struct rte_flow
*flow
= dev_flow
->flow
;
856 (*flow
->queue
)[0] = queue
->index
;
857 flow
->rss
.queue_num
= 1;
861 * Convert the @p action into a Verbs specification. This function assumes that
862 * the input is valid and that there is space to insert the requested action
866 * Action configuration.
867 * @param[in, out] action_flags
868 * Pointer to the detected actions.
869 * @param[in] dev_flow
870 * Pointer to mlx5_flow.
873 flow_verbs_translate_action_rss(struct mlx5_flow
*dev_flow
,
874 const struct rte_flow_action
*action
)
876 const struct rte_flow_action_rss
*rss
= action
->conf
;
877 const uint8_t *rss_key
;
878 struct rte_flow
*flow
= dev_flow
->flow
;
881 memcpy((*flow
->queue
), rss
->queue
,
882 rss
->queue_num
* sizeof(uint16_t));
883 flow
->rss
.queue_num
= rss
->queue_num
;
884 /* NULL RSS key indicates default RSS key. */
885 rss_key
= !rss
->key
? rss_hash_default_key
: rss
->key
;
886 memcpy(flow
->key
, rss_key
, MLX5_RSS_HASH_KEY_LEN
);
887 /* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
888 flow
->rss
.types
= !rss
->types
? ETH_RSS_IP
: rss
->types
;
889 flow
->rss
.level
= rss
->level
;
893 * Convert the @p action into a Verbs specification. This function assumes that
894 * the input is valid and that there is space to insert the requested action
897 * @param[in] dev_flow
898 * Pointer to mlx5_flow.
900 * Action configuration.
903 flow_verbs_translate_action_flag
904 (struct mlx5_flow
*dev_flow
,
905 const struct rte_flow_action
*action __rte_unused
)
907 unsigned int size
= sizeof(struct ibv_flow_spec_action_tag
);
908 struct ibv_flow_spec_action_tag tag
= {
909 .type
= IBV_FLOW_SPEC_ACTION_TAG
,
911 .tag_id
= mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT
),
914 flow_verbs_spec_add(&dev_flow
->verbs
, &tag
, size
);
918 * Convert the @p action into a Verbs specification. This function assumes that
919 * the input is valid and that there is space to insert the requested action
922 * @param[in] dev_flow
923 * Pointer to mlx5_flow.
925 * Action configuration.
928 flow_verbs_translate_action_mark(struct mlx5_flow
*dev_flow
,
929 const struct rte_flow_action
*action
)
931 const struct rte_flow_action_mark
*mark
= action
->conf
;
932 unsigned int size
= sizeof(struct ibv_flow_spec_action_tag
);
933 struct ibv_flow_spec_action_tag tag
= {
934 .type
= IBV_FLOW_SPEC_ACTION_TAG
,
936 .tag_id
= mlx5_flow_mark_set(mark
->id
),
939 flow_verbs_spec_add(&dev_flow
->verbs
, &tag
, size
);
943 * Convert the @p action into a Verbs specification. This function assumes that
944 * the input is valid and that there is space to insert the requested action
948 * Pointer to the Ethernet device structure.
950 * Action configuration.
951 * @param[in] dev_flow
952 * Pointer to mlx5_flow.
954 * Pointer to error structure.
957 * 0 On success else a negative errno value is returned and rte_errno is set.
960 flow_verbs_translate_action_count(struct mlx5_flow
*dev_flow
,
961 const struct rte_flow_action
*action
,
962 struct rte_eth_dev
*dev
,
963 struct rte_flow_error
*error
)
965 const struct rte_flow_action_count
*count
= action
->conf
;
966 struct rte_flow
*flow
= dev_flow
->flow
;
967 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
968 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
969 unsigned int size
= sizeof(struct ibv_flow_spec_counter_action
);
970 struct ibv_flow_spec_counter_action counter
= {
971 .type
= IBV_FLOW_SPEC_ACTION_COUNT
,
976 if (!flow
->counter
) {
977 flow
->counter
= flow_verbs_counter_new(dev
, count
->shared
,
980 return rte_flow_error_set(error
, rte_errno
,
981 RTE_FLOW_ERROR_TYPE_ACTION
,
986 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
987 counter
.counter_set_handle
= flow
->counter
->cs
->handle
;
988 flow_verbs_spec_add(&dev_flow
->verbs
, &counter
, size
);
989 #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
990 counter
.counters
= flow
->counter
->cs
;
991 flow_verbs_spec_add(&dev_flow
->verbs
, &counter
, size
);
997 * Internal validation function. For validating both actions and items.
1000 * Pointer to the Ethernet device structure.
1002 * Pointer to the flow attributes.
1004 * Pointer to the list of items.
1005 * @param[in] actions
1006 * Pointer to the list of actions.
1008 * Pointer to the error structure.
1011 * 0 on success, a negative errno value otherwise and rte_errno is set.
1014 flow_verbs_validate(struct rte_eth_dev
*dev
,
1015 const struct rte_flow_attr
*attr
,
1016 const struct rte_flow_item items
[],
1017 const struct rte_flow_action actions
[],
1018 struct rte_flow_error
*error
)
1021 uint64_t action_flags
= 0;
1022 uint64_t item_flags
= 0;
1023 uint64_t last_item
= 0;
1024 uint8_t next_protocol
= 0xff;
1028 ret
= mlx5_flow_validate_attributes(dev
, attr
, error
);
1031 for (; items
->type
!= RTE_FLOW_ITEM_TYPE_END
; items
++) {
1032 int tunnel
= !!(item_flags
& MLX5_FLOW_LAYER_TUNNEL
);
1035 switch (items
->type
) {
1036 case RTE_FLOW_ITEM_TYPE_VOID
:
1038 case RTE_FLOW_ITEM_TYPE_ETH
:
1039 ret
= mlx5_flow_validate_item_eth(items
, item_flags
,
1043 last_item
= tunnel
? MLX5_FLOW_LAYER_INNER_L2
:
1044 MLX5_FLOW_LAYER_OUTER_L2
;
1046 case RTE_FLOW_ITEM_TYPE_VLAN
:
1047 ret
= mlx5_flow_validate_item_vlan(items
, item_flags
,
1051 last_item
= tunnel
? (MLX5_FLOW_LAYER_INNER_L2
|
1052 MLX5_FLOW_LAYER_INNER_VLAN
) :
1053 (MLX5_FLOW_LAYER_OUTER_L2
|
1054 MLX5_FLOW_LAYER_OUTER_VLAN
);
1056 case RTE_FLOW_ITEM_TYPE_IPV4
:
1057 ret
= mlx5_flow_validate_item_ipv4(items
, item_flags
,
1061 last_item
= tunnel
? MLX5_FLOW_LAYER_INNER_L3_IPV4
:
1062 MLX5_FLOW_LAYER_OUTER_L3_IPV4
;
1063 if (items
->mask
!= NULL
&&
1064 ((const struct rte_flow_item_ipv4
*)
1065 items
->mask
)->hdr
.next_proto_id
) {
1067 ((const struct rte_flow_item_ipv4
*)
1068 (items
->spec
))->hdr
.next_proto_id
;
1070 ((const struct rte_flow_item_ipv4
*)
1071 (items
->mask
))->hdr
.next_proto_id
;
1073 /* Reset for inner layer. */
1074 next_protocol
= 0xff;
1077 case RTE_FLOW_ITEM_TYPE_IPV6
:
1078 ret
= mlx5_flow_validate_item_ipv6(items
, item_flags
,
1082 last_item
= tunnel
? MLX5_FLOW_LAYER_INNER_L3_IPV6
:
1083 MLX5_FLOW_LAYER_OUTER_L3_IPV6
;
1084 if (items
->mask
!= NULL
&&
1085 ((const struct rte_flow_item_ipv6
*)
1086 items
->mask
)->hdr
.proto
) {
1088 ((const struct rte_flow_item_ipv6
*)
1089 items
->spec
)->hdr
.proto
;
1091 ((const struct rte_flow_item_ipv6
*)
1092 items
->mask
)->hdr
.proto
;
1094 /* Reset for inner layer. */
1095 next_protocol
= 0xff;
1098 case RTE_FLOW_ITEM_TYPE_UDP
:
1099 ret
= mlx5_flow_validate_item_udp(items
, item_flags
,
1104 last_item
= tunnel
? MLX5_FLOW_LAYER_INNER_L4_UDP
:
1105 MLX5_FLOW_LAYER_OUTER_L4_UDP
;
1107 case RTE_FLOW_ITEM_TYPE_TCP
:
1108 ret
= mlx5_flow_validate_item_tcp
1111 &rte_flow_item_tcp_mask
,
1115 last_item
= tunnel
? MLX5_FLOW_LAYER_INNER_L4_TCP
:
1116 MLX5_FLOW_LAYER_OUTER_L4_TCP
;
1118 case RTE_FLOW_ITEM_TYPE_VXLAN
:
1119 ret
= mlx5_flow_validate_item_vxlan(items
, item_flags
,
1123 last_item
= MLX5_FLOW_LAYER_VXLAN
;
1125 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE
:
1126 ret
= mlx5_flow_validate_item_vxlan_gpe(items
,
1131 last_item
= MLX5_FLOW_LAYER_VXLAN_GPE
;
1133 case RTE_FLOW_ITEM_TYPE_GRE
:
1134 ret
= mlx5_flow_validate_item_gre(items
, item_flags
,
1135 next_protocol
, error
);
1138 last_item
= MLX5_FLOW_LAYER_GRE
;
1140 case RTE_FLOW_ITEM_TYPE_MPLS
:
1141 ret
= mlx5_flow_validate_item_mpls(dev
, items
,
1146 last_item
= MLX5_FLOW_LAYER_MPLS
;
1149 return rte_flow_error_set(error
, ENOTSUP
,
1150 RTE_FLOW_ERROR_TYPE_ITEM
,
1151 NULL
, "item not supported");
1153 item_flags
|= last_item
;
1155 for (; actions
->type
!= RTE_FLOW_ACTION_TYPE_END
; actions
++) {
1156 switch (actions
->type
) {
1157 case RTE_FLOW_ACTION_TYPE_VOID
:
1159 case RTE_FLOW_ACTION_TYPE_FLAG
:
1160 ret
= mlx5_flow_validate_action_flag(action_flags
,
1165 action_flags
|= MLX5_FLOW_ACTION_FLAG
;
1167 case RTE_FLOW_ACTION_TYPE_MARK
:
1168 ret
= mlx5_flow_validate_action_mark(actions
,
1174 action_flags
|= MLX5_FLOW_ACTION_MARK
;
1176 case RTE_FLOW_ACTION_TYPE_DROP
:
1177 ret
= mlx5_flow_validate_action_drop(action_flags
,
1182 action_flags
|= MLX5_FLOW_ACTION_DROP
;
1184 case RTE_FLOW_ACTION_TYPE_QUEUE
:
1185 ret
= mlx5_flow_validate_action_queue(actions
,
1191 action_flags
|= MLX5_FLOW_ACTION_QUEUE
;
1193 case RTE_FLOW_ACTION_TYPE_RSS
:
1194 ret
= mlx5_flow_validate_action_rss(actions
,
1200 action_flags
|= MLX5_FLOW_ACTION_RSS
;
1202 case RTE_FLOW_ACTION_TYPE_COUNT
:
1203 ret
= mlx5_flow_validate_action_count(dev
, attr
, error
);
1206 action_flags
|= MLX5_FLOW_ACTION_COUNT
;
1209 return rte_flow_error_set(error
, ENOTSUP
,
1210 RTE_FLOW_ERROR_TYPE_ACTION
,
1212 "action not supported");
1215 if (!(action_flags
& MLX5_FLOW_FATE_ACTIONS
))
1216 return rte_flow_error_set(error
, EINVAL
,
1217 RTE_FLOW_ERROR_TYPE_ACTION
, actions
,
1218 "no fate action is found");
1223 * Calculate the required bytes that are needed for the action part of the verbs
1226 * @param[in] actions
1227 * Pointer to the list of actions.
1230 * The size of the memory needed for all actions.
1233 flow_verbs_get_actions_size(const struct rte_flow_action actions
[])
1237 for (; actions
->type
!= RTE_FLOW_ACTION_TYPE_END
; actions
++) {
1238 switch (actions
->type
) {
1239 case RTE_FLOW_ACTION_TYPE_VOID
:
1241 case RTE_FLOW_ACTION_TYPE_FLAG
:
1242 size
+= sizeof(struct ibv_flow_spec_action_tag
);
1244 case RTE_FLOW_ACTION_TYPE_MARK
:
1245 size
+= sizeof(struct ibv_flow_spec_action_tag
);
1247 case RTE_FLOW_ACTION_TYPE_DROP
:
1248 size
+= sizeof(struct ibv_flow_spec_action_drop
);
1250 case RTE_FLOW_ACTION_TYPE_QUEUE
:
1252 case RTE_FLOW_ACTION_TYPE_RSS
:
1254 case RTE_FLOW_ACTION_TYPE_COUNT
:
1255 #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
1256 defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
1257 size
+= sizeof(struct ibv_flow_spec_counter_action
);
1268 * Calculate the required bytes that are needed for the item part of the verbs
1272 * Pointer to the list of items.
1275 * The size of the memory needed for all items.
1278 flow_verbs_get_items_size(const struct rte_flow_item items
[])
1282 for (; items
->type
!= RTE_FLOW_ITEM_TYPE_END
; items
++) {
1283 switch (items
->type
) {
1284 case RTE_FLOW_ITEM_TYPE_VOID
:
1286 case RTE_FLOW_ITEM_TYPE_ETH
:
1287 size
+= sizeof(struct ibv_flow_spec_eth
);
1289 case RTE_FLOW_ITEM_TYPE_VLAN
:
1290 size
+= sizeof(struct ibv_flow_spec_eth
);
1292 case RTE_FLOW_ITEM_TYPE_IPV4
:
1293 size
+= sizeof(struct ibv_flow_spec_ipv4_ext
);
1295 case RTE_FLOW_ITEM_TYPE_IPV6
:
1296 size
+= sizeof(struct ibv_flow_spec_ipv6
);
1298 case RTE_FLOW_ITEM_TYPE_UDP
:
1299 size
+= sizeof(struct ibv_flow_spec_tcp_udp
);
1301 case RTE_FLOW_ITEM_TYPE_TCP
:
1302 size
+= sizeof(struct ibv_flow_spec_tcp_udp
);
1304 case RTE_FLOW_ITEM_TYPE_VXLAN
:
1305 size
+= sizeof(struct ibv_flow_spec_tunnel
);
1307 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE
:
1308 size
+= sizeof(struct ibv_flow_spec_tunnel
);
1310 #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
1311 case RTE_FLOW_ITEM_TYPE_GRE
:
1312 size
+= sizeof(struct ibv_flow_spec_gre
);
1314 case RTE_FLOW_ITEM_TYPE_MPLS
:
1315 size
+= sizeof(struct ibv_flow_spec_mpls
);
1318 case RTE_FLOW_ITEM_TYPE_GRE
:
1319 size
+= sizeof(struct ibv_flow_spec_tunnel
);
1330 * Internal preparation function. Allocate mlx5_flow with the required size.
1331 * The required size is calculate based on the actions and items. This function
1332 * also returns the detected actions and items for later use.
1335 * Pointer to the flow attributes.
1337 * Pointer to the list of items.
1338 * @param[in] actions
1339 * Pointer to the list of actions.
1341 * Pointer to the error structure.
1344 * Pointer to mlx5_flow object on success, otherwise NULL and rte_errno
1347 static struct mlx5_flow
*
1348 flow_verbs_prepare(const struct rte_flow_attr
*attr __rte_unused
,
1349 const struct rte_flow_item items
[],
1350 const struct rte_flow_action actions
[],
1351 struct rte_flow_error
*error
)
1353 uint32_t size
= sizeof(struct mlx5_flow
) + sizeof(struct ibv_flow_attr
);
1354 struct mlx5_flow
*flow
;
1356 size
+= flow_verbs_get_actions_size(actions
);
1357 size
+= flow_verbs_get_items_size(items
);
1358 flow
= rte_calloc(__func__
, 1, size
, 0);
1360 rte_flow_error_set(error
, ENOMEM
,
1361 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
, NULL
,
1362 "not enough memory to create flow");
1365 flow
->verbs
.attr
= (void *)(flow
+ 1);
1367 (uint8_t *)(flow
+ 1) + sizeof(struct ibv_flow_attr
);
1372 * Fill the flow with verb spec.
1375 * Pointer to Ethernet device.
1376 * @param[in, out] dev_flow
1377 * Pointer to the mlx5 flow.
1379 * Pointer to the flow attributes.
1381 * Pointer to the list of items.
1382 * @param[in] actions
1383 * Pointer to the list of actions.
1385 * Pointer to the error structure.
1388 * 0 on success, else a negative errno value otherwise and rte_errno is set.
1391 flow_verbs_translate(struct rte_eth_dev
*dev
,
1392 struct mlx5_flow
*dev_flow
,
1393 const struct rte_flow_attr
*attr
,
1394 const struct rte_flow_item items
[],
1395 const struct rte_flow_action actions
[],
1396 struct rte_flow_error
*error
)
1398 struct rte_flow
*flow
= dev_flow
->flow
;
1399 uint64_t item_flags
= 0;
1400 uint64_t action_flags
= 0;
1401 uint64_t priority
= attr
->priority
;
1402 uint32_t subpriority
= 0;
1403 struct mlx5_priv
*priv
= dev
->data
->dev_private
;
1405 if (priority
== MLX5_FLOW_PRIO_RSVD
)
1406 priority
= priv
->config
.flow_prio
- 1;
1407 for (; actions
->type
!= RTE_FLOW_ACTION_TYPE_END
; actions
++) {
1410 switch (actions
->type
) {
1411 case RTE_FLOW_ACTION_TYPE_VOID
:
1413 case RTE_FLOW_ACTION_TYPE_FLAG
:
1414 flow_verbs_translate_action_flag(dev_flow
, actions
);
1415 action_flags
|= MLX5_FLOW_ACTION_FLAG
;
1417 case RTE_FLOW_ACTION_TYPE_MARK
:
1418 flow_verbs_translate_action_mark(dev_flow
, actions
);
1419 action_flags
|= MLX5_FLOW_ACTION_MARK
;
1421 case RTE_FLOW_ACTION_TYPE_DROP
:
1422 flow_verbs_translate_action_drop(dev_flow
, actions
);
1423 action_flags
|= MLX5_FLOW_ACTION_DROP
;
1425 case RTE_FLOW_ACTION_TYPE_QUEUE
:
1426 flow_verbs_translate_action_queue(dev_flow
, actions
);
1427 action_flags
|= MLX5_FLOW_ACTION_QUEUE
;
1429 case RTE_FLOW_ACTION_TYPE_RSS
:
1430 flow_verbs_translate_action_rss(dev_flow
, actions
);
1431 action_flags
|= MLX5_FLOW_ACTION_RSS
;
1433 case RTE_FLOW_ACTION_TYPE_COUNT
:
1434 ret
= flow_verbs_translate_action_count(dev_flow
,
1439 action_flags
|= MLX5_FLOW_ACTION_COUNT
;
1442 return rte_flow_error_set(error
, ENOTSUP
,
1443 RTE_FLOW_ERROR_TYPE_ACTION
,
1445 "action not supported");
1448 flow
->actions
= action_flags
;
1449 for (; items
->type
!= RTE_FLOW_ITEM_TYPE_END
; items
++) {
1450 int tunnel
= !!(item_flags
& MLX5_FLOW_LAYER_TUNNEL
);
1452 switch (items
->type
) {
1453 case RTE_FLOW_ITEM_TYPE_VOID
:
1455 case RTE_FLOW_ITEM_TYPE_ETH
:
1456 flow_verbs_translate_item_eth(dev_flow
, items
,
1458 subpriority
= MLX5_PRIORITY_MAP_L2
;
1459 item_flags
|= tunnel
? MLX5_FLOW_LAYER_INNER_L2
:
1460 MLX5_FLOW_LAYER_OUTER_L2
;
1462 case RTE_FLOW_ITEM_TYPE_VLAN
:
1463 flow_verbs_translate_item_vlan(dev_flow
, items
,
1465 subpriority
= MLX5_PRIORITY_MAP_L2
;
1466 item_flags
|= tunnel
? (MLX5_FLOW_LAYER_INNER_L2
|
1467 MLX5_FLOW_LAYER_INNER_VLAN
) :
1468 (MLX5_FLOW_LAYER_OUTER_L2
|
1469 MLX5_FLOW_LAYER_OUTER_VLAN
);
1471 case RTE_FLOW_ITEM_TYPE_IPV4
:
1472 flow_verbs_translate_item_ipv4(dev_flow
, items
,
1474 subpriority
= MLX5_PRIORITY_MAP_L3
;
1475 dev_flow
->verbs
.hash_fields
|=
1476 mlx5_flow_hashfields_adjust
1478 MLX5_IPV4_LAYER_TYPES
,
1479 MLX5_IPV4_IBV_RX_HASH
);
1480 item_flags
|= tunnel
? MLX5_FLOW_LAYER_INNER_L3_IPV4
:
1481 MLX5_FLOW_LAYER_OUTER_L3_IPV4
;
1483 case RTE_FLOW_ITEM_TYPE_IPV6
:
1484 flow_verbs_translate_item_ipv6(dev_flow
, items
,
1486 subpriority
= MLX5_PRIORITY_MAP_L3
;
1487 dev_flow
->verbs
.hash_fields
|=
1488 mlx5_flow_hashfields_adjust
1490 MLX5_IPV6_LAYER_TYPES
,
1491 MLX5_IPV6_IBV_RX_HASH
);
1492 item_flags
|= tunnel
? MLX5_FLOW_LAYER_INNER_L3_IPV6
:
1493 MLX5_FLOW_LAYER_OUTER_L3_IPV6
;
1495 case RTE_FLOW_ITEM_TYPE_TCP
:
1496 flow_verbs_translate_item_tcp(dev_flow
, items
,
1498 subpriority
= MLX5_PRIORITY_MAP_L4
;
1499 dev_flow
->verbs
.hash_fields
|=
1500 mlx5_flow_hashfields_adjust
1501 (dev_flow
, tunnel
, ETH_RSS_TCP
,
1502 (IBV_RX_HASH_SRC_PORT_TCP
|
1503 IBV_RX_HASH_DST_PORT_TCP
));
1504 item_flags
|= tunnel
? MLX5_FLOW_LAYER_INNER_L4_TCP
:
1505 MLX5_FLOW_LAYER_OUTER_L4_TCP
;
1507 case RTE_FLOW_ITEM_TYPE_UDP
:
1508 flow_verbs_translate_item_udp(dev_flow
, items
,
1510 subpriority
= MLX5_PRIORITY_MAP_L4
;
1511 dev_flow
->verbs
.hash_fields
|=
1512 mlx5_flow_hashfields_adjust
1513 (dev_flow
, tunnel
, ETH_RSS_UDP
,
1514 (IBV_RX_HASH_SRC_PORT_UDP
|
1515 IBV_RX_HASH_DST_PORT_UDP
));
1516 item_flags
|= tunnel
? MLX5_FLOW_LAYER_INNER_L4_UDP
:
1517 MLX5_FLOW_LAYER_OUTER_L4_UDP
;
1519 case RTE_FLOW_ITEM_TYPE_VXLAN
:
1520 flow_verbs_translate_item_vxlan(dev_flow
, items
,
1522 subpriority
= MLX5_PRIORITY_MAP_L2
;
1523 item_flags
|= MLX5_FLOW_LAYER_VXLAN
;
1525 case RTE_FLOW_ITEM_TYPE_VXLAN_GPE
:
1526 flow_verbs_translate_item_vxlan_gpe(dev_flow
, items
,
1528 subpriority
= MLX5_PRIORITY_MAP_L2
;
1529 item_flags
|= MLX5_FLOW_LAYER_VXLAN_GPE
;
1531 case RTE_FLOW_ITEM_TYPE_GRE
:
1532 flow_verbs_translate_item_gre(dev_flow
, items
,
1534 subpriority
= MLX5_PRIORITY_MAP_L2
;
1535 item_flags
|= MLX5_FLOW_LAYER_GRE
;
1537 case RTE_FLOW_ITEM_TYPE_MPLS
:
1538 flow_verbs_translate_item_mpls(dev_flow
, items
,
1540 subpriority
= MLX5_PRIORITY_MAP_L2
;
1541 item_flags
|= MLX5_FLOW_LAYER_MPLS
;
1544 return rte_flow_error_set(error
, ENOTSUP
,
1545 RTE_FLOW_ERROR_TYPE_ITEM
,
1547 "item not supported");
1550 dev_flow
->layers
= item_flags
;
1551 dev_flow
->verbs
.attr
->priority
=
1552 mlx5_flow_adjust_priority(dev
, priority
, subpriority
);
1553 dev_flow
->verbs
.attr
->port
= (uint8_t)priv
->ibv_port
;
1558 * Remove the flow from the NIC but keeps it in memory.
1561 * Pointer to the Ethernet device structure.
1562 * @param[in, out] flow
1563 * Pointer to flow structure.
1566 flow_verbs_remove(struct rte_eth_dev
*dev
, struct rte_flow
*flow
)
1568 struct mlx5_flow_verbs
*verbs
;
1569 struct mlx5_flow
*dev_flow
;
1573 LIST_FOREACH(dev_flow
, &flow
->dev_flows
, next
) {
1574 verbs
= &dev_flow
->verbs
;
1576 claim_zero(mlx5_glue
->destroy_flow(verbs
->flow
));
1580 if (flow
->actions
& MLX5_FLOW_ACTION_DROP
)
1581 mlx5_hrxq_drop_release(dev
);
1583 mlx5_hrxq_release(dev
, verbs
->hrxq
);
1590 * Remove the flow from the NIC and the memory.
1593 * Pointer to the Ethernet device structure.
1594 * @param[in, out] flow
1595 * Pointer to flow structure.
1598 flow_verbs_destroy(struct rte_eth_dev
*dev
, struct rte_flow
*flow
)
1600 struct mlx5_flow
*dev_flow
;
1604 flow_verbs_remove(dev
, flow
);
1605 while (!LIST_EMPTY(&flow
->dev_flows
)) {
1606 dev_flow
= LIST_FIRST(&flow
->dev_flows
);
1607 LIST_REMOVE(dev_flow
, next
);
1610 if (flow
->counter
) {
1611 flow_verbs_counter_release(flow
->counter
);
1612 flow
->counter
= NULL
;
1617 * Apply the flow to the NIC.
1620 * Pointer to the Ethernet device structure.
1621 * @param[in, out] flow
1622 * Pointer to flow structure.
1624 * Pointer to error structure.
1627 * 0 on success, a negative errno value otherwise and rte_errno is set.
1630 flow_verbs_apply(struct rte_eth_dev
*dev
, struct rte_flow
*flow
,
1631 struct rte_flow_error
*error
)
1633 struct mlx5_flow_verbs
*verbs
;
1634 struct mlx5_flow
*dev_flow
;
1637 LIST_FOREACH(dev_flow
, &flow
->dev_flows
, next
) {
1638 verbs
= &dev_flow
->verbs
;
1639 if (flow
->actions
& MLX5_FLOW_ACTION_DROP
) {
1640 verbs
->hrxq
= mlx5_hrxq_drop_new(dev
);
1644 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
, NULL
,
1645 "cannot get drop hash queue");
1649 struct mlx5_hrxq
*hrxq
;
1651 hrxq
= mlx5_hrxq_get(dev
, flow
->key
,
1652 MLX5_RSS_HASH_KEY_LEN
,
1655 flow
->rss
.queue_num
);
1657 hrxq
= mlx5_hrxq_new(dev
, flow
->key
,
1658 MLX5_RSS_HASH_KEY_LEN
,
1661 flow
->rss
.queue_num
,
1662 !!(dev_flow
->layers
&
1663 MLX5_FLOW_LAYER_TUNNEL
));
1667 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
, NULL
,
1668 "cannot get hash queue");
1673 verbs
->flow
= mlx5_glue
->create_flow(verbs
->hrxq
->qp
,
1676 rte_flow_error_set(error
, errno
,
1677 RTE_FLOW_ERROR_TYPE_UNSPECIFIED
,
1679 "hardware refuses to create flow");
1685 err
= rte_errno
; /* Save rte_errno before cleanup. */
1686 LIST_FOREACH(dev_flow
, &flow
->dev_flows
, next
) {
1687 verbs
= &dev_flow
->verbs
;
1689 if (flow
->actions
& MLX5_FLOW_ACTION_DROP
)
1690 mlx5_hrxq_drop_release(dev
);
1692 mlx5_hrxq_release(dev
, verbs
->hrxq
);
1696 rte_errno
= err
; /* Restore rte_errno. */
1703 * @see rte_flow_query()
1707 flow_verbs_query(struct rte_eth_dev
*dev
,
1708 struct rte_flow
*flow
,
1709 const struct rte_flow_action
*actions
,
1711 struct rte_flow_error
*error
)
1715 for (; actions
->type
!= RTE_FLOW_ACTION_TYPE_END
; actions
++) {
1716 switch (actions
->type
) {
1717 case RTE_FLOW_ACTION_TYPE_VOID
:
1719 case RTE_FLOW_ACTION_TYPE_COUNT
:
1720 ret
= flow_verbs_counter_query(dev
, flow
, data
, error
);
1723 return rte_flow_error_set(error
, ENOTSUP
,
1724 RTE_FLOW_ERROR_TYPE_ACTION
,
1726 "action not supported");
1732 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops
= {
1733 .validate
= flow_verbs_validate
,
1734 .prepare
= flow_verbs_prepare
,
1735 .translate
= flow_verbs_translate
,
1736 .apply
= flow_verbs_apply
,
1737 .remove
= flow_verbs_remove
,
1738 .destroy
= flow_verbs_destroy
,
1739 .query
= flow_verbs_query
,