2 * Copyright (c) 2014, 2015, 2016, 2017 Nicira, Inc.
3 * Copyright (c) 2019 Mellanox Technologies, Ltd.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 #include <sys/types.h>
20 #include <netinet/ip6.h>
24 #include "dpif-netdev.h"
25 #include "netdev-offload-provider.h"
26 #include "netdev-provider.h"
27 #include "openvswitch/match.h"
28 #include "openvswitch/vlog.h"
32 VLOG_DEFINE_THIS_MODULE(netdev_offload_dpdk
);
33 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(100, 5);
38 * Below API is NOT thread safe in following terms:
40 * - The caller must be sure that none of these functions will be called
41 * simultaneously. Even for different 'netdev's.
43 * - The caller must be sure that 'netdev' will not be destructed/deallocated.
45 * - The caller must be sure that 'netdev' configuration will not be changed.
46 * For example, simultaneous call of 'netdev_reconfigure()' for the same
47 * 'netdev' is forbidden.
49 * For current implementation all above restrictions could be fulfilled by
50 * taking the datapath 'port_mutex' in lib/dpif-netdev.c. */
53 * A mapping from ufid to dpdk rte_flow.
55 static struct cmap ufid_to_rte_flow
= CMAP_INITIALIZER
;
57 struct ufid_to_rte_flow_data
{
58 struct cmap_node node
;
60 struct rte_flow
*rte_flow
;
61 bool actions_offloaded
;
62 struct dpif_flow_stats stats
;
65 /* Find rte_flow with @ufid. */
66 static struct ufid_to_rte_flow_data
*
67 ufid_to_rte_flow_data_find(const ovs_u128
*ufid
)
69 size_t hash
= hash_bytes(ufid
, sizeof *ufid
, 0);
70 struct ufid_to_rte_flow_data
*data
;
72 CMAP_FOR_EACH_WITH_HASH (data
, node
, hash
, &ufid_to_rte_flow
) {
73 if (ovs_u128_equals(*ufid
, data
->ufid
)) {
82 ufid_to_rte_flow_associate(const ovs_u128
*ufid
,
83 struct rte_flow
*rte_flow
, bool actions_offloaded
)
85 size_t hash
= hash_bytes(ufid
, sizeof *ufid
, 0);
86 struct ufid_to_rte_flow_data
*data
= xzalloc(sizeof *data
);
87 struct ufid_to_rte_flow_data
*data_prev
;
90 * We should not simply overwrite an existing rte flow.
91 * We should have deleted it first before re-adding it.
92 * Thus, if following assert triggers, something is wrong:
93 * the rte_flow is not destroyed.
95 data_prev
= ufid_to_rte_flow_data_find(ufid
);
97 ovs_assert(data_prev
->rte_flow
== NULL
);
101 data
->rte_flow
= rte_flow
;
102 data
->actions_offloaded
= actions_offloaded
;
104 cmap_insert(&ufid_to_rte_flow
,
105 CONST_CAST(struct cmap_node
*, &data
->node
), hash
);
109 ufid_to_rte_flow_disassociate(const ovs_u128
*ufid
)
111 size_t hash
= hash_bytes(ufid
, sizeof *ufid
, 0);
112 struct ufid_to_rte_flow_data
*data
;
114 CMAP_FOR_EACH_WITH_HASH (data
, node
, hash
, &ufid_to_rte_flow
) {
115 if (ovs_u128_equals(*ufid
, data
->ufid
)) {
116 cmap_remove(&ufid_to_rte_flow
,
117 CONST_CAST(struct cmap_node
*, &data
->node
), hash
);
118 ovsrcu_postpone(free
, data
);
123 VLOG_WARN("ufid "UUID_FMT
" is not associated with an rte flow",
124 UUID_ARGS((struct uuid
*) ufid
));
128 * To avoid individual xrealloc calls for each new element, a 'curent_max'
129 * is used to keep track of current allocated number of elements. Starts
130 * by 8 and doubles on each xrealloc call.
132 struct flow_patterns
{
133 struct rte_flow_item
*items
;
138 struct flow_actions
{
139 struct rte_flow_action
*actions
;
145 dump_flow_attr(struct ds
*s
, const struct rte_flow_attr
*attr
)
147 ds_put_format(s
, "%s%spriority %"PRIu32
" group %"PRIu32
" %s",
148 attr
->ingress
? "ingress " : "",
149 attr
->egress
? "egress " : "", attr
->priority
, attr
->group
,
150 attr
->transfer
? "transfer " : "");
153 /* Adds one pattern item 'field' with the 'mask' to dynamic string 's' using
154 * 'testpmd command'-like format. */
155 #define DUMP_PATTERN_ITEM(mask, field, fmt, spec_pri, mask_pri) \
156 if (is_all_ones(&mask, sizeof mask)) { \
157 ds_put_format(s, field " is " fmt " ", spec_pri); \
158 } else if (!is_all_zeros(&mask, sizeof mask)) { \
159 ds_put_format(s, field " spec " fmt " " field " mask " fmt " ", \
160 spec_pri, mask_pri); \
164 dump_flow_pattern(struct ds
*s
, const struct rte_flow_item
*item
)
166 if (item
->type
== RTE_FLOW_ITEM_TYPE_ETH
) {
167 const struct rte_flow_item_eth
*eth_spec
= item
->spec
;
168 const struct rte_flow_item_eth
*eth_mask
= item
->mask
;
170 ds_put_cstr(s
, "eth ");
173 eth_mask
= &rte_flow_item_eth_mask
;
175 DUMP_PATTERN_ITEM(eth_mask
->src
, "src", ETH_ADDR_FMT
,
176 ETH_ADDR_BYTES_ARGS(eth_spec
->src
.addr_bytes
),
177 ETH_ADDR_BYTES_ARGS(eth_mask
->src
.addr_bytes
));
178 DUMP_PATTERN_ITEM(eth_mask
->dst
, "dst", ETH_ADDR_FMT
,
179 ETH_ADDR_BYTES_ARGS(eth_spec
->dst
.addr_bytes
),
180 ETH_ADDR_BYTES_ARGS(eth_mask
->dst
.addr_bytes
));
181 DUMP_PATTERN_ITEM(eth_mask
->type
, "type", "0x%04"PRIx16
,
182 ntohs(eth_spec
->type
),
183 ntohs(eth_mask
->type
));
185 ds_put_cstr(s
, "/ ");
186 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_VLAN
) {
187 const struct rte_flow_item_vlan
*vlan_spec
= item
->spec
;
188 const struct rte_flow_item_vlan
*vlan_mask
= item
->mask
;
190 ds_put_cstr(s
, "vlan ");
193 vlan_mask
= &rte_flow_item_vlan_mask
;
195 DUMP_PATTERN_ITEM(vlan_mask
->inner_type
, "inner_type", "0x%"PRIx16
,
196 ntohs(vlan_spec
->inner_type
),
197 ntohs(vlan_mask
->inner_type
));
198 DUMP_PATTERN_ITEM(vlan_mask
->tci
, "tci", "0x%"PRIx16
,
199 ntohs(vlan_spec
->tci
), ntohs(vlan_mask
->tci
));
201 ds_put_cstr(s
, "/ ");
202 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_IPV4
) {
203 const struct rte_flow_item_ipv4
*ipv4_spec
= item
->spec
;
204 const struct rte_flow_item_ipv4
*ipv4_mask
= item
->mask
;
206 ds_put_cstr(s
, "ipv4 ");
209 ipv4_mask
= &rte_flow_item_ipv4_mask
;
211 DUMP_PATTERN_ITEM(ipv4_mask
->hdr
.src_addr
, "src", IP_FMT
,
212 IP_ARGS(ipv4_spec
->hdr
.src_addr
),
213 IP_ARGS(ipv4_mask
->hdr
.src_addr
));
214 DUMP_PATTERN_ITEM(ipv4_mask
->hdr
.dst_addr
, "dst", IP_FMT
,
215 IP_ARGS(ipv4_spec
->hdr
.dst_addr
),
216 IP_ARGS(ipv4_mask
->hdr
.dst_addr
));
217 DUMP_PATTERN_ITEM(ipv4_mask
->hdr
.next_proto_id
, "proto",
218 "0x%"PRIx8
, ipv4_spec
->hdr
.next_proto_id
,
219 ipv4_mask
->hdr
.next_proto_id
);
220 DUMP_PATTERN_ITEM(ipv4_mask
->hdr
.type_of_service
, "tos",
221 "0x%"PRIx8
, ipv4_spec
->hdr
.type_of_service
,
222 ipv4_mask
->hdr
.type_of_service
);
223 DUMP_PATTERN_ITEM(ipv4_mask
->hdr
.time_to_live
, "ttl",
224 "0x%"PRIx8
, ipv4_spec
->hdr
.time_to_live
,
225 ipv4_mask
->hdr
.time_to_live
);
227 ds_put_cstr(s
, "/ ");
228 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_UDP
) {
229 const struct rte_flow_item_udp
*udp_spec
= item
->spec
;
230 const struct rte_flow_item_udp
*udp_mask
= item
->mask
;
232 ds_put_cstr(s
, "udp ");
235 udp_mask
= &rte_flow_item_udp_mask
;
237 DUMP_PATTERN_ITEM(udp_mask
->hdr
.src_port
, "src", "%"PRIu16
,
238 ntohs(udp_spec
->hdr
.src_port
),
239 ntohs(udp_mask
->hdr
.src_port
));
240 DUMP_PATTERN_ITEM(udp_mask
->hdr
.dst_port
, "dst", "%"PRIu16
,
241 ntohs(udp_spec
->hdr
.dst_port
),
242 ntohs(udp_mask
->hdr
.dst_port
));
244 ds_put_cstr(s
, "/ ");
245 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_SCTP
) {
246 const struct rte_flow_item_sctp
*sctp_spec
= item
->spec
;
247 const struct rte_flow_item_sctp
*sctp_mask
= item
->mask
;
249 ds_put_cstr(s
, "sctp ");
252 sctp_mask
= &rte_flow_item_sctp_mask
;
254 DUMP_PATTERN_ITEM(sctp_mask
->hdr
.src_port
, "src", "%"PRIu16
,
255 ntohs(sctp_spec
->hdr
.src_port
),
256 ntohs(sctp_mask
->hdr
.src_port
));
257 DUMP_PATTERN_ITEM(sctp_mask
->hdr
.dst_port
, "dst", "%"PRIu16
,
258 ntohs(sctp_spec
->hdr
.dst_port
),
259 ntohs(sctp_mask
->hdr
.dst_port
));
261 ds_put_cstr(s
, "/ ");
262 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_ICMP
) {
263 const struct rte_flow_item_icmp
*icmp_spec
= item
->spec
;
264 const struct rte_flow_item_icmp
*icmp_mask
= item
->mask
;
266 ds_put_cstr(s
, "icmp ");
269 icmp_mask
= &rte_flow_item_icmp_mask
;
271 DUMP_PATTERN_ITEM(icmp_mask
->hdr
.icmp_type
, "icmp_type", "%"PRIu8
,
272 icmp_spec
->hdr
.icmp_type
,
273 icmp_mask
->hdr
.icmp_type
);
274 DUMP_PATTERN_ITEM(icmp_mask
->hdr
.icmp_code
, "icmp_code", "%"PRIu8
,
275 icmp_spec
->hdr
.icmp_code
,
276 icmp_mask
->hdr
.icmp_code
);
278 ds_put_cstr(s
, "/ ");
279 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_TCP
) {
280 const struct rte_flow_item_tcp
*tcp_spec
= item
->spec
;
281 const struct rte_flow_item_tcp
*tcp_mask
= item
->mask
;
283 ds_put_cstr(s
, "tcp ");
286 tcp_mask
= &rte_flow_item_tcp_mask
;
288 DUMP_PATTERN_ITEM(tcp_mask
->hdr
.src_port
, "src", "%"PRIu16
,
289 ntohs(tcp_spec
->hdr
.src_port
),
290 ntohs(tcp_mask
->hdr
.src_port
));
291 DUMP_PATTERN_ITEM(tcp_mask
->hdr
.dst_port
, "dst", "%"PRIu16
,
292 ntohs(tcp_spec
->hdr
.dst_port
),
293 ntohs(tcp_mask
->hdr
.dst_port
));
294 DUMP_PATTERN_ITEM(tcp_mask
->hdr
.tcp_flags
, "flags", "0x%"PRIx8
,
295 tcp_spec
->hdr
.tcp_flags
,
296 tcp_mask
->hdr
.tcp_flags
);
298 ds_put_cstr(s
, "/ ");
299 } else if (item
->type
== RTE_FLOW_ITEM_TYPE_IPV6
) {
300 const struct rte_flow_item_ipv6
*ipv6_spec
= item
->spec
;
301 const struct rte_flow_item_ipv6
*ipv6_mask
= item
->mask
;
303 char addr_str
[INET6_ADDRSTRLEN
];
304 char mask_str
[INET6_ADDRSTRLEN
];
305 struct in6_addr addr
, mask
;
307 ds_put_cstr(s
, "ipv6 ");
310 ipv6_mask
= &rte_flow_item_ipv6_mask
;
312 memcpy(&addr
, ipv6_spec
->hdr
.src_addr
, sizeof addr
);
313 memcpy(&mask
, ipv6_mask
->hdr
.src_addr
, sizeof mask
);
314 ipv6_string_mapped(addr_str
, &addr
);
315 ipv6_string_mapped(mask_str
, &mask
);
316 DUMP_PATTERN_ITEM(mask
, "src", "%s", addr_str
, mask_str
);
318 memcpy(&addr
, ipv6_spec
->hdr
.dst_addr
, sizeof addr
);
319 memcpy(&mask
, ipv6_mask
->hdr
.dst_addr
, sizeof mask
);
320 ipv6_string_mapped(addr_str
, &addr
);
321 ipv6_string_mapped(mask_str
, &mask
);
322 DUMP_PATTERN_ITEM(mask
, "dst", "%s", addr_str
, mask_str
);
324 DUMP_PATTERN_ITEM(ipv6_mask
->hdr
.proto
, "proto", "%"PRIu8
,
325 ipv6_spec
->hdr
.proto
, ipv6_mask
->hdr
.proto
);
326 DUMP_PATTERN_ITEM(ipv6_mask
->hdr
.vtc_flow
, "tc", "0x%"PRIx32
,
327 ntohl(ipv6_spec
->hdr
.vtc_flow
),
328 ntohl(ipv6_mask
->hdr
.vtc_flow
));
329 DUMP_PATTERN_ITEM(ipv6_mask
->hdr
.hop_limits
, "hop", "%"PRIu8
,
330 ipv6_spec
->hdr
.hop_limits
,
331 ipv6_mask
->hdr
.hop_limits
);
333 ds_put_cstr(s
, "/ ");
335 ds_put_format(s
, "unknown rte flow pattern (%d)\n", item
->type
);
340 dump_flow_action(struct ds
*s
, struct ds
*s_extra
,
341 const struct rte_flow_action
*actions
)
343 if (actions
->type
== RTE_FLOW_ACTION_TYPE_MARK
) {
344 const struct rte_flow_action_mark
*mark
= actions
->conf
;
346 ds_put_cstr(s
, "mark ");
348 ds_put_format(s
, "id %d ", mark
->id
);
350 ds_put_cstr(s
, "/ ");
351 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_RSS
) {
352 ds_put_cstr(s
, "rss / ");
353 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_COUNT
) {
354 ds_put_cstr(s
, "count / ");
355 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_PORT_ID
) {
356 const struct rte_flow_action_port_id
*port_id
= actions
->conf
;
358 ds_put_cstr(s
, "port_id ");
360 ds_put_format(s
, "original %d id %d ",
361 port_id
->original
, port_id
->id
);
363 ds_put_cstr(s
, "/ ");
364 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_DROP
) {
365 ds_put_cstr(s
, "drop / ");
366 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_SET_MAC_SRC
||
367 actions
->type
== RTE_FLOW_ACTION_TYPE_SET_MAC_DST
) {
368 const struct rte_flow_action_set_mac
*set_mac
= actions
->conf
;
370 char *dirstr
= actions
->type
== RTE_FLOW_ACTION_TYPE_SET_MAC_DST
373 ds_put_format(s
, "set_mac_%s ", dirstr
);
375 ds_put_format(s
, "mac_addr "ETH_ADDR_FMT
" ",
376 ETH_ADDR_BYTES_ARGS(set_mac
->mac_addr
));
378 ds_put_cstr(s
, "/ ");
379 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC
||
380 actions
->type
== RTE_FLOW_ACTION_TYPE_SET_IPV4_DST
) {
381 const struct rte_flow_action_set_ipv4
*set_ipv4
= actions
->conf
;
382 char *dirstr
= actions
->type
== RTE_FLOW_ACTION_TYPE_SET_IPV4_DST
385 ds_put_format(s
, "set_ipv4_%s ", dirstr
);
387 ds_put_format(s
, "ipv4_addr "IP_FMT
" ",
388 IP_ARGS(set_ipv4
->ipv4_addr
));
390 ds_put_cstr(s
, "/ ");
391 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_SET_TTL
) {
392 const struct rte_flow_action_set_ttl
*set_ttl
= actions
->conf
;
394 ds_put_cstr(s
, "set_ttl ");
396 ds_put_format(s
, "ttl_value %d ", set_ttl
->ttl_value
);
398 ds_put_cstr(s
, "/ ");
399 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_SET_TP_SRC
||
400 actions
->type
== RTE_FLOW_ACTION_TYPE_SET_TP_DST
) {
401 const struct rte_flow_action_set_tp
*set_tp
= actions
->conf
;
402 char *dirstr
= actions
->type
== RTE_FLOW_ACTION_TYPE_SET_TP_DST
405 ds_put_format(s
, "set_tp_%s ", dirstr
);
407 ds_put_format(s
, "port %"PRIu16
" ", ntohs(set_tp
->port
));
409 ds_put_cstr(s
, "/ ");
410 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN
) {
411 const struct rte_flow_action_of_push_vlan
*of_push_vlan
=
414 ds_put_cstr(s
, "of_push_vlan ");
416 ds_put_format(s
, "ethertype 0x%"PRIx16
" ",
417 ntohs(of_push_vlan
->ethertype
));
419 ds_put_cstr(s
, "/ ");
420 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP
) {
421 const struct rte_flow_action_of_set_vlan_pcp
*of_set_vlan_pcp
=
424 ds_put_cstr(s
, "of_set_vlan_pcp ");
425 if (of_set_vlan_pcp
) {
426 ds_put_format(s
, "vlan_pcp %"PRIu8
" ", of_set_vlan_pcp
->vlan_pcp
);
428 ds_put_cstr(s
, "/ ");
429 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID
) {
430 const struct rte_flow_action_of_set_vlan_vid
*of_set_vlan_vid
=
433 ds_put_cstr(s
, "of_set_vlan_vid ");
434 if (of_set_vlan_vid
) {
435 ds_put_format(s
, "vlan_vid %"PRIu16
" ",
436 ntohs(of_set_vlan_vid
->vlan_vid
));
438 ds_put_cstr(s
, "/ ");
439 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_OF_POP_VLAN
) {
440 ds_put_cstr(s
, "of_pop_vlan / ");
441 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC
||
442 actions
->type
== RTE_FLOW_ACTION_TYPE_SET_IPV6_DST
) {
443 const struct rte_flow_action_set_ipv6
*set_ipv6
= actions
->conf
;
445 char *dirstr
= actions
->type
== RTE_FLOW_ACTION_TYPE_SET_IPV6_DST
448 ds_put_format(s
, "set_ipv6_%s ", dirstr
);
450 ds_put_cstr(s
, "ipv6_addr ");
451 ipv6_format_addr((struct in6_addr
*) &set_ipv6
->ipv6_addr
, s
);
454 ds_put_cstr(s
, "/ ");
455 } else if (actions
->type
== RTE_FLOW_ACTION_TYPE_RAW_ENCAP
) {
456 const struct rte_flow_action_raw_encap
*raw_encap
= actions
->conf
;
458 ds_put_cstr(s
, "raw_encap index 0 / ");
460 ds_put_format(s_extra
, "Raw-encap size=%ld set raw_encap 0 raw "
461 "pattern is ", raw_encap
->size
);
462 for (int i
= 0; i
< raw_encap
->size
; i
++) {
463 ds_put_format(s_extra
, "%02x", raw_encap
->data
[i
]);
465 ds_put_cstr(s_extra
, " / end_set;");
468 ds_put_format(s
, "unknown rte flow action (%d)\n", actions
->type
);
473 dump_flow(struct ds
*s
, struct ds
*s_extra
,
474 const struct rte_flow_attr
*attr
,
475 const struct rte_flow_item
*items
,
476 const struct rte_flow_action
*actions
)
479 dump_flow_attr(s
, attr
);
481 ds_put_cstr(s
, "pattern ");
482 while (items
&& items
->type
!= RTE_FLOW_ITEM_TYPE_END
) {
483 dump_flow_pattern(s
, items
++);
485 ds_put_cstr(s
, "end actions ");
486 while (actions
&& actions
->type
!= RTE_FLOW_ACTION_TYPE_END
) {
487 dump_flow_action(s
, s_extra
, actions
++);
489 ds_put_cstr(s
, "end");
493 static struct rte_flow
*
494 netdev_offload_dpdk_flow_create(struct netdev
*netdev
,
495 const struct rte_flow_attr
*attr
,
496 const struct rte_flow_item
*items
,
497 const struct rte_flow_action
*actions
,
498 struct rte_flow_error
*error
)
500 struct ds s_extra
= DS_EMPTY_INITIALIZER
;
501 struct ds s
= DS_EMPTY_INITIALIZER
;
502 struct rte_flow
*flow
;
505 flow
= netdev_dpdk_rte_flow_create(netdev
, attr
, items
, actions
, error
);
507 if (!VLOG_DROP_DBG(&rl
)) {
508 dump_flow(&s
, &s_extra
, attr
, items
, actions
);
509 extra_str
= ds_cstr(&s_extra
);
510 VLOG_DBG_RL(&rl
, "%s: rte_flow 0x%"PRIxPTR
" %s flow create %d %s",
511 netdev_get_name(netdev
), (intptr_t) flow
, extra_str
,
512 netdev_dpdk_get_port_id(netdev
), ds_cstr(&s
));
515 enum vlog_level level
= VLL_WARN
;
517 if (error
->type
== RTE_FLOW_ERROR_TYPE_ACTION
) {
520 VLOG_RL(&rl
, level
, "%s: rte_flow creation failed: %d (%s).",
521 netdev_get_name(netdev
), error
->type
, error
->message
);
522 if (!vlog_should_drop(&this_module
, level
, &rl
)) {
523 dump_flow(&s
, &s_extra
, attr
, items
, actions
);
524 extra_str
= ds_cstr(&s_extra
);
525 VLOG_RL(&rl
, level
, "%s: Failed flow: %s flow create %d %s",
526 netdev_get_name(netdev
), extra_str
,
527 netdev_dpdk_get_port_id(netdev
), ds_cstr(&s
));
531 ds_destroy(&s_extra
);
536 add_flow_pattern(struct flow_patterns
*patterns
, enum rte_flow_item_type type
,
537 const void *spec
, const void *mask
)
539 int cnt
= patterns
->cnt
;
542 patterns
->current_max
= 8;
543 patterns
->items
= xcalloc(patterns
->current_max
,
544 sizeof *patterns
->items
);
545 } else if (cnt
== patterns
->current_max
) {
546 patterns
->current_max
*= 2;
547 patterns
->items
= xrealloc(patterns
->items
, patterns
->current_max
*
548 sizeof *patterns
->items
);
551 patterns
->items
[cnt
].type
= type
;
552 patterns
->items
[cnt
].spec
= spec
;
553 patterns
->items
[cnt
].mask
= mask
;
554 patterns
->items
[cnt
].last
= NULL
;
559 add_flow_action(struct flow_actions
*actions
, enum rte_flow_action_type type
,
562 int cnt
= actions
->cnt
;
565 actions
->current_max
= 8;
566 actions
->actions
= xcalloc(actions
->current_max
,
567 sizeof *actions
->actions
);
568 } else if (cnt
== actions
->current_max
) {
569 actions
->current_max
*= 2;
570 actions
->actions
= xrealloc(actions
->actions
, actions
->current_max
*
571 sizeof *actions
->actions
);
574 actions
->actions
[cnt
].type
= type
;
575 actions
->actions
[cnt
].conf
= conf
;
580 free_flow_patterns(struct flow_patterns
*patterns
)
584 for (i
= 0; i
< patterns
->cnt
; i
++) {
585 if (patterns
->items
[i
].spec
) {
586 free(CONST_CAST(void *, patterns
->items
[i
].spec
));
588 if (patterns
->items
[i
].mask
) {
589 free(CONST_CAST(void *, patterns
->items
[i
].mask
));
592 free(patterns
->items
);
593 patterns
->items
= NULL
;
598 free_flow_actions(struct flow_actions
*actions
)
602 for (i
= 0; i
< actions
->cnt
; i
++) {
603 if (actions
->actions
[i
].conf
) {
604 free(CONST_CAST(void *, actions
->actions
[i
].conf
));
607 free(actions
->actions
);
608 actions
->actions
= NULL
;
613 parse_flow_match(struct flow_patterns
*patterns
,
616 uint8_t *next_proto_mask
= NULL
;
617 struct flow
*consumed_masks
;
620 consumed_masks
= &match
->wc
.masks
;
622 memset(&consumed_masks
->in_port
, 0, sizeof consumed_masks
->in_port
);
623 /* recirc id must be zero. */
624 if (match
->wc
.masks
.recirc_id
& match
->flow
.recirc_id
) {
627 consumed_masks
->recirc_id
= 0;
628 consumed_masks
->packet_type
= 0;
631 if (match
->wc
.masks
.dl_type
||
632 !eth_addr_is_zero(match
->wc
.masks
.dl_src
) ||
633 !eth_addr_is_zero(match
->wc
.masks
.dl_dst
)) {
634 struct rte_flow_item_eth
*spec
, *mask
;
636 spec
= xzalloc(sizeof *spec
);
637 mask
= xzalloc(sizeof *mask
);
639 memcpy(&spec
->dst
, &match
->flow
.dl_dst
, sizeof spec
->dst
);
640 memcpy(&spec
->src
, &match
->flow
.dl_src
, sizeof spec
->src
);
641 spec
->type
= match
->flow
.dl_type
;
643 memcpy(&mask
->dst
, &match
->wc
.masks
.dl_dst
, sizeof mask
->dst
);
644 memcpy(&mask
->src
, &match
->wc
.masks
.dl_src
, sizeof mask
->src
);
645 mask
->type
= match
->wc
.masks
.dl_type
;
647 memset(&consumed_masks
->dl_dst
, 0, sizeof consumed_masks
->dl_dst
);
648 memset(&consumed_masks
->dl_src
, 0, sizeof consumed_masks
->dl_src
);
649 consumed_masks
->dl_type
= 0;
651 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_ETH
, spec
, mask
);
655 if (match
->wc
.masks
.vlans
[0].tci
&& match
->flow
.vlans
[0].tci
) {
656 struct rte_flow_item_vlan
*spec
, *mask
;
658 spec
= xzalloc(sizeof *spec
);
659 mask
= xzalloc(sizeof *mask
);
661 spec
->tci
= match
->flow
.vlans
[0].tci
& ~htons(VLAN_CFI
);
662 mask
->tci
= match
->wc
.masks
.vlans
[0].tci
& ~htons(VLAN_CFI
);
664 /* Match any protocols. */
665 mask
->inner_type
= 0;
667 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_VLAN
, spec
, mask
);
669 /* For untagged matching match->wc.masks.vlans[0].tci is 0xFFFF and
670 * match->flow.vlans[0].tci is 0. Consuming is needed outside of the if
671 * scope to handle that.
673 memset(&consumed_masks
->vlans
[0], 0, sizeof consumed_masks
->vlans
[0]);
676 if (match
->flow
.dl_type
== htons(ETH_TYPE_IP
)) {
677 struct rte_flow_item_ipv4
*spec
, *mask
;
679 spec
= xzalloc(sizeof *spec
);
680 mask
= xzalloc(sizeof *mask
);
682 spec
->hdr
.type_of_service
= match
->flow
.nw_tos
;
683 spec
->hdr
.time_to_live
= match
->flow
.nw_ttl
;
684 spec
->hdr
.next_proto_id
= match
->flow
.nw_proto
;
685 spec
->hdr
.src_addr
= match
->flow
.nw_src
;
686 spec
->hdr
.dst_addr
= match
->flow
.nw_dst
;
688 mask
->hdr
.type_of_service
= match
->wc
.masks
.nw_tos
;
689 mask
->hdr
.time_to_live
= match
->wc
.masks
.nw_ttl
;
690 mask
->hdr
.next_proto_id
= match
->wc
.masks
.nw_proto
;
691 mask
->hdr
.src_addr
= match
->wc
.masks
.nw_src
;
692 mask
->hdr
.dst_addr
= match
->wc
.masks
.nw_dst
;
694 consumed_masks
->nw_tos
= 0;
695 consumed_masks
->nw_ttl
= 0;
696 consumed_masks
->nw_proto
= 0;
697 consumed_masks
->nw_src
= 0;
698 consumed_masks
->nw_dst
= 0;
700 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_IPV4
, spec
, mask
);
702 /* Save proto for L4 protocol setup. */
703 proto
= spec
->hdr
.next_proto_id
&
704 mask
->hdr
.next_proto_id
;
705 next_proto_mask
= &mask
->hdr
.next_proto_id
;
707 /* If fragmented, then don't HW accelerate - for now. */
708 if (match
->wc
.masks
.nw_frag
& match
->flow
.nw_frag
) {
711 consumed_masks
->nw_frag
= 0;
714 if (match
->flow
.dl_type
== htons(ETH_TYPE_IPV6
)) {
715 struct rte_flow_item_ipv6
*spec
, *mask
;
717 spec
= xzalloc(sizeof *spec
);
718 mask
= xzalloc(sizeof *mask
);
720 spec
->hdr
.proto
= match
->flow
.nw_proto
;
721 spec
->hdr
.hop_limits
= match
->flow
.nw_ttl
;
723 htonl((uint32_t) match
->flow
.nw_tos
<< RTE_IPV6_HDR_TC_SHIFT
);
724 memcpy(spec
->hdr
.src_addr
, &match
->flow
.ipv6_src
,
725 sizeof spec
->hdr
.src_addr
);
726 memcpy(spec
->hdr
.dst_addr
, &match
->flow
.ipv6_dst
,
727 sizeof spec
->hdr
.dst_addr
);
729 mask
->hdr
.proto
= match
->wc
.masks
.nw_proto
;
730 mask
->hdr
.hop_limits
= match
->wc
.masks
.nw_ttl
;
732 htonl((uint32_t) match
->wc
.masks
.nw_tos
<< RTE_IPV6_HDR_TC_SHIFT
);
733 memcpy(mask
->hdr
.src_addr
, &match
->wc
.masks
.ipv6_src
,
734 sizeof mask
->hdr
.src_addr
);
735 memcpy(mask
->hdr
.dst_addr
, &match
->wc
.masks
.ipv6_dst
,
736 sizeof mask
->hdr
.dst_addr
);
738 consumed_masks
->nw_proto
= 0;
739 consumed_masks
->nw_ttl
= 0;
740 consumed_masks
->nw_tos
= 0;
741 memset(&consumed_masks
->ipv6_src
, 0, sizeof consumed_masks
->ipv6_src
);
742 memset(&consumed_masks
->ipv6_dst
, 0, sizeof consumed_masks
->ipv6_dst
);
744 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_IPV6
, spec
, mask
);
746 /* Save proto for L4 protocol setup. */
747 proto
= spec
->hdr
.proto
& mask
->hdr
.proto
;
748 next_proto_mask
= &mask
->hdr
.proto
;
751 if (proto
!= IPPROTO_ICMP
&& proto
!= IPPROTO_UDP
&&
752 proto
!= IPPROTO_SCTP
&& proto
!= IPPROTO_TCP
&&
753 (match
->wc
.masks
.tp_src
||
754 match
->wc
.masks
.tp_dst
||
755 match
->wc
.masks
.tcp_flags
)) {
756 VLOG_DBG("L4 Protocol (%u) not supported", proto
);
760 if (proto
== IPPROTO_TCP
) {
761 struct rte_flow_item_tcp
*spec
, *mask
;
763 spec
= xzalloc(sizeof *spec
);
764 mask
= xzalloc(sizeof *mask
);
766 spec
->hdr
.src_port
= match
->flow
.tp_src
;
767 spec
->hdr
.dst_port
= match
->flow
.tp_dst
;
768 spec
->hdr
.data_off
= ntohs(match
->flow
.tcp_flags
) >> 8;
769 spec
->hdr
.tcp_flags
= ntohs(match
->flow
.tcp_flags
) & 0xff;
771 mask
->hdr
.src_port
= match
->wc
.masks
.tp_src
;
772 mask
->hdr
.dst_port
= match
->wc
.masks
.tp_dst
;
773 mask
->hdr
.data_off
= ntohs(match
->wc
.masks
.tcp_flags
) >> 8;
774 mask
->hdr
.tcp_flags
= ntohs(match
->wc
.masks
.tcp_flags
) & 0xff;
776 consumed_masks
->tp_src
= 0;
777 consumed_masks
->tp_dst
= 0;
778 consumed_masks
->tcp_flags
= 0;
780 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_TCP
, spec
, mask
);
782 /* proto == TCP and ITEM_TYPE_TCP, thus no need for proto match. */
783 if (next_proto_mask
) {
784 *next_proto_mask
= 0;
786 } else if (proto
== IPPROTO_UDP
) {
787 struct rte_flow_item_udp
*spec
, *mask
;
789 spec
= xzalloc(sizeof *spec
);
790 mask
= xzalloc(sizeof *mask
);
792 spec
->hdr
.src_port
= match
->flow
.tp_src
;
793 spec
->hdr
.dst_port
= match
->flow
.tp_dst
;
795 mask
->hdr
.src_port
= match
->wc
.masks
.tp_src
;
796 mask
->hdr
.dst_port
= match
->wc
.masks
.tp_dst
;
798 consumed_masks
->tp_src
= 0;
799 consumed_masks
->tp_dst
= 0;
801 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_UDP
, spec
, mask
);
803 /* proto == UDP and ITEM_TYPE_UDP, thus no need for proto match. */
804 if (next_proto_mask
) {
805 *next_proto_mask
= 0;
807 } else if (proto
== IPPROTO_SCTP
) {
808 struct rte_flow_item_sctp
*spec
, *mask
;
810 spec
= xzalloc(sizeof *spec
);
811 mask
= xzalloc(sizeof *mask
);
813 spec
->hdr
.src_port
= match
->flow
.tp_src
;
814 spec
->hdr
.dst_port
= match
->flow
.tp_dst
;
816 mask
->hdr
.src_port
= match
->wc
.masks
.tp_src
;
817 mask
->hdr
.dst_port
= match
->wc
.masks
.tp_dst
;
819 consumed_masks
->tp_src
= 0;
820 consumed_masks
->tp_dst
= 0;
822 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_SCTP
, spec
, mask
);
824 /* proto == SCTP and ITEM_TYPE_SCTP, thus no need for proto match. */
825 if (next_proto_mask
) {
826 *next_proto_mask
= 0;
828 } else if (proto
== IPPROTO_ICMP
) {
829 struct rte_flow_item_icmp
*spec
, *mask
;
831 spec
= xzalloc(sizeof *spec
);
832 mask
= xzalloc(sizeof *mask
);
834 spec
->hdr
.icmp_type
= (uint8_t) ntohs(match
->flow
.tp_src
);
835 spec
->hdr
.icmp_code
= (uint8_t) ntohs(match
->flow
.tp_dst
);
837 mask
->hdr
.icmp_type
= (uint8_t) ntohs(match
->wc
.masks
.tp_src
);
838 mask
->hdr
.icmp_code
= (uint8_t) ntohs(match
->wc
.masks
.tp_dst
);
840 consumed_masks
->tp_src
= 0;
841 consumed_masks
->tp_dst
= 0;
843 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_ICMP
, spec
, mask
);
845 /* proto == ICMP and ITEM_TYPE_ICMP, thus no need for proto match. */
846 if (next_proto_mask
) {
847 *next_proto_mask
= 0;
851 add_flow_pattern(patterns
, RTE_FLOW_ITEM_TYPE_END
, NULL
, NULL
);
853 if (!is_all_zeros(consumed_masks
, sizeof *consumed_masks
)) {
860 add_flow_mark_rss_actions(struct flow_actions
*actions
,
862 const struct netdev
*netdev
)
864 struct rte_flow_action_mark
*mark
;
865 struct action_rss_data
{
866 struct rte_flow_action_rss conf
;
869 BUILD_ASSERT_DECL(offsetof(struct action_rss_data
, conf
) == 0);
872 mark
= xzalloc(sizeof *mark
);
874 mark
->id
= flow_mark
;
875 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_MARK
, mark
);
877 rss_data
= xmalloc(sizeof *rss_data
+
878 netdev_n_rxq(netdev
) * sizeof rss_data
->queue
[0]);
879 *rss_data
= (struct action_rss_data
) {
880 .conf
= (struct rte_flow_action_rss
) {
881 .func
= RTE_ETH_HASH_FUNCTION_DEFAULT
,
884 .queue_num
= netdev_n_rxq(netdev
),
885 .queue
= rss_data
->queue
,
891 /* Override queue array with default. */
892 for (i
= 0; i
< netdev_n_rxq(netdev
); i
++) {
893 rss_data
->queue
[i
] = i
;
896 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_RSS
, &rss_data
->conf
);
897 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_END
, NULL
);
900 static struct rte_flow
*
901 netdev_offload_dpdk_mark_rss(struct flow_patterns
*patterns
,
902 struct netdev
*netdev
,
905 struct flow_actions actions
= { .actions
= NULL
, .cnt
= 0 };
906 const struct rte_flow_attr flow_attr
= {
912 struct rte_flow_error error
;
913 struct rte_flow
*flow
;
915 add_flow_mark_rss_actions(&actions
, flow_mark
, netdev
);
917 flow
= netdev_offload_dpdk_flow_create(netdev
, &flow_attr
, patterns
->items
,
918 actions
.actions
, &error
);
920 free_flow_actions(&actions
);
925 add_count_action(struct flow_actions
*actions
)
927 struct rte_flow_action_count
*count
= xzalloc(sizeof *count
);
929 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_COUNT
, count
);
933 add_port_id_action(struct flow_actions
*actions
,
934 struct netdev
*outdev
)
936 struct rte_flow_action_port_id
*port_id
;
939 outdev_id
= netdev_dpdk_get_port_id(outdev
);
943 port_id
= xzalloc(sizeof *port_id
);
944 port_id
->id
= outdev_id
;
945 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_PORT_ID
, port_id
);
950 add_output_action(struct netdev
*netdev
,
951 struct flow_actions
*actions
,
952 const struct nlattr
*nla
)
954 struct netdev
*outdev
;
958 port
= nl_attr_get_odp_port(nla
);
959 outdev
= netdev_ports_get(port
, netdev
->dpif_type
);
960 if (outdev
== NULL
) {
961 VLOG_DBG_RL(&rl
, "Cannot find netdev for odp port %"PRIu32
, port
);
964 if (!netdev_flow_api_equals(netdev
, outdev
) ||
965 add_port_id_action(actions
, outdev
)) {
966 VLOG_DBG_RL(&rl
, "%s: Output to port \'%s\' cannot be offloaded.",
967 netdev_get_name(netdev
), netdev_get_name(outdev
));
970 netdev_close(outdev
);
975 add_set_flow_action__(struct flow_actions
*actions
,
976 const void *value
, void *mask
,
977 const size_t size
, const int attr
)
982 /* DPDK does not support partially masked set actions. In such
983 * case, fail the offload.
985 if (is_all_zeros(mask
, size
)) {
988 if (!is_all_ones(mask
, size
)) {
989 VLOG_DBG_RL(&rl
, "Partial mask is not supported");
994 spec
= xzalloc(size
);
995 memcpy(spec
, value
, size
);
996 add_flow_action(actions
, attr
, spec
);
998 /* Clear used mask for later checking. */
1000 memset(mask
, 0, size
);
1005 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_mac
) ==
1006 MEMBER_SIZEOF(struct ovs_key_ethernet
, eth_src
));
1007 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_mac
) ==
1008 MEMBER_SIZEOF(struct ovs_key_ethernet
, eth_dst
));
1009 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_ipv4
) ==
1010 MEMBER_SIZEOF(struct ovs_key_ipv4
, ipv4_src
));
1011 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_ipv4
) ==
1012 MEMBER_SIZEOF(struct ovs_key_ipv4
, ipv4_dst
));
1013 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_ttl
) ==
1014 MEMBER_SIZEOF(struct ovs_key_ipv4
, ipv4_ttl
));
1015 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_ipv6
) ==
1016 MEMBER_SIZEOF(struct ovs_key_ipv6
, ipv6_src
));
1017 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_ipv6
) ==
1018 MEMBER_SIZEOF(struct ovs_key_ipv6
, ipv6_dst
));
1019 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_ttl
) ==
1020 MEMBER_SIZEOF(struct ovs_key_ipv6
, ipv6_hlimit
));
1021 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_tp
) ==
1022 MEMBER_SIZEOF(struct ovs_key_tcp
, tcp_src
));
1023 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_tp
) ==
1024 MEMBER_SIZEOF(struct ovs_key_tcp
, tcp_dst
));
1025 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_tp
) ==
1026 MEMBER_SIZEOF(struct ovs_key_udp
, udp_src
));
1027 BUILD_ASSERT_DECL(sizeof(struct rte_flow_action_set_tp
) ==
1028 MEMBER_SIZEOF(struct ovs_key_udp
, udp_dst
));
1031 parse_set_actions(struct flow_actions
*actions
,
1032 const struct nlattr
*set_actions
,
1033 const size_t set_actions_len
,
1036 const struct nlattr
*sa
;
1039 #define add_set_flow_action(field, type) \
1040 if (add_set_flow_action__(actions, &key->field, \
1041 mask ? CONST_CAST(void *, &mask->field) : NULL, \
1042 sizeof key->field, type)) { \
1046 NL_ATTR_FOR_EACH_UNSAFE (sa
, sleft
, set_actions
, set_actions_len
) {
1047 if (nl_attr_type(sa
) == OVS_KEY_ATTR_ETHERNET
) {
1048 const struct ovs_key_ethernet
*key
= nl_attr_get(sa
);
1049 const struct ovs_key_ethernet
*mask
= masked
? key
+ 1 : NULL
;
1051 add_set_flow_action(eth_src
, RTE_FLOW_ACTION_TYPE_SET_MAC_SRC
);
1052 add_set_flow_action(eth_dst
, RTE_FLOW_ACTION_TYPE_SET_MAC_DST
);
1054 if (mask
&& !is_all_zeros(mask
, sizeof *mask
)) {
1055 VLOG_DBG_RL(&rl
, "Unsupported ETHERNET set action");
1058 } else if (nl_attr_type(sa
) == OVS_KEY_ATTR_IPV4
) {
1059 const struct ovs_key_ipv4
*key
= nl_attr_get(sa
);
1060 const struct ovs_key_ipv4
*mask
= masked
? key
+ 1 : NULL
;
1062 add_set_flow_action(ipv4_src
, RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC
);
1063 add_set_flow_action(ipv4_dst
, RTE_FLOW_ACTION_TYPE_SET_IPV4_DST
);
1064 add_set_flow_action(ipv4_ttl
, RTE_FLOW_ACTION_TYPE_SET_TTL
);
1066 if (mask
&& !is_all_zeros(mask
, sizeof *mask
)) {
1067 VLOG_DBG_RL(&rl
, "Unsupported IPv4 set action");
1070 } else if (nl_attr_type(sa
) == OVS_KEY_ATTR_IPV6
) {
1071 const struct ovs_key_ipv6
*key
= nl_attr_get(sa
);
1072 const struct ovs_key_ipv6
*mask
= masked
? key
+ 1 : NULL
;
1074 add_set_flow_action(ipv6_src
, RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC
);
1075 add_set_flow_action(ipv6_dst
, RTE_FLOW_ACTION_TYPE_SET_IPV6_DST
);
1076 add_set_flow_action(ipv6_hlimit
, RTE_FLOW_ACTION_TYPE_SET_TTL
);
1078 if (mask
&& !is_all_zeros(mask
, sizeof *mask
)) {
1079 VLOG_DBG_RL(&rl
, "Unsupported IPv6 set action");
1082 } else if (nl_attr_type(sa
) == OVS_KEY_ATTR_TCP
) {
1083 const struct ovs_key_tcp
*key
= nl_attr_get(sa
);
1084 const struct ovs_key_tcp
*mask
= masked
? key
+ 1 : NULL
;
1086 add_set_flow_action(tcp_src
, RTE_FLOW_ACTION_TYPE_SET_TP_SRC
);
1087 add_set_flow_action(tcp_dst
, RTE_FLOW_ACTION_TYPE_SET_TP_DST
);
1089 if (mask
&& !is_all_zeros(mask
, sizeof *mask
)) {
1090 VLOG_DBG_RL(&rl
, "Unsupported TCP set action");
1093 } else if (nl_attr_type(sa
) == OVS_KEY_ATTR_UDP
) {
1094 const struct ovs_key_udp
*key
= nl_attr_get(sa
);
1095 const struct ovs_key_udp
*mask
= masked
? key
+ 1 : NULL
;
1097 add_set_flow_action(udp_src
, RTE_FLOW_ACTION_TYPE_SET_TP_SRC
);
1098 add_set_flow_action(udp_dst
, RTE_FLOW_ACTION_TYPE_SET_TP_DST
);
1100 if (mask
&& !is_all_zeros(mask
, sizeof *mask
)) {
1101 VLOG_DBG_RL(&rl
, "Unsupported UDP set action");
1106 "Unsupported set action type %d", nl_attr_type(sa
));
1115 parse_vlan_push_action(struct flow_actions
*actions
,
1116 const struct ovs_action_push_vlan
*vlan_push
)
1118 struct rte_flow_action_of_push_vlan
*rte_push_vlan
;
1119 struct rte_flow_action_of_set_vlan_pcp
*rte_vlan_pcp
;
1120 struct rte_flow_action_of_set_vlan_vid
*rte_vlan_vid
;
1122 rte_push_vlan
= xzalloc(sizeof *rte_push_vlan
);
1123 rte_push_vlan
->ethertype
= vlan_push
->vlan_tpid
;
1124 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN
, rte_push_vlan
);
1126 rte_vlan_pcp
= xzalloc(sizeof *rte_vlan_pcp
);
1127 rte_vlan_pcp
->vlan_pcp
= vlan_tci_to_pcp(vlan_push
->vlan_tci
);
1128 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP
,
1131 rte_vlan_vid
= xzalloc(sizeof *rte_vlan_vid
);
1132 rte_vlan_vid
->vlan_vid
= htons(vlan_tci_to_vid(vlan_push
->vlan_tci
));
1133 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID
,
1139 parse_clone_actions(struct netdev
*netdev
,
1140 struct flow_actions
*actions
,
1141 const struct nlattr
*clone_actions
,
1142 const size_t clone_actions_len
)
1144 const struct nlattr
*ca
;
1147 NL_ATTR_FOR_EACH_UNSAFE (ca
, cleft
, clone_actions
, clone_actions_len
) {
1148 int clone_type
= nl_attr_type(ca
);
1150 if (clone_type
== OVS_ACTION_ATTR_TUNNEL_PUSH
) {
1151 const struct ovs_action_push_tnl
*tnl_push
= nl_attr_get(ca
);
1152 struct rte_flow_action_raw_encap
*raw_encap
=
1153 xzalloc(sizeof *raw_encap
);
1155 raw_encap
->data
= (uint8_t *) tnl_push
->header
;
1156 raw_encap
->preserve
= NULL
;
1157 raw_encap
->size
= tnl_push
->header_len
;
1159 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_RAW_ENCAP
,
1161 } else if (clone_type
== OVS_ACTION_ATTR_OUTPUT
) {
1162 if (add_output_action(netdev
, actions
, ca
)) {
1167 "Unsupported nested action inside clone(), "
1168 "action type: %d", clone_type
);
1176 parse_flow_actions(struct netdev
*netdev
,
1177 struct flow_actions
*actions
,
1178 struct nlattr
*nl_actions
,
1179 size_t nl_actions_len
)
1184 add_count_action(actions
);
1185 NL_ATTR_FOR_EACH_UNSAFE (nla
, left
, nl_actions
, nl_actions_len
) {
1186 if (nl_attr_type(nla
) == OVS_ACTION_ATTR_OUTPUT
) {
1187 if (add_output_action(netdev
, actions
, nla
)) {
1190 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_DROP
) {
1191 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_DROP
, NULL
);
1192 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_SET
||
1193 nl_attr_type(nla
) == OVS_ACTION_ATTR_SET_MASKED
) {
1194 const struct nlattr
*set_actions
= nl_attr_get(nla
);
1195 const size_t set_actions_len
= nl_attr_get_size(nla
);
1196 bool masked
= nl_attr_type(nla
) == OVS_ACTION_ATTR_SET_MASKED
;
1198 if (parse_set_actions(actions
, set_actions
, set_actions_len
,
1202 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_PUSH_VLAN
) {
1203 const struct ovs_action_push_vlan
*vlan
= nl_attr_get(nla
);
1205 if (parse_vlan_push_action(actions
, vlan
)) {
1208 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_POP_VLAN
) {
1209 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_OF_POP_VLAN
, NULL
);
1210 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_CLONE
&&
1211 left
<= NLA_ALIGN(nla
->nla_len
)) {
1212 const struct nlattr
*clone_actions
= nl_attr_get(nla
);
1213 size_t clone_actions_len
= nl_attr_get_size(nla
);
1215 if (parse_clone_actions(netdev
, actions
, clone_actions
,
1216 clone_actions_len
)) {
1220 VLOG_DBG_RL(&rl
, "Unsupported action type %d", nl_attr_type(nla
));
1225 if (nl_actions_len
== 0) {
1226 VLOG_DBG_RL(&rl
, "No actions provided");
1230 add_flow_action(actions
, RTE_FLOW_ACTION_TYPE_END
, NULL
);
1234 static struct rte_flow
*
1235 netdev_offload_dpdk_actions(struct netdev
*netdev
,
1236 struct flow_patterns
*patterns
,
1237 struct nlattr
*nl_actions
,
1240 const struct rte_flow_attr flow_attr
= { .ingress
= 1, .transfer
= 1 };
1241 struct flow_actions actions
= { .actions
= NULL
, .cnt
= 0 };
1242 struct rte_flow
*flow
= NULL
;
1243 struct rte_flow_error error
;
1246 ret
= parse_flow_actions(netdev
, &actions
, nl_actions
, actions_len
);
1250 flow
= netdev_offload_dpdk_flow_create(netdev
, &flow_attr
, patterns
->items
,
1251 actions
.actions
, &error
);
1253 free_flow_actions(&actions
);
1258 netdev_offload_dpdk_add_flow(struct netdev
*netdev
,
1259 struct match
*match
,
1260 struct nlattr
*nl_actions
,
1262 const ovs_u128
*ufid
,
1263 struct offload_info
*info
)
1265 struct flow_patterns patterns
= { .items
= NULL
, .cnt
= 0 };
1266 bool actions_offloaded
= true;
1267 struct rte_flow
*flow
;
1270 ret
= parse_flow_match(&patterns
, match
);
1272 VLOG_DBG_RL(&rl
, "%s: matches of ufid "UUID_FMT
" are not supported",
1273 netdev_get_name(netdev
), UUID_ARGS((struct uuid
*) ufid
));
1277 flow
= netdev_offload_dpdk_actions(netdev
, &patterns
, nl_actions
,
1280 /* If we failed to offload the rule actions fallback to MARK+RSS
1283 flow
= netdev_offload_dpdk_mark_rss(&patterns
, netdev
,
1285 actions_offloaded
= false;
1292 ufid_to_rte_flow_associate(ufid
, flow
, actions_offloaded
);
1293 VLOG_DBG("%s: installed flow %p by ufid "UUID_FMT
,
1294 netdev_get_name(netdev
), flow
, UUID_ARGS((struct uuid
*)ufid
));
1297 free_flow_patterns(&patterns
);
1302 netdev_offload_dpdk_destroy_flow(struct netdev
*netdev
,
1303 const ovs_u128
*ufid
,
1304 struct rte_flow
*rte_flow
)
1306 struct rte_flow_error error
;
1307 int ret
= netdev_dpdk_rte_flow_destroy(netdev
, rte_flow
, &error
);
1310 ufid_to_rte_flow_disassociate(ufid
);
1311 VLOG_DBG_RL(&rl
, "%s: rte_flow 0x%"PRIxPTR
1312 " flow destroy %d ufid " UUID_FMT
,
1313 netdev_get_name(netdev
), (intptr_t) rte_flow
,
1314 netdev_dpdk_get_port_id(netdev
),
1315 UUID_ARGS((struct uuid
*) ufid
));
1317 VLOG_ERR("Failed flow: %s: flow destroy %d ufid " UUID_FMT
,
1318 netdev_get_name(netdev
), netdev_dpdk_get_port_id(netdev
),
1319 UUID_ARGS((struct uuid
*) ufid
));
1326 netdev_offload_dpdk_flow_put(struct netdev
*netdev
, struct match
*match
,
1327 struct nlattr
*actions
, size_t actions_len
,
1328 const ovs_u128
*ufid
, struct offload_info
*info
,
1329 struct dpif_flow_stats
*stats
)
1331 struct ufid_to_rte_flow_data
*rte_flow_data
;
1335 * If an old rte_flow exists, it means it's a flow modification.
1336 * Here destroy the old rte flow first before adding a new one.
1338 rte_flow_data
= ufid_to_rte_flow_data_find(ufid
);
1339 if (rte_flow_data
&& rte_flow_data
->rte_flow
) {
1340 ret
= netdev_offload_dpdk_destroy_flow(netdev
, ufid
,
1341 rte_flow_data
->rte_flow
);
1348 memset(stats
, 0, sizeof *stats
);
1350 return netdev_offload_dpdk_add_flow(netdev
, match
, actions
,
1351 actions_len
, ufid
, info
);
1355 netdev_offload_dpdk_flow_del(struct netdev
*netdev
, const ovs_u128
*ufid
,
1356 struct dpif_flow_stats
*stats
)
1358 struct ufid_to_rte_flow_data
*rte_flow_data
;
1360 rte_flow_data
= ufid_to_rte_flow_data_find(ufid
);
1361 if (!rte_flow_data
|| !rte_flow_data
->rte_flow
) {
1366 memset(stats
, 0, sizeof *stats
);
1368 return netdev_offload_dpdk_destroy_flow(netdev
, ufid
,
1369 rte_flow_data
->rte_flow
);
1373 netdev_offload_dpdk_init_flow_api(struct netdev
*netdev
)
1375 return netdev_dpdk_flow_api_supported(netdev
) ? 0 : EOPNOTSUPP
;
1379 netdev_offload_dpdk_flow_get(struct netdev
*netdev
,
1380 struct match
*match OVS_UNUSED
,
1381 struct nlattr
**actions OVS_UNUSED
,
1382 const ovs_u128
*ufid
,
1383 struct dpif_flow_stats
*stats
,
1384 struct dpif_flow_attrs
*attrs
,
1385 struct ofpbuf
*buf OVS_UNUSED
)
1387 struct rte_flow_query_count query
= { .reset
= 1 };
1388 struct ufid_to_rte_flow_data
*rte_flow_data
;
1389 struct rte_flow_error error
;
1392 rte_flow_data
= ufid_to_rte_flow_data_find(ufid
);
1393 if (!rte_flow_data
|| !rte_flow_data
->rte_flow
) {
1398 attrs
->offloaded
= true;
1399 if (!rte_flow_data
->actions_offloaded
) {
1400 attrs
->dp_layer
= "ovs";
1401 memset(stats
, 0, sizeof *stats
);
1404 attrs
->dp_layer
= "dpdk";
1405 ret
= netdev_dpdk_rte_flow_query_count(netdev
, rte_flow_data
->rte_flow
,
1408 VLOG_DBG_RL(&rl
, "%s: Failed to query ufid "UUID_FMT
" flow: %p",
1409 netdev_get_name(netdev
), UUID_ARGS((struct uuid
*) ufid
),
1410 rte_flow_data
->rte_flow
);
1413 rte_flow_data
->stats
.n_packets
+= (query
.hits_set
) ? query
.hits
: 0;
1414 rte_flow_data
->stats
.n_bytes
+= (query
.bytes_set
) ? query
.bytes
: 0;
1415 if (query
.hits_set
&& query
.hits
) {
1416 rte_flow_data
->stats
.used
= time_msec();
1418 memcpy(stats
, &rte_flow_data
->stats
, sizeof *stats
);
1420 attrs
->dp_extra_info
= NULL
;
1424 const struct netdev_flow_api netdev_offload_dpdk
= {
1425 .type
= "dpdk_flow_api",
1426 .flow_put
= netdev_offload_dpdk_flow_put
,
1427 .flow_del
= netdev_offload_dpdk_flow_del
,
1428 .init_flow_api
= netdev_offload_dpdk_init_flow_api
,
1429 .flow_get
= netdev_offload_dpdk_flow_get
,