2 * Copyright (c) 2016 Mellanox Technologies, Ltd.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 #include "netdev-tc-offloads.h"
21 #include <linux/if_ether.h>
25 #include "openvswitch/hmap.h"
26 #include "openvswitch/match.h"
27 #include "openvswitch/ofpbuf.h"
28 #include "openvswitch/thread.h"
29 #include "openvswitch/types.h"
30 #include "openvswitch/util.h"
31 #include "openvswitch/vlog.h"
32 #include "netdev-linux.h"
34 #include "netlink-socket.h"
35 #include "odp-netlink.h"
38 #include "unaligned.h"
41 VLOG_DEFINE_THIS_MODULE(netdev_tc_offloads
);
43 static struct vlog_rate_limit error_rl
= VLOG_RATE_LIMIT_INIT(60, 5);
45 static struct hmap ufid_tc
= HMAP_INITIALIZER(&ufid_tc
);
47 struct netlink_field
{
53 static struct netlink_field set_flower_map
[][3] = {
54 [OVS_KEY_ATTR_IPV4
] = {
55 { offsetof(struct ovs_key_ipv4
, ipv4_src
),
56 offsetof(struct tc_flower_key
, ipv4
.ipv4_src
),
57 MEMBER_SIZEOF(struct tc_flower_key
, ipv4
.ipv4_src
)
59 { offsetof(struct ovs_key_ipv4
, ipv4_dst
),
60 offsetof(struct tc_flower_key
, ipv4
.ipv4_dst
),
61 MEMBER_SIZEOF(struct tc_flower_key
, ipv4
.ipv4_dst
)
63 { offsetof(struct ovs_key_ipv4
, ipv4_ttl
),
64 offsetof(struct tc_flower_key
, ipv4
.rewrite_ttl
),
65 MEMBER_SIZEOF(struct tc_flower_key
, ipv4
.rewrite_ttl
)
68 [OVS_KEY_ATTR_IPV6
] = {
69 { offsetof(struct ovs_key_ipv6
, ipv6_src
),
70 offsetof(struct tc_flower_key
, ipv6
.ipv6_src
),
71 MEMBER_SIZEOF(struct tc_flower_key
, ipv6
.ipv6_src
)
73 { offsetof(struct ovs_key_ipv6
, ipv6_dst
),
74 offsetof(struct tc_flower_key
, ipv6
.ipv6_dst
),
75 MEMBER_SIZEOF(struct tc_flower_key
, ipv6
.ipv6_dst
)
78 [OVS_KEY_ATTR_ETHERNET
] = {
79 { offsetof(struct ovs_key_ethernet
, eth_src
),
80 offsetof(struct tc_flower_key
, src_mac
),
81 MEMBER_SIZEOF(struct tc_flower_key
, src_mac
)
83 { offsetof(struct ovs_key_ethernet
, eth_dst
),
84 offsetof(struct tc_flower_key
, dst_mac
),
85 MEMBER_SIZEOF(struct tc_flower_key
, dst_mac
)
88 [OVS_KEY_ATTR_ETHERTYPE
] = {
90 offsetof(struct tc_flower_key
, eth_type
),
91 MEMBER_SIZEOF(struct tc_flower_key
, eth_type
)
94 [OVS_KEY_ATTR_TCP
] = {
95 { offsetof(struct ovs_key_tcp
, tcp_src
),
96 offsetof(struct tc_flower_key
, tcp_src
),
97 MEMBER_SIZEOF(struct tc_flower_key
, tcp_src
)
99 { offsetof(struct ovs_key_tcp
, tcp_dst
),
100 offsetof(struct tc_flower_key
, tcp_dst
),
101 MEMBER_SIZEOF(struct tc_flower_key
, tcp_dst
)
104 [OVS_KEY_ATTR_UDP
] = {
105 { offsetof(struct ovs_key_udp
, udp_src
),
106 offsetof(struct tc_flower_key
, udp_src
),
107 MEMBER_SIZEOF(struct tc_flower_key
, udp_src
)
109 { offsetof(struct ovs_key_udp
, udp_dst
),
110 offsetof(struct tc_flower_key
, udp_dst
),
111 MEMBER_SIZEOF(struct tc_flower_key
, udp_dst
)
116 static struct ovs_mutex ufid_lock
= OVS_MUTEX_INITIALIZER
;
119 * struct ufid_tc_data - data entry for ufid_tc hmap.
120 * @ufid_node: Element in @ufid_tc hash table by ufid key.
121 * @tc_node: Element in @ufid_tc hash table by prio/handle/ifindex key.
122 * @ufid: ufid assigned to the flow
125 * @ifindex: netdev ifindex.
126 * @netdev: netdev associated with the tc rule
128 struct ufid_tc_data
{
129 struct hmap_node ufid_node
;
130 struct hmap_node tc_node
;
135 struct netdev
*netdev
;
138 /* Remove matching ufid entry from ufid_tc hashmap. */
140 del_ufid_tc_mapping(const ovs_u128
*ufid
)
142 size_t ufid_hash
= hash_bytes(ufid
, sizeof *ufid
, 0);
143 struct ufid_tc_data
*data
;
145 ovs_mutex_lock(&ufid_lock
);
146 HMAP_FOR_EACH_WITH_HASH(data
, ufid_node
, ufid_hash
, &ufid_tc
) {
147 if (ovs_u128_equals(*ufid
, data
->ufid
)) {
153 ovs_mutex_unlock(&ufid_lock
);
157 hmap_remove(&ufid_tc
, &data
->ufid_node
);
158 hmap_remove(&ufid_tc
, &data
->tc_node
);
159 netdev_close(data
->netdev
);
161 ovs_mutex_unlock(&ufid_lock
);
164 /* Add ufid entry to ufid_tc hashmap.
165 * If entry exists already it will be replaced. */
167 add_ufid_tc_mapping(const ovs_u128
*ufid
, int prio
, int handle
,
168 struct netdev
*netdev
, int ifindex
)
170 size_t ufid_hash
= hash_bytes(ufid
, sizeof *ufid
, 0);
171 size_t tc_hash
= hash_int(hash_int(prio
, handle
), ifindex
);
172 struct ufid_tc_data
*new_data
= xzalloc(sizeof *new_data
);
174 del_ufid_tc_mapping(ufid
);
176 new_data
->ufid
= *ufid
;
177 new_data
->prio
= prio
;
178 new_data
->handle
= handle
;
179 new_data
->netdev
= netdev_ref(netdev
);
180 new_data
->ifindex
= ifindex
;
182 ovs_mutex_lock(&ufid_lock
);
183 hmap_insert(&ufid_tc
, &new_data
->ufid_node
, ufid_hash
);
184 hmap_insert(&ufid_tc
, &new_data
->tc_node
, tc_hash
);
185 ovs_mutex_unlock(&ufid_lock
);
188 /* Get ufid from ufid_tc hashmap.
190 * If netdev output param is not NULL then the function will return
191 * associated netdev on success and a refcount is taken on that netdev.
192 * The caller is then responsible to close the netdev.
194 * Returns handle if successful and fill prio and netdev for that ufid.
195 * Otherwise returns 0.
198 get_ufid_tc_mapping(const ovs_u128
*ufid
, int *prio
, struct netdev
**netdev
)
200 size_t ufid_hash
= hash_bytes(ufid
, sizeof *ufid
, 0);
201 struct ufid_tc_data
*data
;
204 ovs_mutex_lock(&ufid_lock
);
205 HMAP_FOR_EACH_WITH_HASH(data
, ufid_node
, ufid_hash
, &ufid_tc
) {
206 if (ovs_u128_equals(*ufid
, data
->ufid
)) {
211 *netdev
= netdev_ref(data
->netdev
);
213 handle
= data
->handle
;
217 ovs_mutex_unlock(&ufid_lock
);
222 /* Find ufid entry in ufid_tc hashmap using prio, handle and netdev.
223 * The result is saved in ufid.
225 * Returns true on success.
228 find_ufid(int prio
, int handle
, struct netdev
*netdev
, ovs_u128
*ufid
)
230 int ifindex
= netdev_get_ifindex(netdev
);
231 struct ufid_tc_data
*data
;
232 size_t tc_hash
= hash_int(hash_int(prio
, handle
), ifindex
);
234 ovs_mutex_lock(&ufid_lock
);
235 HMAP_FOR_EACH_WITH_HASH(data
, tc_node
, tc_hash
, &ufid_tc
) {
236 if (data
->prio
== prio
&& data
->handle
== handle
237 && data
->ifindex
== ifindex
) {
242 ovs_mutex_unlock(&ufid_lock
);
244 return (data
!= NULL
);
247 struct prio_map_data
{
248 struct hmap_node node
;
249 struct tc_flower_key mask
;
254 /* Get free prio for tc flower
255 * If prio is already allocated for mask/eth_type combination then return it.
256 * If not assign new prio.
258 * Return prio on success or 0 if we are out of prios.
261 get_prio_for_tc_flower(struct tc_flower
*flower
)
263 static struct hmap prios
= HMAP_INITIALIZER(&prios
);
264 static struct ovs_mutex prios_lock
= OVS_MUTEX_INITIALIZER
;
265 static uint16_t last_prio
= 0;
266 size_t key_len
= sizeof(struct tc_flower_key
);
267 size_t hash
= hash_bytes(&flower
->mask
, key_len
,
268 (OVS_FORCE
uint32_t) flower
->key
.eth_type
);
269 struct prio_map_data
*data
;
270 struct prio_map_data
*new_data
;
272 /* We can use the same prio for same mask/eth combination but must have
273 * different prio if not. Flower classifier will reject same prio for
274 * different mask/eth combination. */
275 ovs_mutex_lock(&prios_lock
);
276 HMAP_FOR_EACH_WITH_HASH(data
, node
, hash
, &prios
) {
277 if (!memcmp(&flower
->mask
, &data
->mask
, key_len
)
278 && data
->protocol
== flower
->key
.eth_type
) {
279 ovs_mutex_unlock(&prios_lock
);
284 if (last_prio
== UINT16_MAX
) {
285 /* last_prio can overflow if there will be many different kinds of
286 * flows which shouldn't happen organically. */
287 ovs_mutex_unlock(&prios_lock
);
291 new_data
= xzalloc(sizeof *new_data
);
292 memcpy(&new_data
->mask
, &flower
->mask
, key_len
);
293 new_data
->prio
= ++last_prio
;
294 new_data
->protocol
= flower
->key
.eth_type
;
295 hmap_insert(&prios
, &new_data
->node
, hash
);
296 ovs_mutex_unlock(&prios_lock
);
298 return new_data
->prio
;
302 netdev_tc_flow_flush(struct netdev
*netdev
)
304 int ifindex
= netdev_get_ifindex(netdev
);
307 VLOG_ERR_RL(&error_rl
, "flow_flush: failed to get ifindex for %s: %s",
308 netdev_get_name(netdev
), ovs_strerror(-ifindex
));
312 return tc_flush(ifindex
);
316 netdev_tc_flow_dump_create(struct netdev
*netdev
,
317 struct netdev_flow_dump
**dump_out
)
319 struct netdev_flow_dump
*dump
;
322 ifindex
= netdev_get_ifindex(netdev
);
324 VLOG_ERR_RL(&error_rl
, "dump_create: failed to get ifindex for %s: %s",
325 netdev_get_name(netdev
), ovs_strerror(-ifindex
));
329 dump
= xzalloc(sizeof *dump
);
330 dump
->nl_dump
= xzalloc(sizeof *dump
->nl_dump
);
331 dump
->netdev
= netdev_ref(netdev
);
332 tc_dump_flower_start(ifindex
, dump
->nl_dump
);
340 netdev_tc_flow_dump_destroy(struct netdev_flow_dump
*dump
)
342 nl_dump_done(dump
->nl_dump
);
343 netdev_close(dump
->netdev
);
350 parse_flower_rewrite_to_netlink_action(struct ofpbuf
*buf
,
351 struct tc_flower
*flower
)
353 char *mask
= (char *) &flower
->rewrite
.mask
;
354 char *data
= (char *) &flower
->rewrite
.key
;
356 for (int type
= 0; type
< ARRAY_SIZE(set_flower_map
); type
++) {
359 int len
= ovs_flow_key_attr_lens
[type
].len
;
365 for (int j
= 0; j
< ARRAY_SIZE(set_flower_map
[type
]); j
++) {
366 struct netlink_field
*f
= &set_flower_map
[type
][j
];
372 if (!is_all_zeros(mask
+ f
->flower_offset
, f
->size
)) {
374 nested
= nl_msg_start_nested(buf
,
375 OVS_ACTION_ATTR_SET_MASKED
);
376 put
= nl_msg_put_unspec_zero(buf
, type
, len
* 2);
379 memcpy(put
+ f
->offset
, data
+ f
->flower_offset
, f
->size
);
380 memcpy(put
+ len
+ f
->offset
,
381 mask
+ f
->flower_offset
, f
->size
);
386 nl_msg_end_nested(buf
, nested
);
392 parse_tc_flower_to_match(struct tc_flower
*flower
,
394 struct nlattr
**actions
,
395 struct dpif_flow_stats
*stats
,
396 struct ofpbuf
*buf
) {
398 struct tc_flower_key
*key
= &flower
->key
;
399 struct tc_flower_key
*mask
= &flower
->mask
;
400 odp_port_t outport
= 0;
402 if (flower
->ifindex_out
) {
403 outport
= netdev_ifindex_to_odp_port(flower
->ifindex_out
);
411 match_init_catchall(match
);
412 match_set_dl_src_masked(match
, key
->src_mac
, mask
->src_mac
);
413 match_set_dl_dst_masked(match
, key
->dst_mac
, mask
->dst_mac
);
415 if (key
->eth_type
== htons(ETH_TYPE_VLAN
)) {
416 match_set_dl_vlan(match
, htons(key
->vlan_id
));
417 match_set_dl_vlan_pcp(match
, key
->vlan_prio
);
418 match_set_dl_type(match
, key
->encap_eth_type
);
419 flow_fix_vlan_tpid(&match
->flow
);
421 match_set_dl_type(match
, key
->eth_type
);
424 if (is_ip_any(&match
->flow
)) {
426 match_set_nw_proto(match
, key
->ip_proto
);
429 match_set_nw_ttl_masked(match
, key
->ip_ttl
, mask
->ip_ttl
);
433 uint8_t flags_mask
= 0;
435 if (mask
->flags
& TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT
) {
436 if (key
->flags
& TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT
) {
437 flags
|= FLOW_NW_FRAG_ANY
;
439 flags_mask
|= FLOW_NW_FRAG_ANY
;
442 if (mask
->flags
& TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST
) {
443 if (!(key
->flags
& TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST
)) {
444 flags
|= FLOW_NW_FRAG_LATER
;
446 flags_mask
|= FLOW_NW_FRAG_LATER
;
449 match_set_nw_frag_masked(match
, flags
, flags_mask
);
452 match_set_nw_src_masked(match
, key
->ipv4
.ipv4_src
, mask
->ipv4
.ipv4_src
);
453 match_set_nw_dst_masked(match
, key
->ipv4
.ipv4_dst
, mask
->ipv4
.ipv4_dst
);
455 match_set_ipv6_src_masked(match
,
456 &key
->ipv6
.ipv6_src
, &mask
->ipv6
.ipv6_src
);
457 match_set_ipv6_dst_masked(match
,
458 &key
->ipv6
.ipv6_dst
, &mask
->ipv6
.ipv6_dst
);
460 if (key
->ip_proto
== IPPROTO_TCP
) {
461 match_set_tp_dst_masked(match
, key
->tcp_dst
, mask
->tcp_dst
);
462 match_set_tp_src_masked(match
, key
->tcp_src
, mask
->tcp_src
);
463 match_set_tcp_flags_masked(match
, key
->tcp_flags
, mask
->tcp_flags
);
464 } else if (key
->ip_proto
== IPPROTO_UDP
) {
465 match_set_tp_dst_masked(match
, key
->udp_dst
, mask
->udp_dst
);
466 match_set_tp_src_masked(match
, key
->udp_src
, mask
->udp_src
);
467 } else if (key
->ip_proto
== IPPROTO_SCTP
) {
468 match_set_tp_dst_masked(match
, key
->sctp_dst
, mask
->sctp_dst
);
469 match_set_tp_src_masked(match
, key
->sctp_src
, mask
->sctp_src
);
473 if (flower
->tunnel
.tunnel
) {
474 match_set_tun_id(match
, flower
->tunnel
.id
);
475 if (flower
->tunnel
.ipv4
.ipv4_dst
) {
476 match_set_tun_src(match
, flower
->tunnel
.ipv4
.ipv4_src
);
477 match_set_tun_dst(match
, flower
->tunnel
.ipv4
.ipv4_dst
);
478 } else if (!is_all_zeros(&flower
->tunnel
.ipv6
.ipv6_dst
,
479 sizeof flower
->tunnel
.ipv6
.ipv6_dst
)) {
480 match_set_tun_ipv6_src(match
, &flower
->tunnel
.ipv6
.ipv6_src
);
481 match_set_tun_ipv6_dst(match
, &flower
->tunnel
.ipv6
.ipv6_dst
);
483 if (flower
->tunnel
.tp_dst
) {
484 match_set_tun_tp_dst(match
, flower
->tunnel
.tp_dst
);
488 act_off
= nl_msg_start_nested(buf
, OVS_FLOW_ATTR_ACTIONS
);
490 if (flower
->vlan_pop
) {
491 nl_msg_put_flag(buf
, OVS_ACTION_ATTR_POP_VLAN
);
494 if (flower
->vlan_push_id
|| flower
->vlan_push_prio
) {
495 struct ovs_action_push_vlan
*push
;
496 push
= nl_msg_put_unspec_zero(buf
, OVS_ACTION_ATTR_PUSH_VLAN
,
499 push
->vlan_tpid
= htons(ETH_TYPE_VLAN
);
500 push
->vlan_tci
= htons(flower
->vlan_push_id
501 | (flower
->vlan_push_prio
<< 13)
505 if (flower
->rewrite
.rewrite
) {
506 parse_flower_rewrite_to_netlink_action(buf
, flower
);
509 if (flower
->set
.set
) {
510 size_t set_offset
= nl_msg_start_nested(buf
, OVS_ACTION_ATTR_SET
);
511 size_t tunnel_offset
=
512 nl_msg_start_nested(buf
, OVS_KEY_ATTR_TUNNEL
);
514 nl_msg_put_be64(buf
, OVS_TUNNEL_KEY_ATTR_ID
, flower
->set
.id
);
515 if (flower
->set
.ipv4
.ipv4_src
) {
516 nl_msg_put_be32(buf
, OVS_TUNNEL_KEY_ATTR_IPV4_SRC
,
517 flower
->set
.ipv4
.ipv4_src
);
519 if (flower
->set
.ipv4
.ipv4_dst
) {
520 nl_msg_put_be32(buf
, OVS_TUNNEL_KEY_ATTR_IPV4_DST
,
521 flower
->set
.ipv4
.ipv4_dst
);
523 if (!is_all_zeros(&flower
->set
.ipv6
.ipv6_src
,
524 sizeof flower
->set
.ipv6
.ipv6_src
)) {
525 nl_msg_put_in6_addr(buf
, OVS_TUNNEL_KEY_ATTR_IPV6_SRC
,
526 &flower
->set
.ipv6
.ipv6_src
);
528 if (!is_all_zeros(&flower
->set
.ipv6
.ipv6_dst
,
529 sizeof flower
->set
.ipv6
.ipv6_dst
)) {
530 nl_msg_put_in6_addr(buf
, OVS_TUNNEL_KEY_ATTR_IPV6_DST
,
531 &flower
->set
.ipv6
.ipv6_dst
);
533 nl_msg_put_be16(buf
, OVS_TUNNEL_KEY_ATTR_TP_DST
,
536 nl_msg_end_nested(buf
, tunnel_offset
);
537 nl_msg_end_nested(buf
, set_offset
);
540 if (flower
->ifindex_out
> 0) {
541 nl_msg_put_u32(buf
, OVS_ACTION_ATTR_OUTPUT
, odp_to_u32(outport
));
545 nl_msg_end_nested(buf
, act_off
);
547 *actions
= ofpbuf_at_assert(buf
, act_off
, sizeof(struct nlattr
));
550 memset(stats
, 0, sizeof *stats
);
551 stats
->n_packets
= get_32aligned_u64(&flower
->stats
.n_packets
);
552 stats
->n_bytes
= get_32aligned_u64(&flower
->stats
.n_bytes
);
553 stats
->used
= flower
->lastused
;
560 netdev_tc_flow_dump_next(struct netdev_flow_dump
*dump
,
562 struct nlattr
**actions
,
563 struct dpif_flow_stats
*stats
,
565 struct ofpbuf
*rbuffer
,
566 struct ofpbuf
*wbuffer
)
568 struct ofpbuf nl_flow
;
570 while (nl_dump_next(dump
->nl_dump
, &nl_flow
, rbuffer
)) {
571 struct tc_flower flower
;
572 struct netdev
*netdev
= dump
->netdev
;
574 if (parse_netlink_to_tc_flower(&nl_flow
, &flower
)) {
578 if (parse_tc_flower_to_match(&flower
, match
, actions
, stats
,
583 if (flower
.act_cookie
.len
) {
584 *ufid
= *((ovs_u128
*) flower
.act_cookie
.data
);
585 } else if (!find_ufid(flower
.prio
, flower
.handle
, netdev
, ufid
)) {
589 match
->wc
.masks
.in_port
.odp_port
= u32_to_odp(UINT32_MAX
);
590 match
->flow
.in_port
.odp_port
= dump
->port
;
599 parse_put_flow_set_masked_action(struct tc_flower
*flower
,
600 const struct nlattr
*set
,
604 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
605 uint64_t set_stub
[1024 / 8];
606 struct ofpbuf set_buf
= OFPBUF_STUB_INITIALIZER(set_stub
);
607 char *set_data
, *set_mask
;
608 char *key
= (char *) &flower
->rewrite
.key
;
609 char *mask
= (char *) &flower
->rewrite
.mask
;
610 const struct nlattr
*attr
;
614 /* copy so we can set attr mask to 0 for used ovs key struct members */
615 attr
= ofpbuf_put(&set_buf
, set
, set_len
);
617 type
= nl_attr_type(attr
);
618 size
= nl_attr_get_size(attr
) / 2;
619 set_data
= CONST_CAST(char *, nl_attr_get(attr
));
620 set_mask
= set_data
+ size
;
622 if (type
>= ARRAY_SIZE(set_flower_map
)
623 || !set_flower_map
[type
][0].size
) {
624 VLOG_DBG_RL(&rl
, "unsupported set action type: %d", type
);
625 ofpbuf_uninit(&set_buf
);
629 for (i
= 0; i
< ARRAY_SIZE(set_flower_map
[type
]); i
++) {
630 struct netlink_field
*f
= &set_flower_map
[type
][i
];
636 /* copy masked value */
637 for (j
= 0; j
< f
->size
; j
++) {
638 char maskval
= hasmask
? set_mask
[f
->offset
+ j
] : 0xFF;
640 key
[f
->flower_offset
+ j
] = maskval
& set_data
[f
->offset
+ j
];
641 mask
[f
->flower_offset
+ j
] = maskval
;
645 /* set its mask to 0 to show it's been used. */
647 memset(set_mask
+ f
->offset
, 0, f
->size
);
651 if (!is_all_zeros(&flower
->rewrite
, sizeof flower
->rewrite
)) {
652 flower
->rewrite
.rewrite
= true;
655 if (hasmask
&& !is_all_zeros(set_mask
, size
)) {
656 VLOG_DBG_RL(&rl
, "unsupported sub attribute of set action type %d",
658 ofpbuf_uninit(&set_buf
);
662 ofpbuf_uninit(&set_buf
);
667 parse_put_flow_set_action(struct tc_flower
*flower
, const struct nlattr
*set
,
670 const struct nlattr
*tunnel
;
671 const struct nlattr
*tun_attr
;
672 size_t tun_left
, tunnel_len
;
674 if (nl_attr_type(set
) != OVS_KEY_ATTR_TUNNEL
) {
675 return parse_put_flow_set_masked_action(flower
, set
, set_len
,
679 tunnel
= nl_attr_get(set
);
680 tunnel_len
= nl_attr_get_size(set
);
682 flower
->set
.set
= true;
683 NL_ATTR_FOR_EACH_UNSAFE(tun_attr
, tun_left
, tunnel
, tunnel_len
) {
684 switch (nl_attr_type(tun_attr
)) {
685 case OVS_TUNNEL_KEY_ATTR_ID
: {
686 flower
->set
.id
= nl_attr_get_be64(tun_attr
);
689 case OVS_TUNNEL_KEY_ATTR_IPV4_SRC
: {
690 flower
->set
.ipv4
.ipv4_src
= nl_attr_get_be32(tun_attr
);
693 case OVS_TUNNEL_KEY_ATTR_IPV4_DST
: {
694 flower
->set
.ipv4
.ipv4_dst
= nl_attr_get_be32(tun_attr
);
697 case OVS_TUNNEL_KEY_ATTR_IPV6_SRC
: {
698 flower
->set
.ipv6
.ipv6_src
=
699 nl_attr_get_in6_addr(tun_attr
);
702 case OVS_TUNNEL_KEY_ATTR_IPV6_DST
: {
703 flower
->set
.ipv6
.ipv6_dst
=
704 nl_attr_get_in6_addr(tun_attr
);
707 case OVS_TUNNEL_KEY_ATTR_TP_SRC
: {
708 flower
->set
.tp_src
= nl_attr_get_be16(tun_attr
);
711 case OVS_TUNNEL_KEY_ATTR_TP_DST
: {
712 flower
->set
.tp_dst
= nl_attr_get_be16(tun_attr
);
722 test_key_and_mask(struct match
*match
)
724 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
725 const struct flow
*key
= &match
->flow
;
726 struct flow
*mask
= &match
->wc
.masks
;
728 if (mask
->pkt_mark
) {
729 VLOG_DBG_RL(&rl
, "offloading attribute pkt_mark isn't supported");
733 if (mask
->recirc_id
&& key
->recirc_id
) {
734 VLOG_DBG_RL(&rl
, "offloading attribute recirc_id isn't supported");
740 VLOG_DBG_RL(&rl
, "offloading attribute dp_hash isn't supported");
745 VLOG_DBG_RL(&rl
, "offloading attribute conj_id isn't supported");
749 if (mask
->skb_priority
) {
750 VLOG_DBG_RL(&rl
, "offloading attribute skb_priority isn't supported");
754 if (mask
->actset_output
) {
756 "offloading attribute actset_output isn't supported");
760 if (mask
->ct_state
) {
761 VLOG_DBG_RL(&rl
, "offloading attribute ct_state isn't supported");
766 VLOG_DBG_RL(&rl
, "offloading attribute ct_zone isn't supported");
771 VLOG_DBG_RL(&rl
, "offloading attribute ct_mark isn't supported");
775 if (mask
->packet_type
&& key
->packet_type
) {
776 VLOG_DBG_RL(&rl
, "offloading attribute packet_type isn't supported");
779 mask
->packet_type
= 0;
781 if (!ovs_u128_is_zero(mask
->ct_label
)) {
782 VLOG_DBG_RL(&rl
, "offloading attribute ct_label isn't supported");
786 for (int i
= 0; i
< FLOW_N_REGS
; i
++) {
789 "offloading attribute regs[%d] isn't supported", i
);
794 if (mask
->metadata
) {
795 VLOG_DBG_RL(&rl
, "offloading attribute metadata isn't supported");
800 VLOG_DBG_RL(&rl
, "offloading attribute nw_tos isn't supported");
804 for (int i
= 0; i
< FLOW_MAX_MPLS_LABELS
; i
++) {
805 if (mask
->mpls_lse
[i
]) {
806 VLOG_DBG_RL(&rl
, "offloading attribute mpls_lse isn't supported");
811 if (key
->dl_type
== htons(ETH_TYPE_IP
) &&
812 key
->nw_proto
== IPPROTO_ICMP
) {
815 "offloading attribute icmp_type isn't supported");
820 "offloading attribute icmp_code isn't supported");
823 } else if (key
->dl_type
== htons(ETH_TYPE_IP
) &&
824 key
->nw_proto
== IPPROTO_IGMP
) {
827 "offloading attribute igmp_type isn't supported");
832 "offloading attribute igmp_code isn't supported");
835 } else if (key
->dl_type
== htons(ETH_TYPE_IPV6
) &&
836 key
->nw_proto
== IPPROTO_ICMPV6
) {
839 "offloading attribute icmp_type isn't supported");
844 "offloading attribute icmp_code isn't supported");
849 if (!is_all_zeros(mask
, sizeof *mask
)) {
850 VLOG_DBG_RL(&rl
, "offloading isn't supported, unknown attribute");
858 netdev_tc_flow_put(struct netdev
*netdev
, struct match
*match
,
859 struct nlattr
*actions
, size_t actions_len
,
860 const ovs_u128
*ufid
, struct offload_info
*info
,
861 struct dpif_flow_stats
*stats OVS_UNUSED
)
863 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
864 struct tc_flower flower
;
865 const struct flow
*key
= &match
->flow
;
866 struct flow
*mask
= &match
->wc
.masks
;
867 const struct flow_tnl
*tnl
= &match
->flow
.tunnel
;
875 ifindex
= netdev_get_ifindex(netdev
);
877 VLOG_ERR_RL(&error_rl
, "flow_put: failed to get ifindex for %s: %s",
878 netdev_get_name(netdev
), ovs_strerror(-ifindex
));
882 memset(&flower
, 0, sizeof flower
);
884 if (flow_tnl_dst_is_set(&key
->tunnel
)) {
886 "tunnel: id %#" PRIx64
" src " IP_FMT
887 " dst " IP_FMT
" tp_src %d tp_dst %d",
889 IP_ARGS(tnl
->ip_src
), IP_ARGS(tnl
->ip_dst
),
890 ntohs(tnl
->tp_src
), ntohs(tnl
->tp_dst
));
891 flower
.tunnel
.id
= tnl
->tun_id
;
892 flower
.tunnel
.ipv4
.ipv4_src
= tnl
->ip_src
;
893 flower
.tunnel
.ipv4
.ipv4_dst
= tnl
->ip_dst
;
894 flower
.tunnel
.ipv6
.ipv6_src
= tnl
->ipv6_src
;
895 flower
.tunnel
.ipv6
.ipv6_dst
= tnl
->ipv6_dst
;
896 flower
.tunnel
.tp_src
= tnl
->tp_src
;
897 flower
.tunnel
.tp_dst
= tnl
->tp_dst
;
898 flower
.tunnel
.tunnel
= true;
900 memset(&mask
->tunnel
, 0, sizeof mask
->tunnel
);
902 flower
.key
.eth_type
= key
->dl_type
;
903 flower
.mask
.eth_type
= mask
->dl_type
;
905 if (mask
->vlans
[0].tci
) {
906 ovs_be16 vid_mask
= mask
->vlans
[0].tci
& htons(VLAN_VID_MASK
);
907 ovs_be16 pcp_mask
= mask
->vlans
[0].tci
& htons(VLAN_PCP_MASK
);
908 ovs_be16 cfi
= mask
->vlans
[0].tci
& htons(VLAN_CFI
);
910 if (cfi
&& key
->vlans
[0].tci
& htons(VLAN_CFI
)
911 && (!vid_mask
|| vid_mask
== htons(VLAN_VID_MASK
))
912 && (!pcp_mask
|| pcp_mask
== htons(VLAN_PCP_MASK
))
913 && (vid_mask
|| pcp_mask
)) {
915 flower
.key
.vlan_id
= vlan_tci_to_vid(key
->vlans
[0].tci
);
916 VLOG_DBG_RL(&rl
, "vlan_id: %d\n", flower
.key
.vlan_id
);
919 flower
.key
.vlan_prio
= vlan_tci_to_pcp(key
->vlans
[0].tci
);
920 VLOG_DBG_RL(&rl
, "vlan_prio: %d\n", flower
.key
.vlan_prio
);
922 flower
.key
.encap_eth_type
= flower
.key
.eth_type
;
923 flower
.key
.eth_type
= htons(ETH_TYPE_VLAN
);
924 } else if (mask
->vlans
[0].tci
== htons(0xffff) &&
925 ntohs(key
->vlans
[0].tci
) == 0) {
926 /* exact && no vlan */
931 } else if (mask
->vlans
[1].tci
) {
934 memset(mask
->vlans
, 0, sizeof mask
->vlans
);
936 flower
.key
.dst_mac
= key
->dl_dst
;
937 flower
.mask
.dst_mac
= mask
->dl_dst
;
938 flower
.key
.src_mac
= key
->dl_src
;
939 flower
.mask
.src_mac
= mask
->dl_src
;
940 memset(&mask
->dl_dst
, 0, sizeof mask
->dl_dst
);
941 memset(&mask
->dl_src
, 0, sizeof mask
->dl_src
);
943 mask
->in_port
.odp_port
= 0;
945 if (is_ip_any(key
)) {
946 flower
.key
.ip_proto
= key
->nw_proto
;
947 flower
.mask
.ip_proto
= mask
->nw_proto
;
948 flower
.key
.ip_ttl
= key
->nw_ttl
;
949 flower
.mask
.ip_ttl
= mask
->nw_ttl
;
951 if (mask
->nw_frag
& FLOW_NW_FRAG_ANY
) {
952 flower
.mask
.flags
|= TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT
;
954 if (key
->nw_frag
& FLOW_NW_FRAG_ANY
) {
955 flower
.key
.flags
|= TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT
;
957 if (mask
->nw_frag
& FLOW_NW_FRAG_LATER
) {
958 flower
.mask
.flags
|= TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST
;
960 if (!(key
->nw_frag
& FLOW_NW_FRAG_LATER
)) {
961 flower
.key
.flags
|= TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST
;
969 if (key
->nw_proto
== IPPROTO_TCP
) {
970 flower
.key
.tcp_dst
= key
->tp_dst
;
971 flower
.mask
.tcp_dst
= mask
->tp_dst
;
972 flower
.key
.tcp_src
= key
->tp_src
;
973 flower
.mask
.tcp_src
= mask
->tp_src
;
974 flower
.key
.tcp_flags
= key
->tcp_flags
;
975 flower
.mask
.tcp_flags
= mask
->tcp_flags
;
979 } else if (key
->nw_proto
== IPPROTO_UDP
) {
980 flower
.key
.udp_dst
= key
->tp_dst
;
981 flower
.mask
.udp_dst
= mask
->tp_dst
;
982 flower
.key
.udp_src
= key
->tp_src
;
983 flower
.mask
.udp_src
= mask
->tp_src
;
986 } else if (key
->nw_proto
== IPPROTO_SCTP
) {
987 flower
.key
.sctp_dst
= key
->tp_dst
;
988 flower
.mask
.sctp_dst
= mask
->tp_dst
;
989 flower
.key
.sctp_src
= key
->tp_src
;
990 flower
.mask
.sctp_src
= mask
->tp_src
;
999 if (key
->dl_type
== htons(ETH_P_IP
)) {
1000 flower
.key
.ipv4
.ipv4_src
= key
->nw_src
;
1001 flower
.mask
.ipv4
.ipv4_src
= mask
->nw_src
;
1002 flower
.key
.ipv4
.ipv4_dst
= key
->nw_dst
;
1003 flower
.mask
.ipv4
.ipv4_dst
= mask
->nw_dst
;
1006 } else if (key
->dl_type
== htons(ETH_P_IPV6
)) {
1007 flower
.key
.ipv6
.ipv6_src
= key
->ipv6_src
;
1008 flower
.mask
.ipv6
.ipv6_src
= mask
->ipv6_src
;
1009 flower
.key
.ipv6
.ipv6_dst
= key
->ipv6_dst
;
1010 flower
.mask
.ipv6
.ipv6_dst
= mask
->ipv6_dst
;
1011 memset(&mask
->ipv6_src
, 0, sizeof mask
->ipv6_src
);
1012 memset(&mask
->ipv6_dst
, 0, sizeof mask
->ipv6_dst
);
1016 err
= test_key_and_mask(match
);
1021 NL_ATTR_FOR_EACH(nla
, left
, actions
, actions_len
) {
1022 if (nl_attr_type(nla
) == OVS_ACTION_ATTR_OUTPUT
) {
1023 odp_port_t port
= nl_attr_get_odp_port(nla
);
1024 struct netdev
*outdev
= netdev_ports_get(port
, info
->dpif_class
);
1026 flower
.ifindex_out
= netdev_get_ifindex(outdev
);
1027 flower
.set
.tp_dst
= info
->tp_dst_port
;
1028 netdev_close(outdev
);
1029 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_PUSH_VLAN
) {
1030 const struct ovs_action_push_vlan
*vlan_push
= nl_attr_get(nla
);
1032 flower
.vlan_push_id
= vlan_tci_to_vid(vlan_push
->vlan_tci
);
1033 flower
.vlan_push_prio
= vlan_tci_to_pcp(vlan_push
->vlan_tci
);
1034 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_POP_VLAN
) {
1035 flower
.vlan_pop
= 1;
1036 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_SET
) {
1037 const struct nlattr
*set
= nl_attr_get(nla
);
1038 const size_t set_len
= nl_attr_get_size(nla
);
1040 err
= parse_put_flow_set_action(&flower
, set
, set_len
);
1044 } else if (nl_attr_type(nla
) == OVS_ACTION_ATTR_SET_MASKED
) {
1045 const struct nlattr
*set
= nl_attr_get(nla
);
1046 const size_t set_len
= nl_attr_get_size(nla
);
1048 err
= parse_put_flow_set_masked_action(&flower
, set
, set_len
,
1054 VLOG_DBG_RL(&rl
, "unsupported put action type: %d",
1060 handle
= get_ufid_tc_mapping(ufid
, &prio
, NULL
);
1061 if (handle
&& prio
) {
1062 VLOG_DBG_RL(&rl
, "updating old handle: %d prio: %d", handle
, prio
);
1063 tc_del_filter(ifindex
, prio
, handle
);
1067 prio
= get_prio_for_tc_flower(&flower
);
1069 VLOG_ERR_RL(&rl
, "couldn't get tc prio: %s", ovs_strerror(ENOSPC
));
1074 flower
.act_cookie
.data
= ufid
;
1075 flower
.act_cookie
.len
= sizeof *ufid
;
1077 err
= tc_replace_flower(ifindex
, prio
, handle
, &flower
);
1079 add_ufid_tc_mapping(ufid
, flower
.prio
, flower
.handle
, netdev
, ifindex
);
1086 netdev_tc_flow_get(struct netdev
*netdev OVS_UNUSED
,
1087 struct match
*match
,
1088 struct nlattr
**actions
,
1089 const ovs_u128
*ufid
,
1090 struct dpif_flow_stats
*stats
,
1093 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1095 struct tc_flower flower
;
1102 handle
= get_ufid_tc_mapping(ufid
, &prio
, &dev
);
1107 ifindex
= netdev_get_ifindex(dev
);
1109 VLOG_ERR_RL(&error_rl
, "flow_get: failed to get ifindex for %s: %s",
1110 netdev_get_name(dev
), ovs_strerror(-ifindex
));
1115 VLOG_DBG_RL(&rl
, "flow get (dev %s prio %d handle %d)",
1116 netdev_get_name(dev
), prio
, handle
);
1117 err
= tc_get_flower(ifindex
, prio
, handle
, &flower
);
1120 VLOG_ERR_RL(&error_rl
, "flow get failed (dev %s prio %d handle %d): %s",
1121 netdev_get_name(dev
), prio
, handle
, ovs_strerror(err
));
1125 in_port
= netdev_ifindex_to_odp_port(ifindex
);
1126 parse_tc_flower_to_match(&flower
, match
, actions
, stats
, buf
);
1128 match
->wc
.masks
.in_port
.odp_port
= u32_to_odp(UINT32_MAX
);
1129 match
->flow
.in_port
.odp_port
= in_port
;
1135 netdev_tc_flow_del(struct netdev
*netdev OVS_UNUSED
,
1136 const ovs_u128
*ufid
,
1137 struct dpif_flow_stats
*stats
)
1139 struct tc_flower flower
;
1146 handle
= get_ufid_tc_mapping(ufid
, &prio
, &dev
);
1151 ifindex
= netdev_get_ifindex(dev
);
1153 VLOG_ERR_RL(&error_rl
, "flow_del: failed to get ifindex for %s: %s",
1154 netdev_get_name(dev
), ovs_strerror(-ifindex
));
1160 memset(stats
, 0, sizeof *stats
);
1161 if (!tc_get_flower(ifindex
, prio
, handle
, &flower
)) {
1162 stats
->n_packets
= get_32aligned_u64(&flower
.stats
.n_packets
);
1163 stats
->n_bytes
= get_32aligned_u64(&flower
.stats
.n_bytes
);
1164 stats
->used
= flower
.lastused
;
1168 error
= tc_del_filter(ifindex
, prio
, handle
);
1169 del_ufid_tc_mapping(ufid
);
1177 netdev_tc_init_flow_api(struct netdev
*netdev
)
1182 ifindex
= netdev_get_ifindex(netdev
);
1184 VLOG_ERR_RL(&error_rl
, "init: failed to get ifindex for %s: %s",
1185 netdev_get_name(netdev
), ovs_strerror(-ifindex
));
1189 error
= tc_add_del_ingress_qdisc(ifindex
, true);
1191 if (error
&& error
!= EEXIST
) {
1192 VLOG_ERR("failed adding ingress qdisc required for offloading: %s",
1193 ovs_strerror(error
));
1197 VLOG_INFO("added ingress qdisc to %s", netdev_get_name(netdev
));