2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <net/flow_dissector.h>
34 #include <net/sch_generic.h>
35 #include <net/pkt_cls.h>
36 #include <net/tc_act/tc_gact.h>
37 #include <net/tc_act/tc_skbedit.h>
38 #include <linux/mlx5/fs.h>
39 #include <linux/mlx5/device.h>
40 #include <linux/rhashtable.h>
41 #include <linux/refcount.h>
42 #include <linux/completion.h>
43 #include <net/tc_act/tc_mirred.h>
44 #include <net/tc_act/tc_vlan.h>
45 #include <net/tc_act/tc_tunnel_key.h>
46 #include <net/tc_act/tc_pedit.h>
47 #include <net/tc_act/tc_csum.h>
49 #include <net/ipv6_stubs.h>
56 #include "en/tc_tun.h"
57 #include "lib/devcom.h"
58 #include "lib/geneve.h"
59 #include "diag/en_tc_tracepoint.h"
61 struct mlx5_nic_flow_attr
{
64 struct mlx5_modify_hdr
*modify_hdr
;
67 struct mlx5_flow_table
*hairpin_ft
;
68 struct mlx5_fc
*counter
;
71 #define MLX5E_TC_FLOW_BASE (MLX5E_TC_FLAG_LAST_EXPORTED_BIT + 1)
74 MLX5E_TC_FLOW_FLAG_INGRESS
= MLX5E_TC_FLAG_INGRESS_BIT
,
75 MLX5E_TC_FLOW_FLAG_EGRESS
= MLX5E_TC_FLAG_EGRESS_BIT
,
76 MLX5E_TC_FLOW_FLAG_ESWITCH
= MLX5E_TC_FLAG_ESW_OFFLOAD_BIT
,
77 MLX5E_TC_FLOW_FLAG_FT
= MLX5E_TC_FLAG_FT_OFFLOAD_BIT
,
78 MLX5E_TC_FLOW_FLAG_NIC
= MLX5E_TC_FLAG_NIC_OFFLOAD_BIT
,
79 MLX5E_TC_FLOW_FLAG_OFFLOADED
= MLX5E_TC_FLOW_BASE
,
80 MLX5E_TC_FLOW_FLAG_HAIRPIN
= MLX5E_TC_FLOW_BASE
+ 1,
81 MLX5E_TC_FLOW_FLAG_HAIRPIN_RSS
= MLX5E_TC_FLOW_BASE
+ 2,
82 MLX5E_TC_FLOW_FLAG_SLOW
= MLX5E_TC_FLOW_BASE
+ 3,
83 MLX5E_TC_FLOW_FLAG_DUP
= MLX5E_TC_FLOW_BASE
+ 4,
84 MLX5E_TC_FLOW_FLAG_NOT_READY
= MLX5E_TC_FLOW_BASE
+ 5,
85 MLX5E_TC_FLOW_FLAG_DELETED
= MLX5E_TC_FLOW_BASE
+ 6,
88 #define MLX5E_TC_MAX_SPLITS 1
90 /* Helper struct for accessing a struct containing list_head array.
99 * To access the containing struct from one of the list_head items:
100 * 1. Get the helper item from the list_head item using
102 * container_of(list_head item, helper struct type, list_head field)
103 * 2. Get the contining struct from the helper item and its index in the array:
104 * containing struct =
105 * container_of(helper item, containing struct type, helper field[index])
107 struct encap_flow_item
{
108 struct mlx5e_encap_entry
*e
; /* attached encap instance */
109 struct list_head list
;
113 struct mlx5e_tc_flow
{
114 struct rhash_head node
;
115 struct mlx5e_priv
*priv
;
118 struct mlx5_flow_handle
*rule
[MLX5E_TC_MAX_SPLITS
+ 1];
119 /* Flow can be associated with multiple encap IDs.
120 * The number of encaps is bounded by the number of supported
123 struct encap_flow_item encaps
[MLX5_MAX_FLOW_FWD_VPORTS
];
124 struct mlx5e_tc_flow
*peer_flow
;
125 struct mlx5e_mod_hdr_entry
*mh
; /* attached mod header instance */
126 struct list_head mod_hdr
; /* flows sharing the same mod hdr ID */
127 struct mlx5e_hairpin_entry
*hpe
; /* attached hairpin instance */
128 struct list_head hairpin
; /* flows sharing the same hairpin */
129 struct list_head peer
; /* flows with peer flow */
130 struct list_head unready
; /* flows not ready to be offloaded (e.g due to missing route) */
132 struct list_head tmp_list
; /* temporary flow list used by neigh update */
134 struct rcu_head rcu_head
;
135 struct completion init_done
;
137 struct mlx5_esw_flow_attr esw_attr
[0];
138 struct mlx5_nic_flow_attr nic_attr
[0];
142 struct mlx5e_tc_flow_parse_attr
{
143 const struct ip_tunnel_info
*tun_info
[MLX5_MAX_FLOW_FWD_VPORTS
];
144 struct net_device
*filter_dev
;
145 struct mlx5_flow_spec spec
;
146 int num_mod_hdr_actions
;
147 int max_mod_hdr_actions
;
148 void *mod_hdr_actions
;
149 int mirred_ifindex
[MLX5_MAX_FLOW_FWD_VPORTS
];
152 #define MLX5E_TC_TABLE_NUM_GROUPS 4
153 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(16)
155 struct mlx5e_hairpin
{
156 struct mlx5_hairpin
*pair
;
158 struct mlx5_core_dev
*func_mdev
;
159 struct mlx5e_priv
*func_priv
;
164 struct mlx5e_rqt indir_rqt
;
165 u32 indir_tirn
[MLX5E_NUM_INDIR_TIRS
];
166 struct mlx5e_ttc_table ttc
;
169 struct mlx5e_hairpin_entry
{
170 /* a node of a hash table which keeps all the hairpin entries */
171 struct hlist_node hairpin_hlist
;
173 /* protects flows list */
174 spinlock_t flows_lock
;
175 /* flows sharing the same hairpin */
176 struct list_head flows
;
177 /* hpe's that were not fully initialized when dead peer update event
178 * function traversed them.
180 struct list_head dead_peer_wait_list
;
184 struct mlx5e_hairpin
*hp
;
186 struct completion res_ready
;
194 struct mlx5e_mod_hdr_entry
{
195 /* a node of a hash table which keeps all the mod_hdr entries */
196 struct hlist_node mod_hdr_hlist
;
198 /* protects flows list */
199 spinlock_t flows_lock
;
200 /* flows sharing the same mod_hdr entry */
201 struct list_head flows
;
203 struct mod_hdr_key key
;
205 struct mlx5_modify_hdr
*modify_hdr
;
208 struct completion res_ready
;
212 #define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto)
214 static void mlx5e_tc_del_flow(struct mlx5e_priv
*priv
,
215 struct mlx5e_tc_flow
*flow
);
217 static struct mlx5e_tc_flow
*mlx5e_flow_get(struct mlx5e_tc_flow
*flow
)
219 if (!flow
|| !refcount_inc_not_zero(&flow
->refcnt
))
220 return ERR_PTR(-EINVAL
);
224 static void mlx5e_flow_put(struct mlx5e_priv
*priv
,
225 struct mlx5e_tc_flow
*flow
)
227 if (refcount_dec_and_test(&flow
->refcnt
)) {
228 mlx5e_tc_del_flow(priv
, flow
);
229 kfree_rcu(flow
, rcu_head
);
233 static void __flow_flag_set(struct mlx5e_tc_flow
*flow
, unsigned long flag
)
235 /* Complete all memory stores before setting bit. */
236 smp_mb__before_atomic();
237 set_bit(flag
, &flow
->flags
);
240 #define flow_flag_set(flow, flag) __flow_flag_set(flow, MLX5E_TC_FLOW_FLAG_##flag)
242 static bool __flow_flag_test_and_set(struct mlx5e_tc_flow
*flow
,
245 /* test_and_set_bit() provides all necessary barriers */
246 return test_and_set_bit(flag
, &flow
->flags
);
249 #define flow_flag_test_and_set(flow, flag) \
250 __flow_flag_test_and_set(flow, \
251 MLX5E_TC_FLOW_FLAG_##flag)
253 static void __flow_flag_clear(struct mlx5e_tc_flow
*flow
, unsigned long flag
)
255 /* Complete all memory stores before clearing bit. */
256 smp_mb__before_atomic();
257 clear_bit(flag
, &flow
->flags
);
260 #define flow_flag_clear(flow, flag) __flow_flag_clear(flow, \
261 MLX5E_TC_FLOW_FLAG_##flag)
263 static bool __flow_flag_test(struct mlx5e_tc_flow
*flow
, unsigned long flag
)
265 bool ret
= test_bit(flag
, &flow
->flags
);
267 /* Read fields of flow structure only after checking flags. */
268 smp_mb__after_atomic();
272 #define flow_flag_test(flow, flag) __flow_flag_test(flow, \
273 MLX5E_TC_FLOW_FLAG_##flag)
275 static bool mlx5e_is_eswitch_flow(struct mlx5e_tc_flow
*flow
)
277 return flow_flag_test(flow
, ESWITCH
);
280 static bool mlx5e_is_ft_flow(struct mlx5e_tc_flow
*flow
)
282 return flow_flag_test(flow
, FT
);
285 static bool mlx5e_is_offloaded_flow(struct mlx5e_tc_flow
*flow
)
287 return flow_flag_test(flow
, OFFLOADED
);
290 static inline u32
hash_mod_hdr_info(struct mod_hdr_key
*key
)
292 return jhash(key
->actions
,
293 key
->num_actions
* MLX5_MH_ACT_SZ
, 0);
296 static inline int cmp_mod_hdr_info(struct mod_hdr_key
*a
,
297 struct mod_hdr_key
*b
)
299 if (a
->num_actions
!= b
->num_actions
)
302 return memcmp(a
->actions
, b
->actions
, a
->num_actions
* MLX5_MH_ACT_SZ
);
305 static struct mod_hdr_tbl
*
306 get_mod_hdr_table(struct mlx5e_priv
*priv
, int namespace)
308 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
310 return namespace == MLX5_FLOW_NAMESPACE_FDB
? &esw
->offloads
.mod_hdr
:
311 &priv
->fs
.tc
.mod_hdr
;
314 static struct mlx5e_mod_hdr_entry
*
315 mlx5e_mod_hdr_get(struct mod_hdr_tbl
*tbl
, struct mod_hdr_key
*key
, u32 hash_key
)
317 struct mlx5e_mod_hdr_entry
*mh
, *found
= NULL
;
319 hash_for_each_possible(tbl
->hlist
, mh
, mod_hdr_hlist
, hash_key
) {
320 if (!cmp_mod_hdr_info(&mh
->key
, key
)) {
321 refcount_inc(&mh
->refcnt
);
330 static void mlx5e_mod_hdr_put(struct mlx5e_priv
*priv
,
331 struct mlx5e_mod_hdr_entry
*mh
,
334 struct mod_hdr_tbl
*tbl
= get_mod_hdr_table(priv
, namespace);
336 if (!refcount_dec_and_mutex_lock(&mh
->refcnt
, &tbl
->lock
))
338 hash_del(&mh
->mod_hdr_hlist
);
339 mutex_unlock(&tbl
->lock
);
341 WARN_ON(!list_empty(&mh
->flows
));
342 if (mh
->compl_result
> 0)
343 mlx5_modify_header_dealloc(priv
->mdev
, mh
->modify_hdr
);
348 static int get_flow_name_space(struct mlx5e_tc_flow
*flow
)
350 return mlx5e_is_eswitch_flow(flow
) ?
351 MLX5_FLOW_NAMESPACE_FDB
: MLX5_FLOW_NAMESPACE_KERNEL
;
353 static int mlx5e_attach_mod_hdr(struct mlx5e_priv
*priv
,
354 struct mlx5e_tc_flow
*flow
,
355 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
357 int num_actions
, actions_size
, namespace, err
;
358 struct mlx5e_mod_hdr_entry
*mh
;
359 struct mod_hdr_tbl
*tbl
;
360 struct mod_hdr_key key
;
363 num_actions
= parse_attr
->num_mod_hdr_actions
;
364 actions_size
= MLX5_MH_ACT_SZ
* num_actions
;
366 key
.actions
= parse_attr
->mod_hdr_actions
;
367 key
.num_actions
= num_actions
;
369 hash_key
= hash_mod_hdr_info(&key
);
371 namespace = get_flow_name_space(flow
);
372 tbl
= get_mod_hdr_table(priv
, namespace);
374 mutex_lock(&tbl
->lock
);
375 mh
= mlx5e_mod_hdr_get(tbl
, &key
, hash_key
);
377 mutex_unlock(&tbl
->lock
);
378 wait_for_completion(&mh
->res_ready
);
380 if (mh
->compl_result
< 0) {
382 goto attach_header_err
;
387 mh
= kzalloc(sizeof(*mh
) + actions_size
, GFP_KERNEL
);
389 mutex_unlock(&tbl
->lock
);
393 mh
->key
.actions
= (void *)mh
+ sizeof(*mh
);
394 memcpy(mh
->key
.actions
, key
.actions
, actions_size
);
395 mh
->key
.num_actions
= num_actions
;
396 spin_lock_init(&mh
->flows_lock
);
397 INIT_LIST_HEAD(&mh
->flows
);
398 refcount_set(&mh
->refcnt
, 1);
399 init_completion(&mh
->res_ready
);
401 hash_add(tbl
->hlist
, &mh
->mod_hdr_hlist
, hash_key
);
402 mutex_unlock(&tbl
->lock
);
404 mh
->modify_hdr
= mlx5_modify_header_alloc(priv
->mdev
, namespace,
407 if (IS_ERR(mh
->modify_hdr
)) {
408 err
= PTR_ERR(mh
->modify_hdr
);
409 mh
->compl_result
= err
;
410 goto alloc_header_err
;
412 mh
->compl_result
= 1;
413 complete_all(&mh
->res_ready
);
417 spin_lock(&mh
->flows_lock
);
418 list_add(&flow
->mod_hdr
, &mh
->flows
);
419 spin_unlock(&mh
->flows_lock
);
420 if (mlx5e_is_eswitch_flow(flow
))
421 flow
->esw_attr
->modify_hdr
= mh
->modify_hdr
;
423 flow
->nic_attr
->modify_hdr
= mh
->modify_hdr
;
428 complete_all(&mh
->res_ready
);
430 mlx5e_mod_hdr_put(priv
, mh
, namespace);
434 static void mlx5e_detach_mod_hdr(struct mlx5e_priv
*priv
,
435 struct mlx5e_tc_flow
*flow
)
437 /* flow wasn't fully initialized */
441 spin_lock(&flow
->mh
->flows_lock
);
442 list_del(&flow
->mod_hdr
);
443 spin_unlock(&flow
->mh
->flows_lock
);
445 mlx5e_mod_hdr_put(priv
, flow
->mh
, get_flow_name_space(flow
));
450 struct mlx5_core_dev
*mlx5e_hairpin_get_mdev(struct net
*net
, int ifindex
)
452 struct net_device
*netdev
;
453 struct mlx5e_priv
*priv
;
455 netdev
= __dev_get_by_index(net
, ifindex
);
456 priv
= netdev_priv(netdev
);
460 static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin
*hp
)
462 u32 in
[MLX5_ST_SZ_DW(create_tir_in
)] = {0};
466 err
= mlx5_core_alloc_transport_domain(hp
->func_mdev
, &hp
->tdn
);
470 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
472 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_DIRECT
);
473 MLX5_SET(tirc
, tirc
, inline_rqn
, hp
->pair
->rqn
[0]);
474 MLX5_SET(tirc
, tirc
, transport_domain
, hp
->tdn
);
476 err
= mlx5_core_create_tir(hp
->func_mdev
, in
, MLX5_ST_SZ_BYTES(create_tir_in
), &hp
->tirn
);
483 mlx5_core_dealloc_transport_domain(hp
->func_mdev
, hp
->tdn
);
488 static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin
*hp
)
490 mlx5_core_destroy_tir(hp
->func_mdev
, hp
->tirn
);
491 mlx5_core_dealloc_transport_domain(hp
->func_mdev
, hp
->tdn
);
494 static void mlx5e_hairpin_fill_rqt_rqns(struct mlx5e_hairpin
*hp
, void *rqtc
)
496 u32 indirection_rqt
[MLX5E_INDIR_RQT_SIZE
], rqn
;
497 struct mlx5e_priv
*priv
= hp
->func_priv
;
498 int i
, ix
, sz
= MLX5E_INDIR_RQT_SIZE
;
500 mlx5e_build_default_indir_rqt(indirection_rqt
, sz
,
503 for (i
= 0; i
< sz
; i
++) {
505 if (priv
->rss_params
.hfunc
== ETH_RSS_HASH_XOR
)
506 ix
= mlx5e_bits_invert(i
, ilog2(sz
));
507 ix
= indirection_rqt
[ix
];
508 rqn
= hp
->pair
->rqn
[ix
];
509 MLX5_SET(rqtc
, rqtc
, rq_num
[i
], rqn
);
513 static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin
*hp
)
515 int inlen
, err
, sz
= MLX5E_INDIR_RQT_SIZE
;
516 struct mlx5e_priv
*priv
= hp
->func_priv
;
517 struct mlx5_core_dev
*mdev
= priv
->mdev
;
521 inlen
= MLX5_ST_SZ_BYTES(create_rqt_in
) + sizeof(u32
) * sz
;
522 in
= kvzalloc(inlen
, GFP_KERNEL
);
526 rqtc
= MLX5_ADDR_OF(create_rqt_in
, in
, rqt_context
);
528 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
529 MLX5_SET(rqtc
, rqtc
, rqt_max_size
, sz
);
531 mlx5e_hairpin_fill_rqt_rqns(hp
, rqtc
);
533 err
= mlx5_core_create_rqt(mdev
, in
, inlen
, &hp
->indir_rqt
.rqtn
);
535 hp
->indir_rqt
.enabled
= true;
541 static int mlx5e_hairpin_create_indirect_tirs(struct mlx5e_hairpin
*hp
)
543 struct mlx5e_priv
*priv
= hp
->func_priv
;
544 u32 in
[MLX5_ST_SZ_DW(create_tir_in
)];
548 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
549 struct mlx5e_tirc_config ttconfig
= mlx5e_tirc_get_default_config(tt
);
551 memset(in
, 0, MLX5_ST_SZ_BYTES(create_tir_in
));
552 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
554 MLX5_SET(tirc
, tirc
, transport_domain
, hp
->tdn
);
555 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_INDIRECT
);
556 MLX5_SET(tirc
, tirc
, indirect_table
, hp
->indir_rqt
.rqtn
);
557 mlx5e_build_indir_tir_ctx_hash(&priv
->rss_params
, &ttconfig
, tirc
, false);
559 err
= mlx5_core_create_tir(hp
->func_mdev
, in
,
560 MLX5_ST_SZ_BYTES(create_tir_in
), &hp
->indir_tirn
[tt
]);
562 mlx5_core_warn(hp
->func_mdev
, "create indirect tirs failed, %d\n", err
);
563 goto err_destroy_tirs
;
569 for (i
= 0; i
< tt
; i
++)
570 mlx5_core_destroy_tir(hp
->func_mdev
, hp
->indir_tirn
[i
]);
574 static void mlx5e_hairpin_destroy_indirect_tirs(struct mlx5e_hairpin
*hp
)
578 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++)
579 mlx5_core_destroy_tir(hp
->func_mdev
, hp
->indir_tirn
[tt
]);
582 static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin
*hp
,
583 struct ttc_params
*ttc_params
)
585 struct mlx5_flow_table_attr
*ft_attr
= &ttc_params
->ft_attr
;
588 memset(ttc_params
, 0, sizeof(*ttc_params
));
590 ttc_params
->any_tt_tirn
= hp
->tirn
;
592 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++)
593 ttc_params
->indir_tirn
[tt
] = hp
->indir_tirn
[tt
];
595 ft_attr
->max_fte
= MLX5E_NUM_TT
;
596 ft_attr
->level
= MLX5E_TC_TTC_FT_LEVEL
;
597 ft_attr
->prio
= MLX5E_TC_PRIO
;
600 static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin
*hp
)
602 struct mlx5e_priv
*priv
= hp
->func_priv
;
603 struct ttc_params ttc_params
;
606 err
= mlx5e_hairpin_create_indirect_rqt(hp
);
610 err
= mlx5e_hairpin_create_indirect_tirs(hp
);
612 goto err_create_indirect_tirs
;
614 mlx5e_hairpin_set_ttc_params(hp
, &ttc_params
);
615 err
= mlx5e_create_ttc_table(priv
, &ttc_params
, &hp
->ttc
);
617 goto err_create_ttc_table
;
619 netdev_dbg(priv
->netdev
, "add hairpin: using %d channels rss ttc table id %x\n",
620 hp
->num_channels
, hp
->ttc
.ft
.t
->id
);
624 err_create_ttc_table
:
625 mlx5e_hairpin_destroy_indirect_tirs(hp
);
626 err_create_indirect_tirs
:
627 mlx5e_destroy_rqt(priv
, &hp
->indir_rqt
);
632 static void mlx5e_hairpin_rss_cleanup(struct mlx5e_hairpin
*hp
)
634 struct mlx5e_priv
*priv
= hp
->func_priv
;
636 mlx5e_destroy_ttc_table(priv
, &hp
->ttc
);
637 mlx5e_hairpin_destroy_indirect_tirs(hp
);
638 mlx5e_destroy_rqt(priv
, &hp
->indir_rqt
);
641 static struct mlx5e_hairpin
*
642 mlx5e_hairpin_create(struct mlx5e_priv
*priv
, struct mlx5_hairpin_params
*params
,
645 struct mlx5_core_dev
*func_mdev
, *peer_mdev
;
646 struct mlx5e_hairpin
*hp
;
647 struct mlx5_hairpin
*pair
;
650 hp
= kzalloc(sizeof(*hp
), GFP_KERNEL
);
652 return ERR_PTR(-ENOMEM
);
654 func_mdev
= priv
->mdev
;
655 peer_mdev
= mlx5e_hairpin_get_mdev(dev_net(priv
->netdev
), peer_ifindex
);
657 pair
= mlx5_core_hairpin_create(func_mdev
, peer_mdev
, params
);
660 goto create_pair_err
;
663 hp
->func_mdev
= func_mdev
;
664 hp
->func_priv
= priv
;
665 hp
->num_channels
= params
->num_channels
;
667 err
= mlx5e_hairpin_create_transport(hp
);
669 goto create_transport_err
;
671 if (hp
->num_channels
> 1) {
672 err
= mlx5e_hairpin_rss_init(hp
);
680 mlx5e_hairpin_destroy_transport(hp
);
681 create_transport_err
:
682 mlx5_core_hairpin_destroy(hp
->pair
);
688 static void mlx5e_hairpin_destroy(struct mlx5e_hairpin
*hp
)
690 if (hp
->num_channels
> 1)
691 mlx5e_hairpin_rss_cleanup(hp
);
692 mlx5e_hairpin_destroy_transport(hp
);
693 mlx5_core_hairpin_destroy(hp
->pair
);
697 static inline u32
hash_hairpin_info(u16 peer_vhca_id
, u8 prio
)
699 return (peer_vhca_id
<< 16 | prio
);
702 static struct mlx5e_hairpin_entry
*mlx5e_hairpin_get(struct mlx5e_priv
*priv
,
703 u16 peer_vhca_id
, u8 prio
)
705 struct mlx5e_hairpin_entry
*hpe
;
706 u32 hash_key
= hash_hairpin_info(peer_vhca_id
, prio
);
708 hash_for_each_possible(priv
->fs
.tc
.hairpin_tbl
, hpe
,
709 hairpin_hlist
, hash_key
) {
710 if (hpe
->peer_vhca_id
== peer_vhca_id
&& hpe
->prio
== prio
) {
711 refcount_inc(&hpe
->refcnt
);
719 static void mlx5e_hairpin_put(struct mlx5e_priv
*priv
,
720 struct mlx5e_hairpin_entry
*hpe
)
722 /* no more hairpin flows for us, release the hairpin pair */
723 if (!refcount_dec_and_mutex_lock(&hpe
->refcnt
, &priv
->fs
.tc
.hairpin_tbl_lock
))
725 hash_del(&hpe
->hairpin_hlist
);
726 mutex_unlock(&priv
->fs
.tc
.hairpin_tbl_lock
);
728 if (!IS_ERR_OR_NULL(hpe
->hp
)) {
729 netdev_dbg(priv
->netdev
, "del hairpin: peer %s\n",
730 dev_name(hpe
->hp
->pair
->peer_mdev
->device
));
732 mlx5e_hairpin_destroy(hpe
->hp
);
735 WARN_ON(!list_empty(&hpe
->flows
));
739 #define UNKNOWN_MATCH_PRIO 8
741 static int mlx5e_hairpin_get_prio(struct mlx5e_priv
*priv
,
742 struct mlx5_flow_spec
*spec
, u8
*match_prio
,
743 struct netlink_ext_ack
*extack
)
745 void *headers_c
, *headers_v
;
746 u8 prio_val
, prio_mask
= 0;
749 #ifdef CONFIG_MLX5_CORE_EN_DCB
750 if (priv
->dcbx_dp
.trust_state
!= MLX5_QPTS_TRUST_PCP
) {
751 NL_SET_ERR_MSG_MOD(extack
,
752 "only PCP trust state supported for hairpin");
756 headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, outer_headers
);
757 headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, outer_headers
);
759 vlan_present
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, cvlan_tag
);
761 prio_mask
= MLX5_GET(fte_match_set_lyr_2_4
, headers_c
, first_prio
);
762 prio_val
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, first_prio
);
765 if (!vlan_present
|| !prio_mask
) {
766 prio_val
= UNKNOWN_MATCH_PRIO
;
767 } else if (prio_mask
!= 0x7) {
768 NL_SET_ERR_MSG_MOD(extack
,
769 "masked priority match not supported for hairpin");
773 *match_prio
= prio_val
;
777 static int mlx5e_hairpin_flow_add(struct mlx5e_priv
*priv
,
778 struct mlx5e_tc_flow
*flow
,
779 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
780 struct netlink_ext_ack
*extack
)
782 int peer_ifindex
= parse_attr
->mirred_ifindex
[0];
783 struct mlx5_hairpin_params params
;
784 struct mlx5_core_dev
*peer_mdev
;
785 struct mlx5e_hairpin_entry
*hpe
;
786 struct mlx5e_hairpin
*hp
;
793 peer_mdev
= mlx5e_hairpin_get_mdev(dev_net(priv
->netdev
), peer_ifindex
);
794 if (!MLX5_CAP_GEN(priv
->mdev
, hairpin
) || !MLX5_CAP_GEN(peer_mdev
, hairpin
)) {
795 NL_SET_ERR_MSG_MOD(extack
, "hairpin is not supported");
799 peer_id
= MLX5_CAP_GEN(peer_mdev
, vhca_id
);
800 err
= mlx5e_hairpin_get_prio(priv
, &parse_attr
->spec
, &match_prio
,
805 mutex_lock(&priv
->fs
.tc
.hairpin_tbl_lock
);
806 hpe
= mlx5e_hairpin_get(priv
, peer_id
, match_prio
);
808 mutex_unlock(&priv
->fs
.tc
.hairpin_tbl_lock
);
809 wait_for_completion(&hpe
->res_ready
);
811 if (IS_ERR(hpe
->hp
)) {
818 hpe
= kzalloc(sizeof(*hpe
), GFP_KERNEL
);
820 mutex_unlock(&priv
->fs
.tc
.hairpin_tbl_lock
);
824 spin_lock_init(&hpe
->flows_lock
);
825 INIT_LIST_HEAD(&hpe
->flows
);
826 INIT_LIST_HEAD(&hpe
->dead_peer_wait_list
);
827 hpe
->peer_vhca_id
= peer_id
;
828 hpe
->prio
= match_prio
;
829 refcount_set(&hpe
->refcnt
, 1);
830 init_completion(&hpe
->res_ready
);
832 hash_add(priv
->fs
.tc
.hairpin_tbl
, &hpe
->hairpin_hlist
,
833 hash_hairpin_info(peer_id
, match_prio
));
834 mutex_unlock(&priv
->fs
.tc
.hairpin_tbl_lock
);
836 params
.log_data_size
= 15;
837 params
.log_data_size
= min_t(u8
, params
.log_data_size
,
838 MLX5_CAP_GEN(priv
->mdev
, log_max_hairpin_wq_data_sz
));
839 params
.log_data_size
= max_t(u8
, params
.log_data_size
,
840 MLX5_CAP_GEN(priv
->mdev
, log_min_hairpin_wq_data_sz
));
842 params
.log_num_packets
= params
.log_data_size
-
843 MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(priv
->mdev
);
844 params
.log_num_packets
= min_t(u8
, params
.log_num_packets
,
845 MLX5_CAP_GEN(priv
->mdev
, log_max_hairpin_num_packets
));
847 params
.q_counter
= priv
->q_counter
;
848 /* set hairpin pair per each 50Gbs share of the link */
849 mlx5e_port_max_linkspeed(priv
->mdev
, &link_speed
);
850 link_speed
= max_t(u32
, link_speed
, 50000);
851 link_speed64
= link_speed
;
852 do_div(link_speed64
, 50000);
853 params
.num_channels
= link_speed64
;
855 hp
= mlx5e_hairpin_create(priv
, ¶ms
, peer_ifindex
);
857 complete_all(&hpe
->res_ready
);
863 netdev_dbg(priv
->netdev
, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d (log) data %d packets %d\n",
864 hp
->tirn
, hp
->pair
->rqn
[0],
865 dev_name(hp
->pair
->peer_mdev
->device
),
866 hp
->pair
->sqn
[0], match_prio
, params
.log_data_size
, params
.log_num_packets
);
869 if (hpe
->hp
->num_channels
> 1) {
870 flow_flag_set(flow
, HAIRPIN_RSS
);
871 flow
->nic_attr
->hairpin_ft
= hpe
->hp
->ttc
.ft
.t
;
873 flow
->nic_attr
->hairpin_tirn
= hpe
->hp
->tirn
;
877 spin_lock(&hpe
->flows_lock
);
878 list_add(&flow
->hairpin
, &hpe
->flows
);
879 spin_unlock(&hpe
->flows_lock
);
884 mlx5e_hairpin_put(priv
, hpe
);
888 static void mlx5e_hairpin_flow_del(struct mlx5e_priv
*priv
,
889 struct mlx5e_tc_flow
*flow
)
891 /* flow wasn't fully initialized */
895 spin_lock(&flow
->hpe
->flows_lock
);
896 list_del(&flow
->hairpin
);
897 spin_unlock(&flow
->hpe
->flows_lock
);
899 mlx5e_hairpin_put(priv
, flow
->hpe
);
904 mlx5e_tc_add_nic_flow(struct mlx5e_priv
*priv
,
905 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
906 struct mlx5e_tc_flow
*flow
,
907 struct netlink_ext_ack
*extack
)
909 struct mlx5_flow_context
*flow_context
= &parse_attr
->spec
.flow_context
;
910 struct mlx5_nic_flow_attr
*attr
= flow
->nic_attr
;
911 struct mlx5_core_dev
*dev
= priv
->mdev
;
912 struct mlx5_flow_destination dest
[2] = {};
913 struct mlx5_flow_act flow_act
= {
914 .action
= attr
->action
,
915 .flags
= FLOW_ACT_NO_APPEND
,
917 struct mlx5_fc
*counter
= NULL
;
918 int err
, dest_ix
= 0;
920 flow_context
->flags
|= FLOW_CONTEXT_HAS_TAG
;
921 flow_context
->flow_tag
= attr
->flow_tag
;
923 if (flow_flag_test(flow
, HAIRPIN
)) {
924 err
= mlx5e_hairpin_flow_add(priv
, flow
, parse_attr
, extack
);
928 if (flow_flag_test(flow
, HAIRPIN_RSS
)) {
929 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
930 dest
[dest_ix
].ft
= attr
->hairpin_ft
;
932 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
933 dest
[dest_ix
].tir_num
= attr
->hairpin_tirn
;
936 } else if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
937 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
938 dest
[dest_ix
].ft
= priv
->fs
.vlan
.ft
.t
;
942 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
943 counter
= mlx5_fc_create(dev
, true);
945 return PTR_ERR(counter
);
947 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
948 dest
[dest_ix
].counter_id
= mlx5_fc_id(counter
);
950 attr
->counter
= counter
;
953 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
) {
954 err
= mlx5e_attach_mod_hdr(priv
, flow
, parse_attr
);
955 flow_act
.modify_hdr
= attr
->modify_hdr
;
956 kfree(parse_attr
->mod_hdr_actions
);
961 mutex_lock(&priv
->fs
.tc
.t_lock
);
962 if (IS_ERR_OR_NULL(priv
->fs
.tc
.t
)) {
963 int tc_grp_size
, tc_tbl_size
;
964 u32 max_flow_counter
;
966 max_flow_counter
= (MLX5_CAP_GEN(dev
, max_flow_counter_31_16
) << 16) |
967 MLX5_CAP_GEN(dev
, max_flow_counter_15_0
);
969 tc_grp_size
= min_t(int, max_flow_counter
, MLX5E_TC_TABLE_MAX_GROUP_SIZE
);
971 tc_tbl_size
= min_t(int, tc_grp_size
* MLX5E_TC_TABLE_NUM_GROUPS
,
972 BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev
, log_max_ft_size
)));
975 mlx5_create_auto_grouped_flow_table(priv
->fs
.ns
,
978 MLX5E_TC_TABLE_NUM_GROUPS
,
979 MLX5E_TC_FT_LEVEL
, 0);
980 if (IS_ERR(priv
->fs
.tc
.t
)) {
981 mutex_unlock(&priv
->fs
.tc
.t_lock
);
982 NL_SET_ERR_MSG_MOD(extack
,
983 "Failed to create tc offload table\n");
984 netdev_err(priv
->netdev
,
985 "Failed to create tc offload table\n");
986 return PTR_ERR(priv
->fs
.tc
.t
);
990 if (attr
->match_level
!= MLX5_MATCH_NONE
)
991 parse_attr
->spec
.match_criteria_enable
|= MLX5_MATCH_OUTER_HEADERS
;
993 flow
->rule
[0] = mlx5_add_flow_rules(priv
->fs
.tc
.t
, &parse_attr
->spec
,
994 &flow_act
, dest
, dest_ix
);
995 mutex_unlock(&priv
->fs
.tc
.t_lock
);
997 return PTR_ERR_OR_ZERO(flow
->rule
[0]);
1000 static void mlx5e_tc_del_nic_flow(struct mlx5e_priv
*priv
,
1001 struct mlx5e_tc_flow
*flow
)
1003 struct mlx5_nic_flow_attr
*attr
= flow
->nic_attr
;
1004 struct mlx5_fc
*counter
= NULL
;
1006 counter
= attr
->counter
;
1007 if (!IS_ERR_OR_NULL(flow
->rule
[0]))
1008 mlx5_del_flow_rules(flow
->rule
[0]);
1009 mlx5_fc_destroy(priv
->mdev
, counter
);
1011 mutex_lock(&priv
->fs
.tc
.t_lock
);
1012 if (!mlx5e_tc_num_filters(priv
, MLX5_TC_FLAG(NIC_OFFLOAD
)) && priv
->fs
.tc
.t
) {
1013 mlx5_destroy_flow_table(priv
->fs
.tc
.t
);
1014 priv
->fs
.tc
.t
= NULL
;
1016 mutex_unlock(&priv
->fs
.tc
.t_lock
);
1018 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
1019 mlx5e_detach_mod_hdr(priv
, flow
);
1021 if (flow_flag_test(flow
, HAIRPIN
))
1022 mlx5e_hairpin_flow_del(priv
, flow
);
1025 static void mlx5e_detach_encap(struct mlx5e_priv
*priv
,
1026 struct mlx5e_tc_flow
*flow
, int out_index
);
1028 static int mlx5e_attach_encap(struct mlx5e_priv
*priv
,
1029 struct mlx5e_tc_flow
*flow
,
1030 struct net_device
*mirred_dev
,
1032 struct netlink_ext_ack
*extack
,
1033 struct net_device
**encap_dev
,
1036 static struct mlx5_flow_handle
*
1037 mlx5e_tc_offload_fdb_rules(struct mlx5_eswitch
*esw
,
1038 struct mlx5e_tc_flow
*flow
,
1039 struct mlx5_flow_spec
*spec
,
1040 struct mlx5_esw_flow_attr
*attr
)
1042 struct mlx5_flow_handle
*rule
;
1044 rule
= mlx5_eswitch_add_offloaded_rule(esw
, spec
, attr
);
1048 if (attr
->split_count
) {
1049 flow
->rule
[1] = mlx5_eswitch_add_fwd_rule(esw
, spec
, attr
);
1050 if (IS_ERR(flow
->rule
[1])) {
1051 mlx5_eswitch_del_offloaded_rule(esw
, rule
, attr
);
1052 return flow
->rule
[1];
1060 mlx5e_tc_unoffload_fdb_rules(struct mlx5_eswitch
*esw
,
1061 struct mlx5e_tc_flow
*flow
,
1062 struct mlx5_esw_flow_attr
*attr
)
1064 flow_flag_clear(flow
, OFFLOADED
);
1066 if (attr
->split_count
)
1067 mlx5_eswitch_del_fwd_rule(esw
, flow
->rule
[1], attr
);
1069 mlx5_eswitch_del_offloaded_rule(esw
, flow
->rule
[0], attr
);
1072 static struct mlx5_flow_handle
*
1073 mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch
*esw
,
1074 struct mlx5e_tc_flow
*flow
,
1075 struct mlx5_flow_spec
*spec
,
1076 struct mlx5_esw_flow_attr
*slow_attr
)
1078 struct mlx5_flow_handle
*rule
;
1080 memcpy(slow_attr
, flow
->esw_attr
, sizeof(*slow_attr
));
1081 slow_attr
->action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1082 slow_attr
->split_count
= 0;
1083 slow_attr
->dest_chain
= FDB_TC_SLOW_PATH_CHAIN
;
1085 rule
= mlx5e_tc_offload_fdb_rules(esw
, flow
, spec
, slow_attr
);
1087 flow_flag_set(flow
, SLOW
);
1093 mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch
*esw
,
1094 struct mlx5e_tc_flow
*flow
,
1095 struct mlx5_esw_flow_attr
*slow_attr
)
1097 memcpy(slow_attr
, flow
->esw_attr
, sizeof(*slow_attr
));
1098 slow_attr
->action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1099 slow_attr
->split_count
= 0;
1100 slow_attr
->dest_chain
= FDB_TC_SLOW_PATH_CHAIN
;
1101 mlx5e_tc_unoffload_fdb_rules(esw
, flow
, slow_attr
);
1102 flow_flag_clear(flow
, SLOW
);
1105 /* Caller must obtain uplink_priv->unready_flows_lock mutex before calling this
1108 static void unready_flow_add(struct mlx5e_tc_flow
*flow
,
1109 struct list_head
*unready_flows
)
1111 flow_flag_set(flow
, NOT_READY
);
1112 list_add_tail(&flow
->unready
, unready_flows
);
1115 /* Caller must obtain uplink_priv->unready_flows_lock mutex before calling this
1118 static void unready_flow_del(struct mlx5e_tc_flow
*flow
)
1120 list_del(&flow
->unready
);
1121 flow_flag_clear(flow
, NOT_READY
);
1124 static void add_unready_flow(struct mlx5e_tc_flow
*flow
)
1126 struct mlx5_rep_uplink_priv
*uplink_priv
;
1127 struct mlx5e_rep_priv
*rpriv
;
1128 struct mlx5_eswitch
*esw
;
1130 esw
= flow
->priv
->mdev
->priv
.eswitch
;
1131 rpriv
= mlx5_eswitch_get_uplink_priv(esw
, REP_ETH
);
1132 uplink_priv
= &rpriv
->uplink_priv
;
1134 mutex_lock(&uplink_priv
->unready_flows_lock
);
1135 unready_flow_add(flow
, &uplink_priv
->unready_flows
);
1136 mutex_unlock(&uplink_priv
->unready_flows_lock
);
1139 static void remove_unready_flow(struct mlx5e_tc_flow
*flow
)
1141 struct mlx5_rep_uplink_priv
*uplink_priv
;
1142 struct mlx5e_rep_priv
*rpriv
;
1143 struct mlx5_eswitch
*esw
;
1145 esw
= flow
->priv
->mdev
->priv
.eswitch
;
1146 rpriv
= mlx5_eswitch_get_uplink_priv(esw
, REP_ETH
);
1147 uplink_priv
= &rpriv
->uplink_priv
;
1149 mutex_lock(&uplink_priv
->unready_flows_lock
);
1150 unready_flow_del(flow
);
1151 mutex_unlock(&uplink_priv
->unready_flows_lock
);
1155 mlx5e_tc_add_fdb_flow(struct mlx5e_priv
*priv
,
1156 struct mlx5e_tc_flow
*flow
,
1157 struct netlink_ext_ack
*extack
)
1159 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1160 u32 max_chain
= mlx5_eswitch_get_chain_range(esw
);
1161 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
1162 struct mlx5e_tc_flow_parse_attr
*parse_attr
= attr
->parse_attr
;
1163 u16 max_prio
= mlx5_eswitch_get_prio_range(esw
);
1164 struct net_device
*out_dev
, *encap_dev
= NULL
;
1165 struct mlx5_fc
*counter
= NULL
;
1166 struct mlx5e_rep_priv
*rpriv
;
1167 struct mlx5e_priv
*out_priv
;
1168 bool encap_valid
= true;
1172 if (!mlx5_eswitch_prios_supported(esw
) && attr
->prio
!= 1) {
1173 NL_SET_ERR_MSG(extack
, "E-switch priorities unsupported, upgrade FW");
1177 /* We check chain range only for tc flows.
1178 * For ft flows, we checked attr->chain was originally 0 and set it to
1179 * FDB_FT_CHAIN which is outside tc range.
1180 * See mlx5e_rep_setup_ft_cb().
1182 if (!mlx5e_is_ft_flow(flow
) && attr
->chain
> max_chain
) {
1183 NL_SET_ERR_MSG(extack
, "Requested chain is out of supported range");
1187 if (attr
->prio
> max_prio
) {
1188 NL_SET_ERR_MSG(extack
, "Requested priority is out of supported range");
1192 for (out_index
= 0; out_index
< MLX5_MAX_FLOW_FWD_VPORTS
; out_index
++) {
1195 if (!(attr
->dests
[out_index
].flags
& MLX5_ESW_DEST_ENCAP
))
1198 mirred_ifindex
= parse_attr
->mirred_ifindex
[out_index
];
1199 out_dev
= __dev_get_by_index(dev_net(priv
->netdev
),
1201 err
= mlx5e_attach_encap(priv
, flow
, out_dev
, out_index
,
1202 extack
, &encap_dev
, &encap_valid
);
1206 out_priv
= netdev_priv(encap_dev
);
1207 rpriv
= out_priv
->ppriv
;
1208 attr
->dests
[out_index
].rep
= rpriv
->rep
;
1209 attr
->dests
[out_index
].mdev
= out_priv
->mdev
;
1212 err
= mlx5_eswitch_add_vlan_action(esw
, attr
);
1216 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
) {
1217 err
= mlx5e_attach_mod_hdr(priv
, flow
, parse_attr
);
1218 kfree(parse_attr
->mod_hdr_actions
);
1223 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
1224 counter
= mlx5_fc_create(attr
->counter_dev
, true);
1225 if (IS_ERR(counter
))
1226 return PTR_ERR(counter
);
1228 attr
->counter
= counter
;
1231 /* we get here if one of the following takes place:
1232 * (1) there's no error
1233 * (2) there's an encap action and we don't have valid neigh
1236 /* continue with goto slow path rule instead */
1237 struct mlx5_esw_flow_attr slow_attr
;
1239 flow
->rule
[0] = mlx5e_tc_offload_to_slow_path(esw
, flow
, &parse_attr
->spec
, &slow_attr
);
1241 flow
->rule
[0] = mlx5e_tc_offload_fdb_rules(esw
, flow
, &parse_attr
->spec
, attr
);
1244 if (IS_ERR(flow
->rule
[0]))
1245 return PTR_ERR(flow
->rule
[0]);
1247 flow_flag_set(flow
, OFFLOADED
);
1252 static bool mlx5_flow_has_geneve_opt(struct mlx5e_tc_flow
*flow
)
1254 struct mlx5_flow_spec
*spec
= &flow
->esw_attr
->parse_attr
->spec
;
1255 void *headers_v
= MLX5_ADDR_OF(fte_match_param
,
1258 u32 geneve_tlv_opt_0_data
= MLX5_GET(fte_match_set_misc3
,
1260 geneve_tlv_option_0_data
);
1262 return !!geneve_tlv_opt_0_data
;
1265 static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv
*priv
,
1266 struct mlx5e_tc_flow
*flow
)
1268 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1269 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
1270 struct mlx5_esw_flow_attr slow_attr
;
1273 if (flow_flag_test(flow
, NOT_READY
)) {
1274 remove_unready_flow(flow
);
1275 kvfree(attr
->parse_attr
);
1279 if (mlx5e_is_offloaded_flow(flow
)) {
1280 if (flow_flag_test(flow
, SLOW
))
1281 mlx5e_tc_unoffload_from_slow_path(esw
, flow
, &slow_attr
);
1283 mlx5e_tc_unoffload_fdb_rules(esw
, flow
, attr
);
1286 if (mlx5_flow_has_geneve_opt(flow
))
1287 mlx5_geneve_tlv_option_del(priv
->mdev
->geneve
);
1289 mlx5_eswitch_del_vlan_action(esw
, attr
);
1291 for (out_index
= 0; out_index
< MLX5_MAX_FLOW_FWD_VPORTS
; out_index
++)
1292 if (attr
->dests
[out_index
].flags
& MLX5_ESW_DEST_ENCAP
) {
1293 mlx5e_detach_encap(priv
, flow
, out_index
);
1294 kfree(attr
->parse_attr
->tun_info
[out_index
]);
1296 kvfree(attr
->parse_attr
);
1298 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
1299 mlx5e_detach_mod_hdr(priv
, flow
);
1301 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
)
1302 mlx5_fc_destroy(attr
->counter_dev
, attr
->counter
);
1305 void mlx5e_tc_encap_flows_add(struct mlx5e_priv
*priv
,
1306 struct mlx5e_encap_entry
*e
,
1307 struct list_head
*flow_list
)
1309 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1310 struct mlx5_esw_flow_attr slow_attr
, *esw_attr
;
1311 struct mlx5_flow_handle
*rule
;
1312 struct mlx5_flow_spec
*spec
;
1313 struct mlx5e_tc_flow
*flow
;
1316 e
->pkt_reformat
= mlx5_packet_reformat_alloc(priv
->mdev
,
1318 e
->encap_size
, e
->encap_header
,
1319 MLX5_FLOW_NAMESPACE_FDB
);
1320 if (IS_ERR(e
->pkt_reformat
)) {
1321 mlx5_core_warn(priv
->mdev
, "Failed to offload cached encapsulation header, %lu\n",
1322 PTR_ERR(e
->pkt_reformat
));
1325 e
->flags
|= MLX5_ENCAP_ENTRY_VALID
;
1326 mlx5e_rep_queue_neigh_stats_work(priv
);
1328 list_for_each_entry(flow
, flow_list
, tmp_list
) {
1329 bool all_flow_encaps_valid
= true;
1332 if (!mlx5e_is_offloaded_flow(flow
))
1334 esw_attr
= flow
->esw_attr
;
1335 spec
= &esw_attr
->parse_attr
->spec
;
1337 esw_attr
->dests
[flow
->tmp_efi_index
].pkt_reformat
= e
->pkt_reformat
;
1338 esw_attr
->dests
[flow
->tmp_efi_index
].flags
|= MLX5_ESW_DEST_ENCAP_VALID
;
1339 /* Flow can be associated with multiple encap entries.
1340 * Before offloading the flow verify that all of them have
1341 * a valid neighbour.
1343 for (i
= 0; i
< MLX5_MAX_FLOW_FWD_VPORTS
; i
++) {
1344 if (!(esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_ENCAP
))
1346 if (!(esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_ENCAP_VALID
)) {
1347 all_flow_encaps_valid
= false;
1351 /* Do not offload flows with unresolved neighbors */
1352 if (!all_flow_encaps_valid
)
1354 /* update from slow path rule to encap rule */
1355 rule
= mlx5e_tc_offload_fdb_rules(esw
, flow
, spec
, esw_attr
);
1357 err
= PTR_ERR(rule
);
1358 mlx5_core_warn(priv
->mdev
, "Failed to update cached encapsulation flow, %d\n",
1363 mlx5e_tc_unoffload_from_slow_path(esw
, flow
, &slow_attr
);
1364 flow
->rule
[0] = rule
;
1365 /* was unset when slow path rule removed */
1366 flow_flag_set(flow
, OFFLOADED
);
1370 void mlx5e_tc_encap_flows_del(struct mlx5e_priv
*priv
,
1371 struct mlx5e_encap_entry
*e
,
1372 struct list_head
*flow_list
)
1374 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1375 struct mlx5_esw_flow_attr slow_attr
;
1376 struct mlx5_flow_handle
*rule
;
1377 struct mlx5_flow_spec
*spec
;
1378 struct mlx5e_tc_flow
*flow
;
1381 list_for_each_entry(flow
, flow_list
, tmp_list
) {
1382 if (!mlx5e_is_offloaded_flow(flow
))
1384 spec
= &flow
->esw_attr
->parse_attr
->spec
;
1386 /* update from encap rule to slow path rule */
1387 rule
= mlx5e_tc_offload_to_slow_path(esw
, flow
, spec
, &slow_attr
);
1388 /* mark the flow's encap dest as non-valid */
1389 flow
->esw_attr
->dests
[flow
->tmp_efi_index
].flags
&= ~MLX5_ESW_DEST_ENCAP_VALID
;
1392 err
= PTR_ERR(rule
);
1393 mlx5_core_warn(priv
->mdev
, "Failed to update slow path (encap) flow, %d\n",
1398 mlx5e_tc_unoffload_fdb_rules(esw
, flow
, flow
->esw_attr
);
1399 flow
->rule
[0] = rule
;
1400 /* was unset when fast path rule removed */
1401 flow_flag_set(flow
, OFFLOADED
);
1404 /* we know that the encap is valid */
1405 e
->flags
&= ~MLX5_ENCAP_ENTRY_VALID
;
1406 mlx5_packet_reformat_dealloc(priv
->mdev
, e
->pkt_reformat
);
1409 static struct mlx5_fc
*mlx5e_tc_get_counter(struct mlx5e_tc_flow
*flow
)
1411 if (mlx5e_is_eswitch_flow(flow
))
1412 return flow
->esw_attr
->counter
;
1414 return flow
->nic_attr
->counter
;
1417 /* Takes reference to all flows attached to encap and adds the flows to
1418 * flow_list using 'tmp_list' list_head in mlx5e_tc_flow.
1420 void mlx5e_take_all_encap_flows(struct mlx5e_encap_entry
*e
, struct list_head
*flow_list
)
1422 struct encap_flow_item
*efi
;
1423 struct mlx5e_tc_flow
*flow
;
1425 list_for_each_entry(efi
, &e
->flows
, list
) {
1426 flow
= container_of(efi
, struct mlx5e_tc_flow
, encaps
[efi
->index
]);
1427 if (IS_ERR(mlx5e_flow_get(flow
)))
1429 wait_for_completion(&flow
->init_done
);
1431 flow
->tmp_efi_index
= efi
->index
;
1432 list_add(&flow
->tmp_list
, flow_list
);
1436 /* Iterate over tmp_list of flows attached to flow_list head. */
1437 void mlx5e_put_encap_flow_list(struct mlx5e_priv
*priv
, struct list_head
*flow_list
)
1439 struct mlx5e_tc_flow
*flow
, *tmp
;
1441 list_for_each_entry_safe(flow
, tmp
, flow_list
, tmp_list
)
1442 mlx5e_flow_put(priv
, flow
);
1445 static struct mlx5e_encap_entry
*
1446 mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry
*nhe
,
1447 struct mlx5e_encap_entry
*e
)
1449 struct mlx5e_encap_entry
*next
= NULL
;
1454 /* find encap with non-zero reference counter value */
1456 list_next_or_null_rcu(&nhe
->encap_list
,
1458 struct mlx5e_encap_entry
,
1460 list_first_or_null_rcu(&nhe
->encap_list
,
1461 struct mlx5e_encap_entry
,
1464 next
= list_next_or_null_rcu(&nhe
->encap_list
,
1466 struct mlx5e_encap_entry
,
1468 if (mlx5e_encap_take(next
))
1473 /* release starting encap */
1475 mlx5e_encap_put(netdev_priv(e
->out_dev
), e
);
1479 /* wait for encap to be fully initialized */
1480 wait_for_completion(&next
->res_ready
);
1481 /* continue searching if encap entry is not in valid state after completion */
1482 if (!(next
->flags
& MLX5_ENCAP_ENTRY_VALID
)) {
1490 void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry
*nhe
)
1492 struct mlx5e_neigh
*m_neigh
= &nhe
->m_neigh
;
1493 struct mlx5e_encap_entry
*e
= NULL
;
1494 struct mlx5e_tc_flow
*flow
;
1495 struct mlx5_fc
*counter
;
1496 struct neigh_table
*tbl
;
1497 bool neigh_used
= false;
1498 struct neighbour
*n
;
1501 if (m_neigh
->family
== AF_INET
)
1503 #if IS_ENABLED(CONFIG_IPV6)
1504 else if (m_neigh
->family
== AF_INET6
)
1505 tbl
= ipv6_stub
->nd_tbl
;
1510 /* mlx5e_get_next_valid_encap() releases previous encap before returning
1513 while ((e
= mlx5e_get_next_valid_encap(nhe
, e
)) != NULL
) {
1514 struct mlx5e_priv
*priv
= netdev_priv(e
->out_dev
);
1515 struct encap_flow_item
*efi
, *tmp
;
1516 struct mlx5_eswitch
*esw
;
1517 LIST_HEAD(flow_list
);
1519 esw
= priv
->mdev
->priv
.eswitch
;
1520 mutex_lock(&esw
->offloads
.encap_tbl_lock
);
1521 list_for_each_entry_safe(efi
, tmp
, &e
->flows
, list
) {
1522 flow
= container_of(efi
, struct mlx5e_tc_flow
,
1523 encaps
[efi
->index
]);
1524 if (IS_ERR(mlx5e_flow_get(flow
)))
1526 list_add(&flow
->tmp_list
, &flow_list
);
1528 if (mlx5e_is_offloaded_flow(flow
)) {
1529 counter
= mlx5e_tc_get_counter(flow
);
1530 lastuse
= mlx5_fc_query_lastuse(counter
);
1531 if (time_after((unsigned long)lastuse
, nhe
->reported_lastuse
)) {
1537 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
1539 mlx5e_put_encap_flow_list(priv
, &flow_list
);
1541 /* release current encap before breaking the loop */
1542 mlx5e_encap_put(priv
, e
);
1547 trace_mlx5e_tc_update_neigh_used_value(nhe
, neigh_used
);
1550 nhe
->reported_lastuse
= jiffies
;
1552 /* find the relevant neigh according to the cached device and
1555 n
= neigh_lookup(tbl
, &m_neigh
->dst_ip
, m_neigh
->dev
);
1559 neigh_event_send(n
, NULL
);
1564 static void mlx5e_encap_dealloc(struct mlx5e_priv
*priv
, struct mlx5e_encap_entry
*e
)
1566 WARN_ON(!list_empty(&e
->flows
));
1568 if (e
->compl_result
> 0) {
1569 mlx5e_rep_encap_entry_detach(netdev_priv(e
->out_dev
), e
);
1571 if (e
->flags
& MLX5_ENCAP_ENTRY_VALID
)
1572 mlx5_packet_reformat_dealloc(priv
->mdev
, e
->pkt_reformat
);
1576 kfree(e
->encap_header
);
1580 void mlx5e_encap_put(struct mlx5e_priv
*priv
, struct mlx5e_encap_entry
*e
)
1582 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1584 if (!refcount_dec_and_mutex_lock(&e
->refcnt
, &esw
->offloads
.encap_tbl_lock
))
1586 hash_del_rcu(&e
->encap_hlist
);
1587 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
1589 mlx5e_encap_dealloc(priv
, e
);
1592 static void mlx5e_detach_encap(struct mlx5e_priv
*priv
,
1593 struct mlx5e_tc_flow
*flow
, int out_index
)
1595 struct mlx5e_encap_entry
*e
= flow
->encaps
[out_index
].e
;
1596 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1598 /* flow wasn't fully initialized */
1602 mutex_lock(&esw
->offloads
.encap_tbl_lock
);
1603 list_del(&flow
->encaps
[out_index
].list
);
1604 flow
->encaps
[out_index
].e
= NULL
;
1605 if (!refcount_dec_and_test(&e
->refcnt
)) {
1606 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
1609 hash_del_rcu(&e
->encap_hlist
);
1610 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
1612 mlx5e_encap_dealloc(priv
, e
);
1615 static void __mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow
*flow
)
1617 struct mlx5_eswitch
*esw
= flow
->priv
->mdev
->priv
.eswitch
;
1619 if (!flow_flag_test(flow
, ESWITCH
) ||
1620 !flow_flag_test(flow
, DUP
))
1623 mutex_lock(&esw
->offloads
.peer_mutex
);
1624 list_del(&flow
->peer
);
1625 mutex_unlock(&esw
->offloads
.peer_mutex
);
1627 flow_flag_clear(flow
, DUP
);
1629 mlx5e_tc_del_fdb_flow(flow
->peer_flow
->priv
, flow
->peer_flow
);
1630 kvfree(flow
->peer_flow
);
1631 flow
->peer_flow
= NULL
;
1634 static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow
*flow
)
1636 struct mlx5_core_dev
*dev
= flow
->priv
->mdev
;
1637 struct mlx5_devcom
*devcom
= dev
->priv
.devcom
;
1638 struct mlx5_eswitch
*peer_esw
;
1640 peer_esw
= mlx5_devcom_get_peer_data(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
1644 __mlx5e_tc_del_fdb_peer_flow(flow
);
1645 mlx5_devcom_release_peer_data(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
1648 static void mlx5e_tc_del_flow(struct mlx5e_priv
*priv
,
1649 struct mlx5e_tc_flow
*flow
)
1651 if (mlx5e_is_eswitch_flow(flow
)) {
1652 mlx5e_tc_del_fdb_peer_flow(flow
);
1653 mlx5e_tc_del_fdb_flow(priv
, flow
);
1655 mlx5e_tc_del_nic_flow(priv
, flow
);
1660 static int parse_tunnel_attr(struct mlx5e_priv
*priv
,
1661 struct mlx5_flow_spec
*spec
,
1662 struct flow_cls_offload
*f
,
1663 struct net_device
*filter_dev
, u8
*match_level
)
1665 struct netlink_ext_ack
*extack
= f
->common
.extack
;
1666 void *headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1668 void *headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1670 struct flow_rule
*rule
= flow_cls_offload_flow_rule(f
);
1673 err
= mlx5e_tc_tun_parse(filter_dev
, priv
, spec
, f
,
1674 headers_c
, headers_v
, match_level
);
1676 NL_SET_ERR_MSG_MOD(extack
,
1677 "failed to parse tunnel attributes");
1681 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ENC_CONTROL
)) {
1682 struct flow_match_control match
;
1685 flow_rule_match_enc_control(rule
, &match
);
1686 addr_type
= match
.key
->addr_type
;
1688 /* For tunnel addr_type used same key id`s as for non-tunnel */
1689 if (addr_type
== FLOW_DISSECTOR_KEY_IPV4_ADDRS
) {
1690 struct flow_match_ipv4_addrs match
;
1692 flow_rule_match_enc_ipv4_addrs(rule
, &match
);
1693 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1694 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
,
1695 ntohl(match
.mask
->src
));
1696 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1697 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
,
1698 ntohl(match
.key
->src
));
1700 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1701 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
,
1702 ntohl(match
.mask
->dst
));
1703 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1704 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
,
1705 ntohl(match
.key
->dst
));
1707 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
,
1709 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ethertype
,
1711 } else if (addr_type
== FLOW_DISSECTOR_KEY_IPV6_ADDRS
) {
1712 struct flow_match_ipv6_addrs match
;
1714 flow_rule_match_enc_ipv6_addrs(rule
, &match
);
1715 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1716 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
1717 &match
.mask
->src
, MLX5_FLD_SZ_BYTES(ipv6_layout
,
1719 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1720 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
1721 &match
.key
->src
, MLX5_FLD_SZ_BYTES(ipv6_layout
,
1724 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1725 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
1726 &match
.mask
->dst
, MLX5_FLD_SZ_BYTES(ipv6_layout
,
1728 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1729 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
1730 &match
.key
->dst
, MLX5_FLD_SZ_BYTES(ipv6_layout
,
1733 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
,
1735 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ethertype
,
1740 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ENC_IP
)) {
1741 struct flow_match_ip match
;
1743 flow_rule_match_enc_ip(rule
, &match
);
1744 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_ecn
,
1745 match
.mask
->tos
& 0x3);
1746 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_ecn
,
1747 match
.key
->tos
& 0x3);
1749 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_dscp
,
1750 match
.mask
->tos
>> 2);
1751 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_dscp
,
1752 match
.key
->tos
>> 2);
1754 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ttl_hoplimit
,
1756 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ttl_hoplimit
,
1759 if (match
.mask
->ttl
&&
1760 !MLX5_CAP_ESW_FLOWTABLE_FDB
1762 ft_field_support
.outer_ipv4_ttl
)) {
1763 NL_SET_ERR_MSG_MOD(extack
,
1764 "Matching on TTL is not supported");
1770 /* Enforce DMAC when offloading incoming tunneled flows.
1771 * Flow counters require a match on the DMAC.
1773 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, dmac_47_16
);
1774 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, dmac_15_0
);
1775 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1776 dmac_47_16
), priv
->netdev
->dev_addr
);
1778 /* let software handle IP fragments */
1779 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, frag
, 1);
1780 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, frag
, 0);
1785 static void *get_match_headers_criteria(u32 flags
,
1786 struct mlx5_flow_spec
*spec
)
1788 return (flags
& MLX5_FLOW_CONTEXT_ACTION_DECAP
) ?
1789 MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1791 MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1795 static void *get_match_headers_value(u32 flags
,
1796 struct mlx5_flow_spec
*spec
)
1798 return (flags
& MLX5_FLOW_CONTEXT_ACTION_DECAP
) ?
1799 MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1801 MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1805 static int __parse_cls_flower(struct mlx5e_priv
*priv
,
1806 struct mlx5_flow_spec
*spec
,
1807 struct flow_cls_offload
*f
,
1808 struct net_device
*filter_dev
,
1809 u8
*inner_match_level
, u8
*outer_match_level
)
1811 struct netlink_ext_ack
*extack
= f
->common
.extack
;
1812 void *headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1814 void *headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1816 void *misc_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1818 void *misc_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1820 struct flow_rule
*rule
= flow_cls_offload_flow_rule(f
);
1821 struct flow_dissector
*dissector
= rule
->match
.dissector
;
1826 match_level
= outer_match_level
;
1828 if (dissector
->used_keys
&
1829 ~(BIT(FLOW_DISSECTOR_KEY_META
) |
1830 BIT(FLOW_DISSECTOR_KEY_CONTROL
) |
1831 BIT(FLOW_DISSECTOR_KEY_BASIC
) |
1832 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS
) |
1833 BIT(FLOW_DISSECTOR_KEY_VLAN
) |
1834 BIT(FLOW_DISSECTOR_KEY_CVLAN
) |
1835 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS
) |
1836 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS
) |
1837 BIT(FLOW_DISSECTOR_KEY_PORTS
) |
1838 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID
) |
1839 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
) |
1840 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
) |
1841 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS
) |
1842 BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL
) |
1843 BIT(FLOW_DISSECTOR_KEY_TCP
) |
1844 BIT(FLOW_DISSECTOR_KEY_IP
) |
1845 BIT(FLOW_DISSECTOR_KEY_ENC_IP
) |
1846 BIT(FLOW_DISSECTOR_KEY_ENC_OPTS
))) {
1847 NL_SET_ERR_MSG_MOD(extack
, "Unsupported key");
1848 netdev_warn(priv
->netdev
, "Unsupported key used: 0x%x\n",
1849 dissector
->used_keys
);
1853 if (mlx5e_get_tc_tun(filter_dev
)) {
1854 if (parse_tunnel_attr(priv
, spec
, f
, filter_dev
,
1858 /* At this point, header pointers should point to the inner
1859 * headers, outer header were already set by parse_tunnel_attr
1861 match_level
= inner_match_level
;
1862 headers_c
= get_match_headers_criteria(MLX5_FLOW_CONTEXT_ACTION_DECAP
,
1864 headers_v
= get_match_headers_value(MLX5_FLOW_CONTEXT_ACTION_DECAP
,
1868 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_BASIC
)) {
1869 struct flow_match_basic match
;
1871 flow_rule_match_basic(rule
, &match
);
1872 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ethertype
,
1873 ntohs(match
.mask
->n_proto
));
1874 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ethertype
,
1875 ntohs(match
.key
->n_proto
));
1877 if (match
.mask
->n_proto
)
1878 *match_level
= MLX5_MATCH_L2
;
1880 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_VLAN
) ||
1881 is_vlan_dev(filter_dev
)) {
1882 struct flow_dissector_key_vlan filter_dev_mask
;
1883 struct flow_dissector_key_vlan filter_dev_key
;
1884 struct flow_match_vlan match
;
1886 if (is_vlan_dev(filter_dev
)) {
1887 match
.key
= &filter_dev_key
;
1888 match
.key
->vlan_id
= vlan_dev_vlan_id(filter_dev
);
1889 match
.key
->vlan_tpid
= vlan_dev_vlan_proto(filter_dev
);
1890 match
.key
->vlan_priority
= 0;
1891 match
.mask
= &filter_dev_mask
;
1892 memset(match
.mask
, 0xff, sizeof(*match
.mask
));
1893 match
.mask
->vlan_priority
= 0;
1895 flow_rule_match_vlan(rule
, &match
);
1897 if (match
.mask
->vlan_id
||
1898 match
.mask
->vlan_priority
||
1899 match
.mask
->vlan_tpid
) {
1900 if (match
.key
->vlan_tpid
== htons(ETH_P_8021AD
)) {
1901 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1903 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1906 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1908 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1912 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, first_vid
,
1913 match
.mask
->vlan_id
);
1914 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, first_vid
,
1915 match
.key
->vlan_id
);
1917 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, first_prio
,
1918 match
.mask
->vlan_priority
);
1919 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, first_prio
,
1920 match
.key
->vlan_priority
);
1922 *match_level
= MLX5_MATCH_L2
;
1924 } else if (*match_level
!= MLX5_MATCH_NONE
) {
1925 /* cvlan_tag enabled in match criteria and
1926 * disabled in match value means both S & C tags
1927 * don't exist (untagged of both)
1929 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, cvlan_tag
, 1);
1930 *match_level
= MLX5_MATCH_L2
;
1933 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_CVLAN
)) {
1934 struct flow_match_vlan match
;
1936 flow_rule_match_cvlan(rule
, &match
);
1937 if (match
.mask
->vlan_id
||
1938 match
.mask
->vlan_priority
||
1939 match
.mask
->vlan_tpid
) {
1940 if (match
.key
->vlan_tpid
== htons(ETH_P_8021AD
)) {
1941 MLX5_SET(fte_match_set_misc
, misc_c
,
1942 outer_second_svlan_tag
, 1);
1943 MLX5_SET(fte_match_set_misc
, misc_v
,
1944 outer_second_svlan_tag
, 1);
1946 MLX5_SET(fte_match_set_misc
, misc_c
,
1947 outer_second_cvlan_tag
, 1);
1948 MLX5_SET(fte_match_set_misc
, misc_v
,
1949 outer_second_cvlan_tag
, 1);
1952 MLX5_SET(fte_match_set_misc
, misc_c
, outer_second_vid
,
1953 match
.mask
->vlan_id
);
1954 MLX5_SET(fte_match_set_misc
, misc_v
, outer_second_vid
,
1955 match
.key
->vlan_id
);
1956 MLX5_SET(fte_match_set_misc
, misc_c
, outer_second_prio
,
1957 match
.mask
->vlan_priority
);
1958 MLX5_SET(fte_match_set_misc
, misc_v
, outer_second_prio
,
1959 match
.key
->vlan_priority
);
1961 *match_level
= MLX5_MATCH_L2
;
1965 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_ETH_ADDRS
)) {
1966 struct flow_match_eth_addrs match
;
1968 flow_rule_match_eth_addrs(rule
, &match
);
1969 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1972 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1976 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1979 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1983 if (!is_zero_ether_addr(match
.mask
->src
) ||
1984 !is_zero_ether_addr(match
.mask
->dst
))
1985 *match_level
= MLX5_MATCH_L2
;
1988 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_CONTROL
)) {
1989 struct flow_match_control match
;
1991 flow_rule_match_control(rule
, &match
);
1992 addr_type
= match
.key
->addr_type
;
1994 /* the HW doesn't support frag first/later */
1995 if (match
.mask
->flags
& FLOW_DIS_FIRST_FRAG
)
1998 if (match
.mask
->flags
& FLOW_DIS_IS_FRAGMENT
) {
1999 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, frag
, 1);
2000 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, frag
,
2001 match
.key
->flags
& FLOW_DIS_IS_FRAGMENT
);
2003 /* the HW doesn't need L3 inline to match on frag=no */
2004 if (!(match
.key
->flags
& FLOW_DIS_IS_FRAGMENT
))
2005 *match_level
= MLX5_MATCH_L2
;
2006 /* *** L2 attributes parsing up to here *** */
2008 *match_level
= MLX5_MATCH_L3
;
2012 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_BASIC
)) {
2013 struct flow_match_basic match
;
2015 flow_rule_match_basic(rule
, &match
);
2016 ip_proto
= match
.key
->ip_proto
;
2018 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_protocol
,
2019 match
.mask
->ip_proto
);
2020 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_protocol
,
2021 match
.key
->ip_proto
);
2023 if (match
.mask
->ip_proto
)
2024 *match_level
= MLX5_MATCH_L3
;
2027 if (addr_type
== FLOW_DISSECTOR_KEY_IPV4_ADDRS
) {
2028 struct flow_match_ipv4_addrs match
;
2030 flow_rule_match_ipv4_addrs(rule
, &match
);
2031 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
2032 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
2033 &match
.mask
->src
, sizeof(match
.mask
->src
));
2034 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
2035 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
2036 &match
.key
->src
, sizeof(match
.key
->src
));
2037 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
2038 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
2039 &match
.mask
->dst
, sizeof(match
.mask
->dst
));
2040 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
2041 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
2042 &match
.key
->dst
, sizeof(match
.key
->dst
));
2044 if (match
.mask
->src
|| match
.mask
->dst
)
2045 *match_level
= MLX5_MATCH_L3
;
2048 if (addr_type
== FLOW_DISSECTOR_KEY_IPV6_ADDRS
) {
2049 struct flow_match_ipv6_addrs match
;
2051 flow_rule_match_ipv6_addrs(rule
, &match
);
2052 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
2053 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
2054 &match
.mask
->src
, sizeof(match
.mask
->src
));
2055 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
2056 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
2057 &match
.key
->src
, sizeof(match
.key
->src
));
2059 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
2060 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
2061 &match
.mask
->dst
, sizeof(match
.mask
->dst
));
2062 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
2063 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
2064 &match
.key
->dst
, sizeof(match
.key
->dst
));
2066 if (ipv6_addr_type(&match
.mask
->src
) != IPV6_ADDR_ANY
||
2067 ipv6_addr_type(&match
.mask
->dst
) != IPV6_ADDR_ANY
)
2068 *match_level
= MLX5_MATCH_L3
;
2071 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_IP
)) {
2072 struct flow_match_ip match
;
2074 flow_rule_match_ip(rule
, &match
);
2075 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_ecn
,
2076 match
.mask
->tos
& 0x3);
2077 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_ecn
,
2078 match
.key
->tos
& 0x3);
2080 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_dscp
,
2081 match
.mask
->tos
>> 2);
2082 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_dscp
,
2083 match
.key
->tos
>> 2);
2085 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ttl_hoplimit
,
2087 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ttl_hoplimit
,
2090 if (match
.mask
->ttl
&&
2091 !MLX5_CAP_ESW_FLOWTABLE_FDB(priv
->mdev
,
2092 ft_field_support
.outer_ipv4_ttl
)) {
2093 NL_SET_ERR_MSG_MOD(extack
,
2094 "Matching on TTL is not supported");
2098 if (match
.mask
->tos
|| match
.mask
->ttl
)
2099 *match_level
= MLX5_MATCH_L3
;
2102 /* *** L3 attributes parsing up to here *** */
2104 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_PORTS
)) {
2105 struct flow_match_ports match
;
2107 flow_rule_match_ports(rule
, &match
);
2110 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
2111 tcp_sport
, ntohs(match
.mask
->src
));
2112 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
2113 tcp_sport
, ntohs(match
.key
->src
));
2115 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
2116 tcp_dport
, ntohs(match
.mask
->dst
));
2117 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
2118 tcp_dport
, ntohs(match
.key
->dst
));
2122 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
2123 udp_sport
, ntohs(match
.mask
->src
));
2124 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
2125 udp_sport
, ntohs(match
.key
->src
));
2127 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
2128 udp_dport
, ntohs(match
.mask
->dst
));
2129 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
2130 udp_dport
, ntohs(match
.key
->dst
));
2133 NL_SET_ERR_MSG_MOD(extack
,
2134 "Only UDP and TCP transports are supported for L4 matching");
2135 netdev_err(priv
->netdev
,
2136 "Only UDP and TCP transport are supported\n");
2140 if (match
.mask
->src
|| match
.mask
->dst
)
2141 *match_level
= MLX5_MATCH_L4
;
2144 if (flow_rule_match_key(rule
, FLOW_DISSECTOR_KEY_TCP
)) {
2145 struct flow_match_tcp match
;
2147 flow_rule_match_tcp(rule
, &match
);
2148 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, tcp_flags
,
2149 ntohs(match
.mask
->flags
));
2150 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, tcp_flags
,
2151 ntohs(match
.key
->flags
));
2153 if (match
.mask
->flags
)
2154 *match_level
= MLX5_MATCH_L4
;
2160 static int parse_cls_flower(struct mlx5e_priv
*priv
,
2161 struct mlx5e_tc_flow
*flow
,
2162 struct mlx5_flow_spec
*spec
,
2163 struct flow_cls_offload
*f
,
2164 struct net_device
*filter_dev
)
2166 u8 inner_match_level
, outer_match_level
, non_tunnel_match_level
;
2167 struct netlink_ext_ack
*extack
= f
->common
.extack
;
2168 struct mlx5_core_dev
*dev
= priv
->mdev
;
2169 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
2170 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
2171 struct mlx5_eswitch_rep
*rep
;
2172 bool is_eswitch_flow
;
2175 inner_match_level
= MLX5_MATCH_NONE
;
2176 outer_match_level
= MLX5_MATCH_NONE
;
2178 err
= __parse_cls_flower(priv
, spec
, f
, filter_dev
, &inner_match_level
,
2179 &outer_match_level
);
2180 non_tunnel_match_level
= (inner_match_level
== MLX5_MATCH_NONE
) ?
2181 outer_match_level
: inner_match_level
;
2183 is_eswitch_flow
= mlx5e_is_eswitch_flow(flow
);
2184 if (!err
&& is_eswitch_flow
) {
2186 if (rep
->vport
!= MLX5_VPORT_UPLINK
&&
2187 (esw
->offloads
.inline_mode
!= MLX5_INLINE_MODE_NONE
&&
2188 esw
->offloads
.inline_mode
< non_tunnel_match_level
)) {
2189 NL_SET_ERR_MSG_MOD(extack
,
2190 "Flow is not offloaded due to min inline setting");
2191 netdev_warn(priv
->netdev
,
2192 "Flow is not offloaded due to min inline setting, required %d actual %d\n",
2193 non_tunnel_match_level
, esw
->offloads
.inline_mode
);
2198 if (is_eswitch_flow
) {
2199 flow
->esw_attr
->inner_match_level
= inner_match_level
;
2200 flow
->esw_attr
->outer_match_level
= outer_match_level
;
2202 flow
->nic_attr
->match_level
= non_tunnel_match_level
;
2208 struct pedit_headers
{
2210 struct vlan_hdr vlan
;
2217 struct pedit_headers_action
{
2218 struct pedit_headers vals
;
2219 struct pedit_headers masks
;
2223 static int pedit_header_offsets
[] = {
2224 [FLOW_ACT_MANGLE_HDR_TYPE_ETH
] = offsetof(struct pedit_headers
, eth
),
2225 [FLOW_ACT_MANGLE_HDR_TYPE_IP4
] = offsetof(struct pedit_headers
, ip4
),
2226 [FLOW_ACT_MANGLE_HDR_TYPE_IP6
] = offsetof(struct pedit_headers
, ip6
),
2227 [FLOW_ACT_MANGLE_HDR_TYPE_TCP
] = offsetof(struct pedit_headers
, tcp
),
2228 [FLOW_ACT_MANGLE_HDR_TYPE_UDP
] = offsetof(struct pedit_headers
, udp
),
2231 #define pedit_header(_ph, _htype) ((void *)(_ph) + pedit_header_offsets[_htype])
2233 static int set_pedit_val(u8 hdr_type
, u32 mask
, u32 val
, u32 offset
,
2234 struct pedit_headers_action
*hdrs
)
2236 u32
*curr_pmask
, *curr_pval
;
2238 curr_pmask
= (u32
*)(pedit_header(&hdrs
->masks
, hdr_type
) + offset
);
2239 curr_pval
= (u32
*)(pedit_header(&hdrs
->vals
, hdr_type
) + offset
);
2241 if (*curr_pmask
& mask
) /* disallow acting twice on the same location */
2244 *curr_pmask
|= mask
;
2245 *curr_pval
|= (val
& mask
);
2253 struct mlx5_fields
{
2261 #define OFFLOAD(fw_field, field_bsize, field_mask, field, off, match_field) \
2262 {MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, field_bsize, field_mask, \
2263 offsetof(struct pedit_headers, field) + (off), \
2264 MLX5_BYTE_OFF(fte_match_set_lyr_2_4, match_field)}
2266 /* masked values are the same and there are no rewrites that do not have a
2269 #define SAME_VAL_MASK(type, valp, maskp, matchvalp, matchmaskp) ({ \
2270 type matchmaskx = *(type *)(matchmaskp); \
2271 type matchvalx = *(type *)(matchvalp); \
2272 type maskx = *(type *)(maskp); \
2273 type valx = *(type *)(valp); \
2275 (valx & maskx) == (matchvalx & matchmaskx) && !(maskx & (maskx ^ \
2279 static bool cmp_val_mask(void *valp
, void *maskp
, void *matchvalp
,
2280 void *matchmaskp
, u8 bsize
)
2286 same
= SAME_VAL_MASK(u8
, valp
, maskp
, matchvalp
, matchmaskp
);
2289 same
= SAME_VAL_MASK(u16
, valp
, maskp
, matchvalp
, matchmaskp
);
2292 same
= SAME_VAL_MASK(u32
, valp
, maskp
, matchvalp
, matchmaskp
);
2299 static struct mlx5_fields fields
[] = {
2300 OFFLOAD(DMAC_47_16
, 32, U32_MAX
, eth
.h_dest
[0], 0, dmac_47_16
),
2301 OFFLOAD(DMAC_15_0
, 16, U16_MAX
, eth
.h_dest
[4], 0, dmac_15_0
),
2302 OFFLOAD(SMAC_47_16
, 32, U32_MAX
, eth
.h_source
[0], 0, smac_47_16
),
2303 OFFLOAD(SMAC_15_0
, 16, U16_MAX
, eth
.h_source
[4], 0, smac_15_0
),
2304 OFFLOAD(ETHERTYPE
, 16, U16_MAX
, eth
.h_proto
, 0, ethertype
),
2305 OFFLOAD(FIRST_VID
, 16, U16_MAX
, vlan
.h_vlan_TCI
, 0, first_vid
),
2307 OFFLOAD(IP_DSCP
, 8, 0xfc, ip4
.tos
, 0, ip_dscp
),
2308 OFFLOAD(IP_TTL
, 8, U8_MAX
, ip4
.ttl
, 0, ttl_hoplimit
),
2309 OFFLOAD(SIPV4
, 32, U32_MAX
, ip4
.saddr
, 0, src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
2310 OFFLOAD(DIPV4
, 32, U32_MAX
, ip4
.daddr
, 0, dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
2312 OFFLOAD(SIPV6_127_96
, 32, U32_MAX
, ip6
.saddr
.s6_addr32
[0], 0,
2313 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
[0]),
2314 OFFLOAD(SIPV6_95_64
, 32, U32_MAX
, ip6
.saddr
.s6_addr32
[1], 0,
2315 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
[4]),
2316 OFFLOAD(SIPV6_63_32
, 32, U32_MAX
, ip6
.saddr
.s6_addr32
[2], 0,
2317 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
[8]),
2318 OFFLOAD(SIPV6_31_0
, 32, U32_MAX
, ip6
.saddr
.s6_addr32
[3], 0,
2319 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
[12]),
2320 OFFLOAD(DIPV6_127_96
, 32, U32_MAX
, ip6
.daddr
.s6_addr32
[0], 0,
2321 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
[0]),
2322 OFFLOAD(DIPV6_95_64
, 32, U32_MAX
, ip6
.daddr
.s6_addr32
[1], 0,
2323 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
[4]),
2324 OFFLOAD(DIPV6_63_32
, 32, U32_MAX
, ip6
.daddr
.s6_addr32
[2], 0,
2325 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
[8]),
2326 OFFLOAD(DIPV6_31_0
, 32, U32_MAX
, ip6
.daddr
.s6_addr32
[3], 0,
2327 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
[12]),
2328 OFFLOAD(IPV6_HOPLIMIT
, 8, U8_MAX
, ip6
.hop_limit
, 0, ttl_hoplimit
),
2330 OFFLOAD(TCP_SPORT
, 16, U16_MAX
, tcp
.source
, 0, tcp_sport
),
2331 OFFLOAD(TCP_DPORT
, 16, U16_MAX
, tcp
.dest
, 0, tcp_dport
),
2332 /* in linux iphdr tcp_flags is 8 bits long */
2333 OFFLOAD(TCP_FLAGS
, 8, U8_MAX
, tcp
.ack_seq
, 5, tcp_flags
),
2335 OFFLOAD(UDP_SPORT
, 16, U16_MAX
, udp
.source
, 0, udp_sport
),
2336 OFFLOAD(UDP_DPORT
, 16, U16_MAX
, udp
.dest
, 0, udp_dport
),
2339 /* On input attr->max_mod_hdr_actions tells how many HW actions can be parsed at
2340 * max from the SW pedit action. On success, attr->num_mod_hdr_actions
2341 * says how many HW actions were actually parsed.
2343 static int offload_pedit_fields(struct pedit_headers_action
*hdrs
,
2344 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2346 struct netlink_ext_ack
*extack
)
2348 struct pedit_headers
*set_masks
, *add_masks
, *set_vals
, *add_vals
;
2349 int i
, action_size
, nactions
, max_actions
, first
, last
, next_z
;
2350 void *headers_c
, *headers_v
, *action
, *vals_p
;
2351 u32
*s_masks_p
, *a_masks_p
, s_mask
, a_mask
;
2352 struct mlx5_fields
*f
;
2358 headers_c
= get_match_headers_criteria(*action_flags
, &parse_attr
->spec
);
2359 headers_v
= get_match_headers_value(*action_flags
, &parse_attr
->spec
);
2361 set_masks
= &hdrs
[0].masks
;
2362 add_masks
= &hdrs
[1].masks
;
2363 set_vals
= &hdrs
[0].vals
;
2364 add_vals
= &hdrs
[1].vals
;
2366 action_size
= MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto
);
2367 action
= parse_attr
->mod_hdr_actions
+
2368 parse_attr
->num_mod_hdr_actions
* action_size
;
2370 max_actions
= parse_attr
->max_mod_hdr_actions
;
2371 nactions
= parse_attr
->num_mod_hdr_actions
;
2373 for (i
= 0; i
< ARRAY_SIZE(fields
); i
++) {
2377 /* avoid seeing bits set from previous iterations */
2381 s_masks_p
= (void *)set_masks
+ f
->offset
;
2382 a_masks_p
= (void *)add_masks
+ f
->offset
;
2384 s_mask
= *s_masks_p
& f
->field_mask
;
2385 a_mask
= *a_masks_p
& f
->field_mask
;
2387 if (!s_mask
&& !a_mask
) /* nothing to offload here */
2390 if (s_mask
&& a_mask
) {
2391 NL_SET_ERR_MSG_MOD(extack
,
2392 "can't set and add to the same HW field");
2393 printk(KERN_WARNING
"mlx5: can't set and add to the same HW field (%x)\n", f
->field
);
2397 if (nactions
== max_actions
) {
2398 NL_SET_ERR_MSG_MOD(extack
,
2399 "too many pedit actions, can't offload");
2400 printk(KERN_WARNING
"mlx5: parsed %d pedit actions, can't do more\n", nactions
);
2406 void *match_mask
= headers_c
+ f
->match_offset
;
2407 void *match_val
= headers_v
+ f
->match_offset
;
2409 cmd
= MLX5_ACTION_TYPE_SET
;
2411 vals_p
= (void *)set_vals
+ f
->offset
;
2412 /* don't rewrite if we have a match on the same value */
2413 if (cmp_val_mask(vals_p
, s_masks_p
, match_val
,
2414 match_mask
, f
->field_bsize
))
2416 /* clear to denote we consumed this field */
2417 *s_masks_p
&= ~f
->field_mask
;
2419 cmd
= MLX5_ACTION_TYPE_ADD
;
2421 vals_p
= (void *)add_vals
+ f
->offset
;
2422 /* add 0 is no change */
2423 if ((*(u32
*)vals_p
& f
->field_mask
) == 0)
2425 /* clear to denote we consumed this field */
2426 *a_masks_p
&= ~f
->field_mask
;
2431 if (f
->field_bsize
== 32) {
2432 mask_be32
= *(__be32
*)&mask
;
2433 mask
= (__force
unsigned long)cpu_to_le32(be32_to_cpu(mask_be32
));
2434 } else if (f
->field_bsize
== 16) {
2435 mask_be16
= *(__be16
*)&mask
;
2436 mask
= (__force
unsigned long)cpu_to_le16(be16_to_cpu(mask_be16
));
2439 first
= find_first_bit(&mask
, f
->field_bsize
);
2440 next_z
= find_next_zero_bit(&mask
, f
->field_bsize
, first
);
2441 last
= find_last_bit(&mask
, f
->field_bsize
);
2442 if (first
< next_z
&& next_z
< last
) {
2443 NL_SET_ERR_MSG_MOD(extack
,
2444 "rewrite of few sub-fields isn't supported");
2445 printk(KERN_WARNING
"mlx5: rewrite of few sub-fields (mask %lx) isn't offloaded\n",
2450 MLX5_SET(set_action_in
, action
, action_type
, cmd
);
2451 MLX5_SET(set_action_in
, action
, field
, f
->field
);
2453 if (cmd
== MLX5_ACTION_TYPE_SET
) {
2456 /* if field is bit sized it can start not from first bit */
2457 start
= find_first_bit((unsigned long *)&f
->field_mask
,
2460 MLX5_SET(set_action_in
, action
, offset
, first
- start
);
2461 /* length is num of bits to be written, zero means length of 32 */
2462 MLX5_SET(set_action_in
, action
, length
, (last
- first
+ 1));
2465 if (f
->field_bsize
== 32)
2466 MLX5_SET(set_action_in
, action
, data
, ntohl(*(__be32
*)vals_p
) >> first
);
2467 else if (f
->field_bsize
== 16)
2468 MLX5_SET(set_action_in
, action
, data
, ntohs(*(__be16
*)vals_p
) >> first
);
2469 else if (f
->field_bsize
== 8)
2470 MLX5_SET(set_action_in
, action
, data
, *(u8
*)vals_p
>> first
);
2472 action
+= action_size
;
2476 parse_attr
->num_mod_hdr_actions
= nactions
;
2480 static int mlx5e_flow_namespace_max_modify_action(struct mlx5_core_dev
*mdev
,
2483 if (namespace == MLX5_FLOW_NAMESPACE_FDB
) /* FDB offloading */
2484 return MLX5_CAP_ESW_FLOWTABLE_FDB(mdev
, max_modify_header_actions
);
2485 else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */
2486 return MLX5_CAP_FLOWTABLE_NIC_RX(mdev
, max_modify_header_actions
);
2489 static int alloc_mod_hdr_actions(struct mlx5e_priv
*priv
,
2490 struct pedit_headers_action
*hdrs
,
2492 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
2494 int nkeys
, action_size
, max_actions
;
2496 nkeys
= hdrs
[TCA_PEDIT_KEY_EX_CMD_SET
].pedits
+
2497 hdrs
[TCA_PEDIT_KEY_EX_CMD_ADD
].pedits
;
2498 action_size
= MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto
);
2500 max_actions
= mlx5e_flow_namespace_max_modify_action(priv
->mdev
, namespace);
2501 /* can get up to crazingly 16 HW actions in 32 bits pedit SW key */
2502 max_actions
= min(max_actions
, nkeys
* 16);
2504 parse_attr
->mod_hdr_actions
= kcalloc(max_actions
, action_size
, GFP_KERNEL
);
2505 if (!parse_attr
->mod_hdr_actions
)
2508 parse_attr
->max_mod_hdr_actions
= max_actions
;
2512 static const struct pedit_headers zero_masks
= {};
2514 static int parse_tc_pedit_action(struct mlx5e_priv
*priv
,
2515 const struct flow_action_entry
*act
, int namespace,
2516 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2517 struct pedit_headers_action
*hdrs
,
2518 struct netlink_ext_ack
*extack
)
2520 u8 cmd
= (act
->id
== FLOW_ACTION_MANGLE
) ? 0 : 1;
2521 int err
= -EOPNOTSUPP
;
2522 u32 mask
, val
, offset
;
2525 htype
= act
->mangle
.htype
;
2526 err
= -EOPNOTSUPP
; /* can't be all optimistic */
2528 if (htype
== FLOW_ACT_MANGLE_UNSPEC
) {
2529 NL_SET_ERR_MSG_MOD(extack
, "legacy pedit isn't offloaded");
2533 if (!mlx5e_flow_namespace_max_modify_action(priv
->mdev
, namespace)) {
2534 NL_SET_ERR_MSG_MOD(extack
,
2535 "The pedit offload action is not supported");
2539 mask
= act
->mangle
.mask
;
2540 val
= act
->mangle
.val
;
2541 offset
= act
->mangle
.offset
;
2543 err
= set_pedit_val(htype
, ~mask
, val
, offset
, &hdrs
[cmd
]);
2554 static int alloc_tc_pedit_action(struct mlx5e_priv
*priv
, int namespace,
2555 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2556 struct pedit_headers_action
*hdrs
,
2558 struct netlink_ext_ack
*extack
)
2560 struct pedit_headers
*cmd_masks
;
2564 if (!parse_attr
->mod_hdr_actions
) {
2565 err
= alloc_mod_hdr_actions(priv
, hdrs
, namespace, parse_attr
);
2570 err
= offload_pedit_fields(hdrs
, parse_attr
, action_flags
, extack
);
2572 goto out_dealloc_parsed_actions
;
2574 for (cmd
= 0; cmd
< __PEDIT_CMD_MAX
; cmd
++) {
2575 cmd_masks
= &hdrs
[cmd
].masks
;
2576 if (memcmp(cmd_masks
, &zero_masks
, sizeof(zero_masks
))) {
2577 NL_SET_ERR_MSG_MOD(extack
,
2578 "attempt to offload an unsupported field");
2579 netdev_warn(priv
->netdev
, "attempt to offload an unsupported field (cmd %d)\n", cmd
);
2580 print_hex_dump(KERN_WARNING
, "mask: ", DUMP_PREFIX_ADDRESS
,
2581 16, 1, cmd_masks
, sizeof(zero_masks
), true);
2583 goto out_dealloc_parsed_actions
;
2589 out_dealloc_parsed_actions
:
2590 kfree(parse_attr
->mod_hdr_actions
);
2595 static bool csum_offload_supported(struct mlx5e_priv
*priv
,
2598 struct netlink_ext_ack
*extack
)
2600 u32 prot_flags
= TCA_CSUM_UPDATE_FLAG_IPV4HDR
| TCA_CSUM_UPDATE_FLAG_TCP
|
2601 TCA_CSUM_UPDATE_FLAG_UDP
;
2603 /* The HW recalcs checksums only if re-writing headers */
2604 if (!(action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)) {
2605 NL_SET_ERR_MSG_MOD(extack
,
2606 "TC csum action is only offloaded with pedit");
2607 netdev_warn(priv
->netdev
,
2608 "TC csum action is only offloaded with pedit\n");
2612 if (update_flags
& ~prot_flags
) {
2613 NL_SET_ERR_MSG_MOD(extack
,
2614 "can't offload TC csum action for some header/s");
2615 netdev_warn(priv
->netdev
,
2616 "can't offload TC csum action for some header/s - flags %#x\n",
2624 struct ip_ttl_word
{
2630 struct ipv6_hoplimit_word
{
2636 static bool is_action_keys_supported(const struct flow_action_entry
*act
)
2641 htype
= act
->mangle
.htype
;
2642 offset
= act
->mangle
.offset
;
2643 mask
= ~act
->mangle
.mask
;
2644 /* For IPv4 & IPv6 header check 4 byte word,
2645 * to determine that modified fields
2646 * are NOT ttl & hop_limit only.
2648 if (htype
== FLOW_ACT_MANGLE_HDR_TYPE_IP4
) {
2649 struct ip_ttl_word
*ttl_word
=
2650 (struct ip_ttl_word
*)&mask
;
2652 if (offset
!= offsetof(struct iphdr
, ttl
) ||
2653 ttl_word
->protocol
||
2657 } else if (htype
== FLOW_ACT_MANGLE_HDR_TYPE_IP6
) {
2658 struct ipv6_hoplimit_word
*hoplimit_word
=
2659 (struct ipv6_hoplimit_word
*)&mask
;
2661 if (offset
!= offsetof(struct ipv6hdr
, payload_len
) ||
2662 hoplimit_word
->payload_len
||
2663 hoplimit_word
->nexthdr
) {
2670 static bool modify_header_match_supported(struct mlx5_flow_spec
*spec
,
2671 struct flow_action
*flow_action
,
2673 struct netlink_ext_ack
*extack
)
2675 const struct flow_action_entry
*act
;
2676 bool modify_ip_header
;
2682 headers_v
= get_match_headers_value(actions
, spec
);
2683 ethertype
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, ethertype
);
2685 /* for non-IP we only re-write MACs, so we're okay */
2686 if (ethertype
!= ETH_P_IP
&& ethertype
!= ETH_P_IPV6
)
2689 modify_ip_header
= false;
2690 flow_action_for_each(i
, act
, flow_action
) {
2691 if (act
->id
!= FLOW_ACTION_MANGLE
&&
2692 act
->id
!= FLOW_ACTION_ADD
)
2695 if (is_action_keys_supported(act
)) {
2696 modify_ip_header
= true;
2701 ip_proto
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, ip_protocol
);
2702 if (modify_ip_header
&& ip_proto
!= IPPROTO_TCP
&&
2703 ip_proto
!= IPPROTO_UDP
&& ip_proto
!= IPPROTO_ICMP
) {
2704 NL_SET_ERR_MSG_MOD(extack
,
2705 "can't offload re-write of non TCP/UDP");
2706 pr_info("can't offload re-write of ip proto %d\n", ip_proto
);
2714 static bool actions_match_supported(struct mlx5e_priv
*priv
,
2715 struct flow_action
*flow_action
,
2716 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2717 struct mlx5e_tc_flow
*flow
,
2718 struct netlink_ext_ack
*extack
)
2722 if (mlx5e_is_eswitch_flow(flow
))
2723 actions
= flow
->esw_attr
->action
;
2725 actions
= flow
->nic_attr
->action
;
2727 if (flow_flag_test(flow
, EGRESS
) &&
2728 !((actions
& MLX5_FLOW_CONTEXT_ACTION_DECAP
) ||
2729 (actions
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
) ||
2730 (actions
& MLX5_FLOW_CONTEXT_ACTION_DROP
)))
2733 if (actions
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
2734 return modify_header_match_supported(&parse_attr
->spec
,
2735 flow_action
, actions
,
2741 static bool same_hw_devs(struct mlx5e_priv
*priv
, struct mlx5e_priv
*peer_priv
)
2743 struct mlx5_core_dev
*fmdev
, *pmdev
;
2744 u64 fsystem_guid
, psystem_guid
;
2747 pmdev
= peer_priv
->mdev
;
2749 fsystem_guid
= mlx5_query_nic_system_image_guid(fmdev
);
2750 psystem_guid
= mlx5_query_nic_system_image_guid(pmdev
);
2752 return (fsystem_guid
== psystem_guid
);
2755 static int add_vlan_rewrite_action(struct mlx5e_priv
*priv
, int namespace,
2756 const struct flow_action_entry
*act
,
2757 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2758 struct pedit_headers_action
*hdrs
,
2759 u32
*action
, struct netlink_ext_ack
*extack
)
2761 u16 mask16
= VLAN_VID_MASK
;
2762 u16 val16
= act
->vlan
.vid
& VLAN_VID_MASK
;
2763 const struct flow_action_entry pedit_act
= {
2764 .id
= FLOW_ACTION_MANGLE
,
2765 .mangle
.htype
= FLOW_ACT_MANGLE_HDR_TYPE_ETH
,
2766 .mangle
.offset
= offsetof(struct vlan_ethhdr
, h_vlan_TCI
),
2767 .mangle
.mask
= ~(u32
)be16_to_cpu(*(__be16
*)&mask16
),
2768 .mangle
.val
= (u32
)be16_to_cpu(*(__be16
*)&val16
),
2770 u8 match_prio_mask
, match_prio_val
;
2771 void *headers_c
, *headers_v
;
2774 headers_c
= get_match_headers_criteria(*action
, &parse_attr
->spec
);
2775 headers_v
= get_match_headers_value(*action
, &parse_attr
->spec
);
2777 if (!(MLX5_GET(fte_match_set_lyr_2_4
, headers_c
, cvlan_tag
) &&
2778 MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, cvlan_tag
))) {
2779 NL_SET_ERR_MSG_MOD(extack
,
2780 "VLAN rewrite action must have VLAN protocol match");
2784 match_prio_mask
= MLX5_GET(fte_match_set_lyr_2_4
, headers_c
, first_prio
);
2785 match_prio_val
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, first_prio
);
2786 if (act
->vlan
.prio
!= (match_prio_val
& match_prio_mask
)) {
2787 NL_SET_ERR_MSG_MOD(extack
,
2788 "Changing VLAN prio is not supported");
2792 err
= parse_tc_pedit_action(priv
, &pedit_act
, namespace, parse_attr
,
2794 *action
|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
2800 add_vlan_prio_tag_rewrite_action(struct mlx5e_priv
*priv
,
2801 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2802 struct pedit_headers_action
*hdrs
,
2803 u32
*action
, struct netlink_ext_ack
*extack
)
2805 const struct flow_action_entry prio_tag_act
= {
2808 MLX5_GET(fte_match_set_lyr_2_4
,
2809 get_match_headers_value(*action
,
2812 MLX5_GET(fte_match_set_lyr_2_4
,
2813 get_match_headers_criteria(*action
,
2818 return add_vlan_rewrite_action(priv
, MLX5_FLOW_NAMESPACE_FDB
,
2819 &prio_tag_act
, parse_attr
, hdrs
, action
,
2823 static int parse_tc_nic_actions(struct mlx5e_priv
*priv
,
2824 struct flow_action
*flow_action
,
2825 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2826 struct mlx5e_tc_flow
*flow
,
2827 struct netlink_ext_ack
*extack
)
2829 struct mlx5_nic_flow_attr
*attr
= flow
->nic_attr
;
2830 struct pedit_headers_action hdrs
[2] = {};
2831 const struct flow_action_entry
*act
;
2835 if (!flow_action_has_entries(flow_action
))
2838 attr
->flow_tag
= MLX5_FS_DEFAULT_FLOW_TAG
;
2840 flow_action_for_each(i
, act
, flow_action
) {
2842 case FLOW_ACTION_DROP
:
2843 action
|= MLX5_FLOW_CONTEXT_ACTION_DROP
;
2844 if (MLX5_CAP_FLOWTABLE(priv
->mdev
,
2845 flow_table_properties_nic_receive
.flow_counter
))
2846 action
|= MLX5_FLOW_CONTEXT_ACTION_COUNT
;
2848 case FLOW_ACTION_MANGLE
:
2849 case FLOW_ACTION_ADD
:
2850 err
= parse_tc_pedit_action(priv
, act
, MLX5_FLOW_NAMESPACE_KERNEL
,
2851 parse_attr
, hdrs
, extack
);
2855 action
|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
|
2856 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
2858 case FLOW_ACTION_VLAN_MANGLE
:
2859 err
= add_vlan_rewrite_action(priv
,
2860 MLX5_FLOW_NAMESPACE_KERNEL
,
2861 act
, parse_attr
, hdrs
,
2867 case FLOW_ACTION_CSUM
:
2868 if (csum_offload_supported(priv
, action
,
2874 case FLOW_ACTION_REDIRECT
: {
2875 struct net_device
*peer_dev
= act
->dev
;
2877 if (priv
->netdev
->netdev_ops
== peer_dev
->netdev_ops
&&
2878 same_hw_devs(priv
, netdev_priv(peer_dev
))) {
2879 parse_attr
->mirred_ifindex
[0] = peer_dev
->ifindex
;
2880 flow_flag_set(flow
, HAIRPIN
);
2881 action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
2882 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
2884 NL_SET_ERR_MSG_MOD(extack
,
2885 "device is not on same HW, can't offload");
2886 netdev_warn(priv
->netdev
, "device %s not on same HW, can't offload\n",
2892 case FLOW_ACTION_MARK
: {
2893 u32 mark
= act
->mark
;
2895 if (mark
& ~MLX5E_TC_FLOW_ID_MASK
) {
2896 NL_SET_ERR_MSG_MOD(extack
,
2897 "Bad flow mark - only 16 bit is supported");
2901 attr
->flow_tag
= mark
;
2902 action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
2906 NL_SET_ERR_MSG_MOD(extack
, "The offload action is not supported");
2911 if (hdrs
[TCA_PEDIT_KEY_EX_CMD_SET
].pedits
||
2912 hdrs
[TCA_PEDIT_KEY_EX_CMD_ADD
].pedits
) {
2913 err
= alloc_tc_pedit_action(priv
, MLX5_FLOW_NAMESPACE_KERNEL
,
2914 parse_attr
, hdrs
, &action
, extack
);
2917 /* in case all pedit actions are skipped, remove the MOD_HDR
2920 if (parse_attr
->num_mod_hdr_actions
== 0) {
2921 action
&= ~MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
2922 kfree(parse_attr
->mod_hdr_actions
);
2926 attr
->action
= action
;
2927 if (!actions_match_supported(priv
, flow_action
, parse_attr
, flow
, extack
))
2934 const struct ip_tunnel_key
*ip_tun_key
;
2935 struct mlx5e_tc_tunnel
*tc_tunnel
;
2938 static inline int cmp_encap_info(struct encap_key
*a
,
2939 struct encap_key
*b
)
2941 return memcmp(a
->ip_tun_key
, b
->ip_tun_key
, sizeof(*a
->ip_tun_key
)) ||
2942 a
->tc_tunnel
->tunnel_type
!= b
->tc_tunnel
->tunnel_type
;
2945 static inline int hash_encap_info(struct encap_key
*key
)
2947 return jhash(key
->ip_tun_key
, sizeof(*key
->ip_tun_key
),
2948 key
->tc_tunnel
->tunnel_type
);
2952 static bool is_merged_eswitch_dev(struct mlx5e_priv
*priv
,
2953 struct net_device
*peer_netdev
)
2955 struct mlx5e_priv
*peer_priv
;
2957 peer_priv
= netdev_priv(peer_netdev
);
2959 return (MLX5_CAP_ESW(priv
->mdev
, merged_eswitch
) &&
2960 mlx5e_eswitch_rep(priv
->netdev
) &&
2961 mlx5e_eswitch_rep(peer_netdev
) &&
2962 same_hw_devs(priv
, peer_priv
));
2967 bool mlx5e_encap_take(struct mlx5e_encap_entry
*e
)
2969 return refcount_inc_not_zero(&e
->refcnt
);
2972 static struct mlx5e_encap_entry
*
2973 mlx5e_encap_get(struct mlx5e_priv
*priv
, struct encap_key
*key
,
2976 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
2977 struct mlx5e_encap_entry
*e
;
2978 struct encap_key e_key
;
2980 hash_for_each_possible_rcu(esw
->offloads
.encap_tbl
, e
,
2981 encap_hlist
, hash_key
) {
2982 e_key
.ip_tun_key
= &e
->tun_info
->key
;
2983 e_key
.tc_tunnel
= e
->tunnel
;
2984 if (!cmp_encap_info(&e_key
, key
) &&
2985 mlx5e_encap_take(e
))
2992 static struct ip_tunnel_info
*dup_tun_info(const struct ip_tunnel_info
*tun_info
)
2994 size_t tun_size
= sizeof(*tun_info
) + tun_info
->options_len
;
2996 return kmemdup(tun_info
, tun_size
, GFP_KERNEL
);
2999 static int mlx5e_attach_encap(struct mlx5e_priv
*priv
,
3000 struct mlx5e_tc_flow
*flow
,
3001 struct net_device
*mirred_dev
,
3003 struct netlink_ext_ack
*extack
,
3004 struct net_device
**encap_dev
,
3007 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
3008 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
3009 struct mlx5e_tc_flow_parse_attr
*parse_attr
;
3010 const struct ip_tunnel_info
*tun_info
;
3011 struct encap_key key
;
3012 struct mlx5e_encap_entry
*e
;
3013 unsigned short family
;
3017 parse_attr
= attr
->parse_attr
;
3018 tun_info
= parse_attr
->tun_info
[out_index
];
3019 family
= ip_tunnel_info_af(tun_info
);
3020 key
.ip_tun_key
= &tun_info
->key
;
3021 key
.tc_tunnel
= mlx5e_get_tc_tun(mirred_dev
);
3022 if (!key
.tc_tunnel
) {
3023 NL_SET_ERR_MSG_MOD(extack
, "Unsupported tunnel");
3027 hash_key
= hash_encap_info(&key
);
3029 mutex_lock(&esw
->offloads
.encap_tbl_lock
);
3030 e
= mlx5e_encap_get(priv
, &key
, hash_key
);
3032 /* must verify if encap is valid or not */
3034 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
3035 wait_for_completion(&e
->res_ready
);
3037 /* Protect against concurrent neigh update. */
3038 mutex_lock(&esw
->offloads
.encap_tbl_lock
);
3039 if (e
->compl_result
< 0) {
3046 e
= kzalloc(sizeof(*e
), GFP_KERNEL
);
3052 refcount_set(&e
->refcnt
, 1);
3053 init_completion(&e
->res_ready
);
3055 tun_info
= dup_tun_info(tun_info
);
3060 e
->tun_info
= tun_info
;
3061 err
= mlx5e_tc_tun_init_encap_attr(mirred_dev
, priv
, e
, extack
);
3065 INIT_LIST_HEAD(&e
->flows
);
3066 hash_add_rcu(esw
->offloads
.encap_tbl
, &e
->encap_hlist
, hash_key
);
3067 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
3069 if (family
== AF_INET
)
3070 err
= mlx5e_tc_tun_create_header_ipv4(priv
, mirred_dev
, e
);
3071 else if (family
== AF_INET6
)
3072 err
= mlx5e_tc_tun_create_header_ipv6(priv
, mirred_dev
, e
);
3074 /* Protect against concurrent neigh update. */
3075 mutex_lock(&esw
->offloads
.encap_tbl_lock
);
3076 complete_all(&e
->res_ready
);
3078 e
->compl_result
= err
;
3081 e
->compl_result
= 1;
3084 flow
->encaps
[out_index
].e
= e
;
3085 list_add(&flow
->encaps
[out_index
].list
, &e
->flows
);
3086 flow
->encaps
[out_index
].index
= out_index
;
3087 *encap_dev
= e
->out_dev
;
3088 if (e
->flags
& MLX5_ENCAP_ENTRY_VALID
) {
3089 attr
->dests
[out_index
].pkt_reformat
= e
->pkt_reformat
;
3090 attr
->dests
[out_index
].flags
|= MLX5_ESW_DEST_ENCAP_VALID
;
3091 *encap_valid
= true;
3093 *encap_valid
= false;
3095 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
3100 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
3102 mlx5e_encap_put(priv
, e
);
3106 mutex_unlock(&esw
->offloads
.encap_tbl_lock
);
3112 static int parse_tc_vlan_action(struct mlx5e_priv
*priv
,
3113 const struct flow_action_entry
*act
,
3114 struct mlx5_esw_flow_attr
*attr
,
3117 u8 vlan_idx
= attr
->total_vlan
;
3119 if (vlan_idx
>= MLX5_FS_VLAN_DEPTH
)
3123 case FLOW_ACTION_VLAN_POP
:
3125 if (!mlx5_eswitch_vlan_actions_supported(priv
->mdev
,
3126 MLX5_FS_VLAN_DEPTH
))
3129 *action
|= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2
;
3131 *action
|= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
;
3134 case FLOW_ACTION_VLAN_PUSH
:
3135 attr
->vlan_vid
[vlan_idx
] = act
->vlan
.vid
;
3136 attr
->vlan_prio
[vlan_idx
] = act
->vlan
.prio
;
3137 attr
->vlan_proto
[vlan_idx
] = act
->vlan
.proto
;
3138 if (!attr
->vlan_proto
[vlan_idx
])
3139 attr
->vlan_proto
[vlan_idx
] = htons(ETH_P_8021Q
);
3142 if (!mlx5_eswitch_vlan_actions_supported(priv
->mdev
,
3143 MLX5_FS_VLAN_DEPTH
))
3146 *action
|= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2
;
3148 if (!mlx5_eswitch_vlan_actions_supported(priv
->mdev
, 1) &&
3149 (act
->vlan
.proto
!= htons(ETH_P_8021Q
) ||
3153 *action
|= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
;
3160 attr
->total_vlan
= vlan_idx
+ 1;
3165 static int add_vlan_push_action(struct mlx5e_priv
*priv
,
3166 struct mlx5_esw_flow_attr
*attr
,
3167 struct net_device
**out_dev
,
3170 struct net_device
*vlan_dev
= *out_dev
;
3171 struct flow_action_entry vlan_act
= {
3172 .id
= FLOW_ACTION_VLAN_PUSH
,
3173 .vlan
.vid
= vlan_dev_vlan_id(vlan_dev
),
3174 .vlan
.proto
= vlan_dev_vlan_proto(vlan_dev
),
3179 err
= parse_tc_vlan_action(priv
, &vlan_act
, attr
, action
);
3183 *out_dev
= dev_get_by_index_rcu(dev_net(vlan_dev
),
3184 dev_get_iflink(vlan_dev
));
3185 if (is_vlan_dev(*out_dev
))
3186 err
= add_vlan_push_action(priv
, attr
, out_dev
, action
);
3191 static int add_vlan_pop_action(struct mlx5e_priv
*priv
,
3192 struct mlx5_esw_flow_attr
*attr
,
3195 int nest_level
= attr
->parse_attr
->filter_dev
->lower_level
;
3196 struct flow_action_entry vlan_act
= {
3197 .id
= FLOW_ACTION_VLAN_POP
,
3201 while (nest_level
--) {
3202 err
= parse_tc_vlan_action(priv
, &vlan_act
, attr
, action
);
3210 bool mlx5e_is_valid_eswitch_fwd_dev(struct mlx5e_priv
*priv
,
3211 struct net_device
*out_dev
)
3213 if (is_merged_eswitch_dev(priv
, out_dev
))
3216 return mlx5e_eswitch_rep(out_dev
) &&
3217 same_hw_devs(priv
, netdev_priv(out_dev
));
3220 static int parse_tc_fdb_actions(struct mlx5e_priv
*priv
,
3221 struct flow_action
*flow_action
,
3222 struct mlx5e_tc_flow
*flow
,
3223 struct netlink_ext_ack
*extack
)
3225 struct pedit_headers_action hdrs
[2] = {};
3226 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
3227 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
3228 struct mlx5e_tc_flow_parse_attr
*parse_attr
= attr
->parse_attr
;
3229 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
3230 const struct ip_tunnel_info
*info
= NULL
;
3231 bool ft_flow
= mlx5e_is_ft_flow(flow
);
3232 const struct flow_action_entry
*act
;
3237 if (!flow_action_has_entries(flow_action
))
3240 flow_action_for_each(i
, act
, flow_action
) {
3242 case FLOW_ACTION_DROP
:
3243 action
|= MLX5_FLOW_CONTEXT_ACTION_DROP
|
3244 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
3246 case FLOW_ACTION_MANGLE
:
3247 case FLOW_ACTION_ADD
:
3248 err
= parse_tc_pedit_action(priv
, act
, MLX5_FLOW_NAMESPACE_FDB
,
3249 parse_attr
, hdrs
, extack
);
3253 action
|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
3254 attr
->split_count
= attr
->out_count
;
3256 case FLOW_ACTION_CSUM
:
3257 if (csum_offload_supported(priv
, action
,
3258 act
->csum_flags
, extack
))
3262 case FLOW_ACTION_REDIRECT
:
3263 case FLOW_ACTION_MIRRED
: {
3264 struct mlx5e_priv
*out_priv
;
3265 struct net_device
*out_dev
;
3269 /* out_dev is NULL when filters with
3270 * non-existing mirred device are replayed to
3276 if (ft_flow
&& out_dev
== priv
->netdev
) {
3277 /* Ignore forward to self rules generated
3278 * by adding both mlx5 devs to the flow table
3279 * block on a normal nft offload setup.
3284 if (attr
->out_count
>= MLX5_MAX_FLOW_FWD_VPORTS
) {
3285 NL_SET_ERR_MSG_MOD(extack
,
3286 "can't support more output ports, can't offload forwarding");
3287 pr_err("can't support more than %d output ports, can't offload forwarding\n",
3292 action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
3293 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
3295 parse_attr
->mirred_ifindex
[attr
->out_count
] =
3297 parse_attr
->tun_info
[attr
->out_count
] = dup_tun_info(info
);
3298 if (!parse_attr
->tun_info
[attr
->out_count
])
3301 attr
->dests
[attr
->out_count
].flags
|=
3302 MLX5_ESW_DEST_ENCAP
;
3304 /* attr->dests[].rep is resolved when we
3307 } else if (netdev_port_same_parent_id(priv
->netdev
, out_dev
)) {
3308 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
3309 struct net_device
*uplink_dev
= mlx5_eswitch_uplink_get_proto_dev(esw
, REP_ETH
);
3310 struct net_device
*uplink_upper
;
3314 netdev_master_upper_dev_get_rcu(uplink_dev
);
3316 netif_is_lag_master(uplink_upper
) &&
3317 uplink_upper
== out_dev
)
3318 out_dev
= uplink_dev
;
3321 if (is_vlan_dev(out_dev
)) {
3322 err
= add_vlan_push_action(priv
, attr
,
3329 if (is_vlan_dev(parse_attr
->filter_dev
)) {
3330 err
= add_vlan_pop_action(priv
, attr
,
3336 if (!mlx5e_is_valid_eswitch_fwd_dev(priv
, out_dev
)) {
3337 NL_SET_ERR_MSG_MOD(extack
,
3338 "devices are not on same switch HW, can't offload forwarding");
3339 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
3340 priv
->netdev
->name
, out_dev
->name
);
3344 out_priv
= netdev_priv(out_dev
);
3345 rpriv
= out_priv
->ppriv
;
3346 attr
->dests
[attr
->out_count
].rep
= rpriv
->rep
;
3347 attr
->dests
[attr
->out_count
].mdev
= out_priv
->mdev
;
3349 } else if (parse_attr
->filter_dev
!= priv
->netdev
) {
3350 /* All mlx5 devices are called to configure
3351 * high level device filters. Therefore, the
3352 * *attempt* to install a filter on invalid
3353 * eswitch should not trigger an explicit error
3357 NL_SET_ERR_MSG_MOD(extack
,
3358 "devices are not on same switch HW, can't offload forwarding");
3359 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
3360 priv
->netdev
->name
, out_dev
->name
);
3365 case FLOW_ACTION_TUNNEL_ENCAP
:
3373 case FLOW_ACTION_VLAN_PUSH
:
3374 case FLOW_ACTION_VLAN_POP
:
3375 if (act
->id
== FLOW_ACTION_VLAN_PUSH
&&
3376 (action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
)) {
3377 /* Replace vlan pop+push with vlan modify */
3378 action
&= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
;
3379 err
= add_vlan_rewrite_action(priv
,
3380 MLX5_FLOW_NAMESPACE_FDB
,
3381 act
, parse_attr
, hdrs
,
3384 err
= parse_tc_vlan_action(priv
, act
, attr
, &action
);
3389 attr
->split_count
= attr
->out_count
;
3391 case FLOW_ACTION_VLAN_MANGLE
:
3392 err
= add_vlan_rewrite_action(priv
,
3393 MLX5_FLOW_NAMESPACE_FDB
,
3394 act
, parse_attr
, hdrs
,
3399 attr
->split_count
= attr
->out_count
;
3401 case FLOW_ACTION_TUNNEL_DECAP
:
3402 action
|= MLX5_FLOW_CONTEXT_ACTION_DECAP
;
3404 case FLOW_ACTION_GOTO
: {
3405 u32 dest_chain
= act
->chain_index
;
3406 u32 max_chain
= mlx5_eswitch_get_chain_range(esw
);
3409 NL_SET_ERR_MSG_MOD(extack
, "Goto action is not supported");
3412 if (dest_chain
<= attr
->chain
) {
3413 NL_SET_ERR_MSG(extack
, "Goto earlier chain isn't supported");
3416 if (dest_chain
> max_chain
) {
3417 NL_SET_ERR_MSG(extack
, "Requested destination chain is out of supported range");
3420 action
|= MLX5_FLOW_CONTEXT_ACTION_COUNT
;
3421 attr
->dest_chain
= dest_chain
;
3425 NL_SET_ERR_MSG_MOD(extack
, "The offload action is not supported");
3430 if (MLX5_CAP_GEN(esw
->dev
, prio_tag_required
) &&
3431 action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
) {
3432 /* For prio tag mode, replace vlan pop with rewrite vlan prio
3435 action
&= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
;
3436 err
= add_vlan_prio_tag_rewrite_action(priv
, parse_attr
, hdrs
,
3442 if (hdrs
[TCA_PEDIT_KEY_EX_CMD_SET
].pedits
||
3443 hdrs
[TCA_PEDIT_KEY_EX_CMD_ADD
].pedits
) {
3444 err
= alloc_tc_pedit_action(priv
, MLX5_FLOW_NAMESPACE_FDB
,
3445 parse_attr
, hdrs
, &action
, extack
);
3448 /* in case all pedit actions are skipped, remove the MOD_HDR
3449 * flag. we might have set split_count either by pedit or
3450 * pop/push. if there is no pop/push either, reset it too.
3452 if (parse_attr
->num_mod_hdr_actions
== 0) {
3453 action
&= ~MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
3454 kfree(parse_attr
->mod_hdr_actions
);
3455 if (!((action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
) ||
3456 (action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
)))
3457 attr
->split_count
= 0;
3461 attr
->action
= action
;
3462 if (!actions_match_supported(priv
, flow_action
, parse_attr
, flow
, extack
))
3465 if (attr
->dest_chain
) {
3466 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
3467 NL_SET_ERR_MSG(extack
, "Mirroring goto chain rules isn't supported");
3470 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
3473 if (!(attr
->action
&
3474 (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
| MLX5_FLOW_CONTEXT_ACTION_DROP
))) {
3475 NL_SET_ERR_MSG(extack
, "Rule must have at least one forward/drop action");
3479 if (attr
->split_count
> 0 && !mlx5_esw_has_fwd_fdb(priv
->mdev
)) {
3480 NL_SET_ERR_MSG_MOD(extack
,
3481 "current firmware doesn't support split rule for port mirroring");
3482 netdev_warn_once(priv
->netdev
, "current firmware doesn't support split rule for port mirroring\n");
3489 static void get_flags(int flags
, unsigned long *flow_flags
)
3491 unsigned long __flow_flags
= 0;
3493 if (flags
& MLX5_TC_FLAG(INGRESS
))
3494 __flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_INGRESS
);
3495 if (flags
& MLX5_TC_FLAG(EGRESS
))
3496 __flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_EGRESS
);
3498 if (flags
& MLX5_TC_FLAG(ESW_OFFLOAD
))
3499 __flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_ESWITCH
);
3500 if (flags
& MLX5_TC_FLAG(NIC_OFFLOAD
))
3501 __flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_NIC
);
3502 if (flags
& MLX5_TC_FLAG(FT_OFFLOAD
))
3503 __flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_FT
);
3505 *flow_flags
= __flow_flags
;
3508 static const struct rhashtable_params tc_ht_params
= {
3509 .head_offset
= offsetof(struct mlx5e_tc_flow
, node
),
3510 .key_offset
= offsetof(struct mlx5e_tc_flow
, cookie
),
3511 .key_len
= sizeof(((struct mlx5e_tc_flow
*)0)->cookie
),
3512 .automatic_shrinking
= true,
3515 static struct rhashtable
*get_tc_ht(struct mlx5e_priv
*priv
,
3516 unsigned long flags
)
3518 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
3519 struct mlx5e_rep_priv
*uplink_rpriv
;
3521 if (flags
& MLX5_TC_FLAG(ESW_OFFLOAD
)) {
3522 uplink_rpriv
= mlx5_eswitch_get_uplink_priv(esw
, REP_ETH
);
3523 return &uplink_rpriv
->uplink_priv
.tc_ht
;
3524 } else /* NIC offload */
3525 return &priv
->fs
.tc
.ht
;
3528 static bool is_peer_flow_needed(struct mlx5e_tc_flow
*flow
)
3530 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
3531 bool is_rep_ingress
= attr
->in_rep
->vport
!= MLX5_VPORT_UPLINK
&&
3532 flow_flag_test(flow
, INGRESS
);
3533 bool act_is_encap
= !!(attr
->action
&
3534 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
);
3535 bool esw_paired
= mlx5_devcom_is_paired(attr
->in_mdev
->priv
.devcom
,
3536 MLX5_DEVCOM_ESW_OFFLOADS
);
3541 if ((mlx5_lag_is_sriov(attr
->in_mdev
) ||
3542 mlx5_lag_is_multipath(attr
->in_mdev
)) &&
3543 (is_rep_ingress
|| act_is_encap
))
3550 mlx5e_alloc_flow(struct mlx5e_priv
*priv
, int attr_size
,
3551 struct flow_cls_offload
*f
, unsigned long flow_flags
,
3552 struct mlx5e_tc_flow_parse_attr
**__parse_attr
,
3553 struct mlx5e_tc_flow
**__flow
)
3555 struct mlx5e_tc_flow_parse_attr
*parse_attr
;
3556 struct mlx5e_tc_flow
*flow
;
3559 flow
= kzalloc(sizeof(*flow
) + attr_size
, GFP_KERNEL
);
3560 parse_attr
= kvzalloc(sizeof(*parse_attr
), GFP_KERNEL
);
3561 if (!parse_attr
|| !flow
) {
3566 flow
->cookie
= f
->cookie
;
3567 flow
->flags
= flow_flags
;
3569 for (out_index
= 0; out_index
< MLX5_MAX_FLOW_FWD_VPORTS
; out_index
++)
3570 INIT_LIST_HEAD(&flow
->encaps
[out_index
].list
);
3571 INIT_LIST_HEAD(&flow
->mod_hdr
);
3572 INIT_LIST_HEAD(&flow
->hairpin
);
3573 refcount_set(&flow
->refcnt
, 1);
3574 init_completion(&flow
->init_done
);
3577 *__parse_attr
= parse_attr
;
3588 mlx5e_flow_esw_attr_init(struct mlx5_esw_flow_attr
*esw_attr
,
3589 struct mlx5e_priv
*priv
,
3590 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
3591 struct flow_cls_offload
*f
,
3592 struct mlx5_eswitch_rep
*in_rep
,
3593 struct mlx5_core_dev
*in_mdev
)
3595 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
3597 esw_attr
->parse_attr
= parse_attr
;
3598 esw_attr
->chain
= f
->common
.chain_index
;
3599 esw_attr
->prio
= f
->common
.prio
;
3601 esw_attr
->in_rep
= in_rep
;
3602 esw_attr
->in_mdev
= in_mdev
;
3604 if (MLX5_CAP_ESW(esw
->dev
, counter_eswitch_affinity
) ==
3605 MLX5_COUNTER_SOURCE_ESWITCH
)
3606 esw_attr
->counter_dev
= in_mdev
;
3608 esw_attr
->counter_dev
= priv
->mdev
;
3611 static struct mlx5e_tc_flow
*
3612 __mlx5e_add_fdb_flow(struct mlx5e_priv
*priv
,
3613 struct flow_cls_offload
*f
,
3614 unsigned long flow_flags
,
3615 struct net_device
*filter_dev
,
3616 struct mlx5_eswitch_rep
*in_rep
,
3617 struct mlx5_core_dev
*in_mdev
)
3619 struct flow_rule
*rule
= flow_cls_offload_flow_rule(f
);
3620 struct netlink_ext_ack
*extack
= f
->common
.extack
;
3621 struct mlx5e_tc_flow_parse_attr
*parse_attr
;
3622 struct mlx5e_tc_flow
*flow
;
3625 flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_ESWITCH
);
3626 attr_size
= sizeof(struct mlx5_esw_flow_attr
);
3627 err
= mlx5e_alloc_flow(priv
, attr_size
, f
, flow_flags
,
3628 &parse_attr
, &flow
);
3632 parse_attr
->filter_dev
= filter_dev
;
3633 mlx5e_flow_esw_attr_init(flow
->esw_attr
,
3635 f
, in_rep
, in_mdev
);
3637 err
= parse_cls_flower(flow
->priv
, flow
, &parse_attr
->spec
,
3642 err
= parse_tc_fdb_actions(priv
, &rule
->action
, flow
, extack
);
3646 err
= mlx5e_tc_add_fdb_flow(priv
, flow
, extack
);
3647 complete_all(&flow
->init_done
);
3649 if (!(err
== -ENETUNREACH
&& mlx5_lag_is_multipath(in_mdev
)))
3652 add_unready_flow(flow
);
3658 mlx5e_flow_put(priv
, flow
);
3660 return ERR_PTR(err
);
3663 static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload
*f
,
3664 struct mlx5e_tc_flow
*flow
,
3665 unsigned long flow_flags
)
3667 struct mlx5e_priv
*priv
= flow
->priv
, *peer_priv
;
3668 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
, *peer_esw
;
3669 struct mlx5_devcom
*devcom
= priv
->mdev
->priv
.devcom
;
3670 struct mlx5e_tc_flow_parse_attr
*parse_attr
;
3671 struct mlx5e_rep_priv
*peer_urpriv
;
3672 struct mlx5e_tc_flow
*peer_flow
;
3673 struct mlx5_core_dev
*in_mdev
;
3676 peer_esw
= mlx5_devcom_get_peer_data(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
3680 peer_urpriv
= mlx5_eswitch_get_uplink_priv(peer_esw
, REP_ETH
);
3681 peer_priv
= netdev_priv(peer_urpriv
->netdev
);
3683 /* in_mdev is assigned of which the packet originated from.
3684 * So packets redirected to uplink use the same mdev of the
3685 * original flow and packets redirected from uplink use the
3688 if (flow
->esw_attr
->in_rep
->vport
== MLX5_VPORT_UPLINK
)
3689 in_mdev
= peer_priv
->mdev
;
3691 in_mdev
= priv
->mdev
;
3693 parse_attr
= flow
->esw_attr
->parse_attr
;
3694 peer_flow
= __mlx5e_add_fdb_flow(peer_priv
, f
, flow_flags
,
3695 parse_attr
->filter_dev
,
3696 flow
->esw_attr
->in_rep
, in_mdev
);
3697 if (IS_ERR(peer_flow
)) {
3698 err
= PTR_ERR(peer_flow
);
3702 flow
->peer_flow
= peer_flow
;
3703 flow_flag_set(flow
, DUP
);
3704 mutex_lock(&esw
->offloads
.peer_mutex
);
3705 list_add_tail(&flow
->peer
, &esw
->offloads
.peer_flows
);
3706 mutex_unlock(&esw
->offloads
.peer_mutex
);
3709 mlx5_devcom_release_peer_data(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
3714 mlx5e_add_fdb_flow(struct mlx5e_priv
*priv
,
3715 struct flow_cls_offload
*f
,
3716 unsigned long flow_flags
,
3717 struct net_device
*filter_dev
,
3718 struct mlx5e_tc_flow
**__flow
)
3720 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
3721 struct mlx5_eswitch_rep
*in_rep
= rpriv
->rep
;
3722 struct mlx5_core_dev
*in_mdev
= priv
->mdev
;
3723 struct mlx5e_tc_flow
*flow
;
3726 flow
= __mlx5e_add_fdb_flow(priv
, f
, flow_flags
, filter_dev
, in_rep
,
3729 return PTR_ERR(flow
);
3731 if (is_peer_flow_needed(flow
)) {
3732 err
= mlx5e_tc_add_fdb_peer_flow(f
, flow
, flow_flags
);
3734 mlx5e_tc_del_fdb_flow(priv
, flow
);
3748 mlx5e_add_nic_flow(struct mlx5e_priv
*priv
,
3749 struct flow_cls_offload
*f
,
3750 unsigned long flow_flags
,
3751 struct net_device
*filter_dev
,
3752 struct mlx5e_tc_flow
**__flow
)
3754 struct flow_rule
*rule
= flow_cls_offload_flow_rule(f
);
3755 struct netlink_ext_ack
*extack
= f
->common
.extack
;
3756 struct mlx5e_tc_flow_parse_attr
*parse_attr
;
3757 struct mlx5e_tc_flow
*flow
;
3760 /* multi-chain not supported for NIC rules */
3761 if (!tc_cls_can_offload_and_chain0(priv
->netdev
, &f
->common
))
3764 flow_flags
|= BIT(MLX5E_TC_FLOW_FLAG_NIC
);
3765 attr_size
= sizeof(struct mlx5_nic_flow_attr
);
3766 err
= mlx5e_alloc_flow(priv
, attr_size
, f
, flow_flags
,
3767 &parse_attr
, &flow
);
3771 parse_attr
->filter_dev
= filter_dev
;
3772 err
= parse_cls_flower(flow
->priv
, flow
, &parse_attr
->spec
,
3777 err
= parse_tc_nic_actions(priv
, &rule
->action
, parse_attr
, flow
, extack
);
3781 err
= mlx5e_tc_add_nic_flow(priv
, parse_attr
, flow
, extack
);
3785 flow_flag_set(flow
, OFFLOADED
);
3792 mlx5e_flow_put(priv
, flow
);
3799 mlx5e_tc_add_flow(struct mlx5e_priv
*priv
,
3800 struct flow_cls_offload
*f
,
3801 unsigned long flags
,
3802 struct net_device
*filter_dev
,
3803 struct mlx5e_tc_flow
**flow
)
3805 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
3806 unsigned long flow_flags
;
3809 get_flags(flags
, &flow_flags
);
3811 if (!tc_can_offload_extack(priv
->netdev
, f
->common
.extack
))
3814 if (esw
&& esw
->mode
== MLX5_ESWITCH_OFFLOADS
)
3815 err
= mlx5e_add_fdb_flow(priv
, f
, flow_flags
,
3818 err
= mlx5e_add_nic_flow(priv
, f
, flow_flags
,
3824 int mlx5e_configure_flower(struct net_device
*dev
, struct mlx5e_priv
*priv
,
3825 struct flow_cls_offload
*f
, unsigned long flags
)
3827 struct netlink_ext_ack
*extack
= f
->common
.extack
;
3828 struct rhashtable
*tc_ht
= get_tc_ht(priv
, flags
);
3829 struct mlx5e_tc_flow
*flow
;
3833 flow
= rhashtable_lookup(tc_ht
, &f
->cookie
, tc_ht_params
);
3836 NL_SET_ERR_MSG_MOD(extack
,
3837 "flow cookie already exists, ignoring");
3838 netdev_warn_once(priv
->netdev
,
3839 "flow cookie %lx already exists, ignoring\n",
3845 trace_mlx5e_configure_flower(f
);
3846 err
= mlx5e_tc_add_flow(priv
, f
, flags
, dev
, &flow
);
3850 err
= rhashtable_lookup_insert_fast(tc_ht
, &flow
->node
, tc_ht_params
);
3857 mlx5e_flow_put(priv
, flow
);
3862 static bool same_flow_direction(struct mlx5e_tc_flow
*flow
, int flags
)
3864 bool dir_ingress
= !!(flags
& MLX5_TC_FLAG(INGRESS
));
3865 bool dir_egress
= !!(flags
& MLX5_TC_FLAG(EGRESS
));
3867 return flow_flag_test(flow
, INGRESS
) == dir_ingress
&&
3868 flow_flag_test(flow
, EGRESS
) == dir_egress
;
3871 int mlx5e_delete_flower(struct net_device
*dev
, struct mlx5e_priv
*priv
,
3872 struct flow_cls_offload
*f
, unsigned long flags
)
3874 struct rhashtable
*tc_ht
= get_tc_ht(priv
, flags
);
3875 struct mlx5e_tc_flow
*flow
;
3879 flow
= rhashtable_lookup(tc_ht
, &f
->cookie
, tc_ht_params
);
3880 if (!flow
|| !same_flow_direction(flow
, flags
)) {
3885 /* Only delete the flow if it doesn't have MLX5E_TC_FLOW_DELETED flag
3888 if (flow_flag_test_and_set(flow
, DELETED
)) {
3892 rhashtable_remove_fast(tc_ht
, &flow
->node
, tc_ht_params
);
3895 trace_mlx5e_delete_flower(f
);
3896 mlx5e_flow_put(priv
, flow
);
3905 int mlx5e_stats_flower(struct net_device
*dev
, struct mlx5e_priv
*priv
,
3906 struct flow_cls_offload
*f
, unsigned long flags
)
3908 struct mlx5_devcom
*devcom
= priv
->mdev
->priv
.devcom
;
3909 struct rhashtable
*tc_ht
= get_tc_ht(priv
, flags
);
3910 struct mlx5_eswitch
*peer_esw
;
3911 struct mlx5e_tc_flow
*flow
;
3912 struct mlx5_fc
*counter
;
3919 flow
= mlx5e_flow_get(rhashtable_lookup(tc_ht
, &f
->cookie
,
3923 return PTR_ERR(flow
);
3925 if (!same_flow_direction(flow
, flags
)) {
3930 if (mlx5e_is_offloaded_flow(flow
)) {
3931 counter
= mlx5e_tc_get_counter(flow
);
3935 mlx5_fc_query_cached(counter
, &bytes
, &packets
, &lastuse
);
3938 /* Under multipath it's possible for one rule to be currently
3939 * un-offloaded while the other rule is offloaded.
3941 peer_esw
= mlx5_devcom_get_peer_data(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
3945 if (flow_flag_test(flow
, DUP
) &&
3946 flow_flag_test(flow
->peer_flow
, OFFLOADED
)) {
3951 counter
= mlx5e_tc_get_counter(flow
->peer_flow
);
3953 goto no_peer_counter
;
3954 mlx5_fc_query_cached(counter
, &bytes2
, &packets2
, &lastuse2
);
3957 packets
+= packets2
;
3958 lastuse
= max_t(u64
, lastuse
, lastuse2
);
3962 mlx5_devcom_release_peer_data(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
3964 flow_stats_update(&f
->stats
, bytes
, packets
, lastuse
);
3965 trace_mlx5e_stats_flower(f
);
3967 mlx5e_flow_put(priv
, flow
);
3971 static int apply_police_params(struct mlx5e_priv
*priv
, u32 rate
,
3972 struct netlink_ext_ack
*extack
)
3974 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
3975 struct mlx5_eswitch
*esw
;
3980 esw
= priv
->mdev
->priv
.eswitch
;
3981 /* rate is given in bytes/sec.
3982 * First convert to bits/sec and then round to the nearest mbit/secs.
3983 * mbit means million bits.
3984 * Moreover, if rate is non zero we choose to configure to a minimum of
3987 rate_mbps
= rate
? max_t(u32
, (rate
* 8 + 500000) / 1000000, 1) : 0;
3988 vport_num
= rpriv
->rep
->vport
;
3990 err
= mlx5_esw_modify_vport_rate(esw
, vport_num
, rate_mbps
);
3992 NL_SET_ERR_MSG_MOD(extack
, "failed applying action to hardware");
3997 static int scan_tc_matchall_fdb_actions(struct mlx5e_priv
*priv
,
3998 struct flow_action
*flow_action
,
3999 struct netlink_ext_ack
*extack
)
4001 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
4002 const struct flow_action_entry
*act
;
4006 if (!flow_action_has_entries(flow_action
)) {
4007 NL_SET_ERR_MSG_MOD(extack
, "matchall called with no action");
4011 if (!flow_offload_has_one_action(flow_action
)) {
4012 NL_SET_ERR_MSG_MOD(extack
, "matchall policing support only a single action");
4016 flow_action_for_each(i
, act
, flow_action
) {
4018 case FLOW_ACTION_POLICE
:
4019 err
= apply_police_params(priv
, act
->police
.rate_bytes_ps
, extack
);
4023 rpriv
->prev_vf_vport_stats
= priv
->stats
.vf_vport
;
4026 NL_SET_ERR_MSG_MOD(extack
, "mlx5 supports only police action for matchall");
4034 int mlx5e_tc_configure_matchall(struct mlx5e_priv
*priv
,
4035 struct tc_cls_matchall_offload
*ma
)
4037 struct netlink_ext_ack
*extack
= ma
->common
.extack
;
4039 if (ma
->common
.prio
!= 1) {
4040 NL_SET_ERR_MSG_MOD(extack
, "only priority 1 is supported");
4044 return scan_tc_matchall_fdb_actions(priv
, &ma
->rule
->action
, extack
);
4047 int mlx5e_tc_delete_matchall(struct mlx5e_priv
*priv
,
4048 struct tc_cls_matchall_offload
*ma
)
4050 struct netlink_ext_ack
*extack
= ma
->common
.extack
;
4052 return apply_police_params(priv
, 0, extack
);
4055 void mlx5e_tc_stats_matchall(struct mlx5e_priv
*priv
,
4056 struct tc_cls_matchall_offload
*ma
)
4058 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
4059 struct rtnl_link_stats64 cur_stats
;
4063 cur_stats
= priv
->stats
.vf_vport
;
4064 dpkts
= cur_stats
.rx_packets
- rpriv
->prev_vf_vport_stats
.rx_packets
;
4065 dbytes
= cur_stats
.rx_bytes
- rpriv
->prev_vf_vport_stats
.rx_bytes
;
4066 rpriv
->prev_vf_vport_stats
= cur_stats
;
4067 flow_stats_update(&ma
->stats
, dpkts
, dbytes
, jiffies
);
4070 static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv
*priv
,
4071 struct mlx5e_priv
*peer_priv
)
4073 struct mlx5_core_dev
*peer_mdev
= peer_priv
->mdev
;
4074 struct mlx5e_hairpin_entry
*hpe
, *tmp
;
4075 LIST_HEAD(init_wait_list
);
4079 if (!same_hw_devs(priv
, peer_priv
))
4082 peer_vhca_id
= MLX5_CAP_GEN(peer_mdev
, vhca_id
);
4084 mutex_lock(&priv
->fs
.tc
.hairpin_tbl_lock
);
4085 hash_for_each(priv
->fs
.tc
.hairpin_tbl
, bkt
, hpe
, hairpin_hlist
)
4086 if (refcount_inc_not_zero(&hpe
->refcnt
))
4087 list_add(&hpe
->dead_peer_wait_list
, &init_wait_list
);
4088 mutex_unlock(&priv
->fs
.tc
.hairpin_tbl_lock
);
4090 list_for_each_entry_safe(hpe
, tmp
, &init_wait_list
, dead_peer_wait_list
) {
4091 wait_for_completion(&hpe
->res_ready
);
4092 if (!IS_ERR_OR_NULL(hpe
->hp
) && hpe
->peer_vhca_id
== peer_vhca_id
)
4093 hpe
->hp
->pair
->peer_gone
= true;
4095 mlx5e_hairpin_put(priv
, hpe
);
4099 static int mlx5e_tc_netdev_event(struct notifier_block
*this,
4100 unsigned long event
, void *ptr
)
4102 struct net_device
*ndev
= netdev_notifier_info_to_dev(ptr
);
4103 struct mlx5e_flow_steering
*fs
;
4104 struct mlx5e_priv
*peer_priv
;
4105 struct mlx5e_tc_table
*tc
;
4106 struct mlx5e_priv
*priv
;
4108 if (ndev
->netdev_ops
!= &mlx5e_netdev_ops
||
4109 event
!= NETDEV_UNREGISTER
||
4110 ndev
->reg_state
== NETREG_REGISTERED
)
4113 tc
= container_of(this, struct mlx5e_tc_table
, netdevice_nb
);
4114 fs
= container_of(tc
, struct mlx5e_flow_steering
, tc
);
4115 priv
= container_of(fs
, struct mlx5e_priv
, fs
);
4116 peer_priv
= netdev_priv(ndev
);
4117 if (priv
== peer_priv
||
4118 !(priv
->netdev
->features
& NETIF_F_HW_TC
))
4121 mlx5e_tc_hairpin_update_dead_peer(priv
, peer_priv
);
4126 int mlx5e_tc_nic_init(struct mlx5e_priv
*priv
)
4128 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
4131 mutex_init(&tc
->t_lock
);
4132 mutex_init(&tc
->mod_hdr
.lock
);
4133 hash_init(tc
->mod_hdr
.hlist
);
4134 mutex_init(&tc
->hairpin_tbl_lock
);
4135 hash_init(tc
->hairpin_tbl
);
4137 err
= rhashtable_init(&tc
->ht
, &tc_ht_params
);
4141 tc
->netdevice_nb
.notifier_call
= mlx5e_tc_netdev_event
;
4142 if (register_netdevice_notifier(&tc
->netdevice_nb
)) {
4143 tc
->netdevice_nb
.notifier_call
= NULL
;
4144 mlx5_core_warn(priv
->mdev
, "Failed to register netdev notifier\n");
4150 static void _mlx5e_tc_del_flow(void *ptr
, void *arg
)
4152 struct mlx5e_tc_flow
*flow
= ptr
;
4153 struct mlx5e_priv
*priv
= flow
->priv
;
4155 mlx5e_tc_del_flow(priv
, flow
);
4159 void mlx5e_tc_nic_cleanup(struct mlx5e_priv
*priv
)
4161 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
4163 if (tc
->netdevice_nb
.notifier_call
)
4164 unregister_netdevice_notifier(&tc
->netdevice_nb
);
4166 mutex_destroy(&tc
->mod_hdr
.lock
);
4167 mutex_destroy(&tc
->hairpin_tbl_lock
);
4169 rhashtable_destroy(&tc
->ht
);
4171 if (!IS_ERR_OR_NULL(tc
->t
)) {
4172 mlx5_destroy_flow_table(tc
->t
);
4175 mutex_destroy(&tc
->t_lock
);
4178 int mlx5e_tc_esw_init(struct rhashtable
*tc_ht
)
4180 return rhashtable_init(tc_ht
, &tc_ht_params
);
4183 void mlx5e_tc_esw_cleanup(struct rhashtable
*tc_ht
)
4185 rhashtable_free_and_destroy(tc_ht
, _mlx5e_tc_del_flow
, NULL
);
4188 int mlx5e_tc_num_filters(struct mlx5e_priv
*priv
, unsigned long flags
)
4190 struct rhashtable
*tc_ht
= get_tc_ht(priv
, flags
);
4192 return atomic_read(&tc_ht
->nelems
);
4195 void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch
*esw
)
4197 struct mlx5e_tc_flow
*flow
, *tmp
;
4199 list_for_each_entry_safe(flow
, tmp
, &esw
->offloads
.peer_flows
, peer
)
4200 __mlx5e_tc_del_fdb_peer_flow(flow
);
4203 void mlx5e_tc_reoffload_flows_work(struct work_struct
*work
)
4205 struct mlx5_rep_uplink_priv
*rpriv
=
4206 container_of(work
, struct mlx5_rep_uplink_priv
,
4207 reoffload_flows_work
);
4208 struct mlx5e_tc_flow
*flow
, *tmp
;
4210 mutex_lock(&rpriv
->unready_flows_lock
);
4211 list_for_each_entry_safe(flow
, tmp
, &rpriv
->unready_flows
, unready
) {
4212 if (!mlx5e_tc_add_fdb_flow(flow
->priv
, flow
, NULL
))
4213 unready_flow_del(flow
);
4215 mutex_unlock(&rpriv
->unready_flows_lock
);