2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <net/flow_dissector.h>
34 #include <net/sch_generic.h>
35 #include <net/pkt_cls.h>
36 #include <net/tc_act/tc_gact.h>
37 #include <net/tc_act/tc_skbedit.h>
38 #include <linux/mlx5/fs.h>
39 #include <linux/mlx5/device.h>
40 #include <linux/rhashtable.h>
41 #include <net/switchdev.h>
42 #include <net/tc_act/tc_mirred.h>
43 #include <net/tc_act/tc_vlan.h>
44 #include <net/tc_act/tc_tunnel_key.h>
45 #include <net/tc_act/tc_pedit.h>
46 #include <net/tc_act/tc_csum.h>
47 #include <net/vxlan.h>
56 struct mlx5_nic_flow_attr
{
61 struct mlx5_flow_table
*hairpin_ft
;
65 MLX5E_TC_FLOW_ESWITCH
= BIT(0),
66 MLX5E_TC_FLOW_NIC
= BIT(1),
67 MLX5E_TC_FLOW_OFFLOADED
= BIT(2),
68 MLX5E_TC_FLOW_HAIRPIN
= BIT(3),
69 MLX5E_TC_FLOW_HAIRPIN_RSS
= BIT(4),
72 struct mlx5e_tc_flow
{
73 struct rhash_head node
;
76 struct mlx5_flow_handle
*rule
;
77 struct list_head encap
; /* flows sharing the same encap ID */
78 struct list_head mod_hdr
; /* flows sharing the same mod hdr ID */
79 struct list_head hairpin
; /* flows sharing the same hairpin */
81 struct mlx5_esw_flow_attr esw_attr
[0];
82 struct mlx5_nic_flow_attr nic_attr
[0];
86 struct mlx5e_tc_flow_parse_attr
{
87 struct ip_tunnel_info tun_info
;
88 struct mlx5_flow_spec spec
;
89 int num_mod_hdr_actions
;
90 void *mod_hdr_actions
;
95 MLX5_HEADER_TYPE_VXLAN
= 0x0,
96 MLX5_HEADER_TYPE_NVGRE
= 0x1,
99 #define MLX5E_TC_TABLE_NUM_GROUPS 4
100 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE (1 << 16)
102 struct mlx5e_hairpin
{
103 struct mlx5_hairpin
*pair
;
105 struct mlx5_core_dev
*func_mdev
;
106 struct mlx5e_priv
*func_priv
;
111 struct mlx5e_rqt indir_rqt
;
112 u32 indir_tirn
[MLX5E_NUM_INDIR_TIRS
];
113 struct mlx5e_ttc_table ttc
;
116 struct mlx5e_hairpin_entry
{
117 /* a node of a hash table which keeps all the hairpin entries */
118 struct hlist_node hairpin_hlist
;
120 /* flows sharing the same hairpin */
121 struct list_head flows
;
125 struct mlx5e_hairpin
*hp
;
133 struct mlx5e_mod_hdr_entry
{
134 /* a node of a hash table which keeps all the mod_hdr entries */
135 struct hlist_node mod_hdr_hlist
;
137 /* flows sharing the same mod_hdr entry */
138 struct list_head flows
;
140 struct mod_hdr_key key
;
145 #define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto)
147 static inline u32
hash_mod_hdr_info(struct mod_hdr_key
*key
)
149 return jhash(key
->actions
,
150 key
->num_actions
* MLX5_MH_ACT_SZ
, 0);
153 static inline int cmp_mod_hdr_info(struct mod_hdr_key
*a
,
154 struct mod_hdr_key
*b
)
156 if (a
->num_actions
!= b
->num_actions
)
159 return memcmp(a
->actions
, b
->actions
, a
->num_actions
* MLX5_MH_ACT_SZ
);
162 static int mlx5e_attach_mod_hdr(struct mlx5e_priv
*priv
,
163 struct mlx5e_tc_flow
*flow
,
164 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
166 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
167 int num_actions
, actions_size
, namespace, err
;
168 struct mlx5e_mod_hdr_entry
*mh
;
169 struct mod_hdr_key key
;
173 num_actions
= parse_attr
->num_mod_hdr_actions
;
174 actions_size
= MLX5_MH_ACT_SZ
* num_actions
;
176 key
.actions
= parse_attr
->mod_hdr_actions
;
177 key
.num_actions
= num_actions
;
179 hash_key
= hash_mod_hdr_info(&key
);
181 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
) {
182 namespace = MLX5_FLOW_NAMESPACE_FDB
;
183 hash_for_each_possible(esw
->offloads
.mod_hdr_tbl
, mh
,
184 mod_hdr_hlist
, hash_key
) {
185 if (!cmp_mod_hdr_info(&mh
->key
, &key
)) {
191 namespace = MLX5_FLOW_NAMESPACE_KERNEL
;
192 hash_for_each_possible(priv
->fs
.tc
.mod_hdr_tbl
, mh
,
193 mod_hdr_hlist
, hash_key
) {
194 if (!cmp_mod_hdr_info(&mh
->key
, &key
)) {
204 mh
= kzalloc(sizeof(*mh
) + actions_size
, GFP_KERNEL
);
208 mh
->key
.actions
= (void *)mh
+ sizeof(*mh
);
209 memcpy(mh
->key
.actions
, key
.actions
, actions_size
);
210 mh
->key
.num_actions
= num_actions
;
211 INIT_LIST_HEAD(&mh
->flows
);
213 err
= mlx5_modify_header_alloc(priv
->mdev
, namespace,
220 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
)
221 hash_add(esw
->offloads
.mod_hdr_tbl
, &mh
->mod_hdr_hlist
, hash_key
);
223 hash_add(priv
->fs
.tc
.mod_hdr_tbl
, &mh
->mod_hdr_hlist
, hash_key
);
226 list_add(&flow
->mod_hdr
, &mh
->flows
);
227 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
)
228 flow
->esw_attr
->mod_hdr_id
= mh
->mod_hdr_id
;
230 flow
->nic_attr
->mod_hdr_id
= mh
->mod_hdr_id
;
239 static void mlx5e_detach_mod_hdr(struct mlx5e_priv
*priv
,
240 struct mlx5e_tc_flow
*flow
)
242 struct list_head
*next
= flow
->mod_hdr
.next
;
244 list_del(&flow
->mod_hdr
);
246 if (list_empty(next
)) {
247 struct mlx5e_mod_hdr_entry
*mh
;
249 mh
= list_entry(next
, struct mlx5e_mod_hdr_entry
, flows
);
251 mlx5_modify_header_dealloc(priv
->mdev
, mh
->mod_hdr_id
);
252 hash_del(&mh
->mod_hdr_hlist
);
258 struct mlx5_core_dev
*mlx5e_hairpin_get_mdev(struct net
*net
, int ifindex
)
260 struct net_device
*netdev
;
261 struct mlx5e_priv
*priv
;
263 netdev
= __dev_get_by_index(net
, ifindex
);
264 priv
= netdev_priv(netdev
);
268 static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin
*hp
)
270 u32 in
[MLX5_ST_SZ_DW(create_tir_in
)] = {0};
274 err
= mlx5_core_alloc_transport_domain(hp
->func_mdev
, &hp
->tdn
);
278 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
280 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_DIRECT
);
281 MLX5_SET(tirc
, tirc
, inline_rqn
, hp
->pair
->rqn
[0]);
282 MLX5_SET(tirc
, tirc
, transport_domain
, hp
->tdn
);
284 err
= mlx5_core_create_tir(hp
->func_mdev
, in
, MLX5_ST_SZ_BYTES(create_tir_in
), &hp
->tirn
);
291 mlx5_core_dealloc_transport_domain(hp
->func_mdev
, hp
->tdn
);
296 static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin
*hp
)
298 mlx5_core_destroy_tir(hp
->func_mdev
, hp
->tirn
);
299 mlx5_core_dealloc_transport_domain(hp
->func_mdev
, hp
->tdn
);
302 static void mlx5e_hairpin_fill_rqt_rqns(struct mlx5e_hairpin
*hp
, void *rqtc
)
304 u32 indirection_rqt
[MLX5E_INDIR_RQT_SIZE
], rqn
;
305 struct mlx5e_priv
*priv
= hp
->func_priv
;
306 int i
, ix
, sz
= MLX5E_INDIR_RQT_SIZE
;
308 mlx5e_build_default_indir_rqt(indirection_rqt
, sz
,
311 for (i
= 0; i
< sz
; i
++) {
313 if (priv
->channels
.params
.rss_hfunc
== ETH_RSS_HASH_XOR
)
314 ix
= mlx5e_bits_invert(i
, ilog2(sz
));
315 ix
= indirection_rqt
[ix
];
316 rqn
= hp
->pair
->rqn
[ix
];
317 MLX5_SET(rqtc
, rqtc
, rq_num
[i
], rqn
);
321 static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin
*hp
)
323 int inlen
, err
, sz
= MLX5E_INDIR_RQT_SIZE
;
324 struct mlx5e_priv
*priv
= hp
->func_priv
;
325 struct mlx5_core_dev
*mdev
= priv
->mdev
;
329 inlen
= MLX5_ST_SZ_BYTES(create_rqt_in
) + sizeof(u32
) * sz
;
330 in
= kvzalloc(inlen
, GFP_KERNEL
);
334 rqtc
= MLX5_ADDR_OF(create_rqt_in
, in
, rqt_context
);
336 MLX5_SET(rqtc
, rqtc
, rqt_actual_size
, sz
);
337 MLX5_SET(rqtc
, rqtc
, rqt_max_size
, sz
);
339 mlx5e_hairpin_fill_rqt_rqns(hp
, rqtc
);
341 err
= mlx5_core_create_rqt(mdev
, in
, inlen
, &hp
->indir_rqt
.rqtn
);
343 hp
->indir_rqt
.enabled
= true;
349 static int mlx5e_hairpin_create_indirect_tirs(struct mlx5e_hairpin
*hp
)
351 struct mlx5e_priv
*priv
= hp
->func_priv
;
352 u32 in
[MLX5_ST_SZ_DW(create_tir_in
)];
356 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++) {
357 memset(in
, 0, MLX5_ST_SZ_BYTES(create_tir_in
));
358 tirc
= MLX5_ADDR_OF(create_tir_in
, in
, ctx
);
360 MLX5_SET(tirc
, tirc
, transport_domain
, hp
->tdn
);
361 MLX5_SET(tirc
, tirc
, disp_type
, MLX5_TIRC_DISP_TYPE_INDIRECT
);
362 MLX5_SET(tirc
, tirc
, indirect_table
, hp
->indir_rqt
.rqtn
);
363 mlx5e_build_indir_tir_ctx_hash(&priv
->channels
.params
, tt
, tirc
, false);
365 err
= mlx5_core_create_tir(hp
->func_mdev
, in
,
366 MLX5_ST_SZ_BYTES(create_tir_in
), &hp
->indir_tirn
[tt
]);
368 mlx5_core_warn(hp
->func_mdev
, "create indirect tirs failed, %d\n", err
);
369 goto err_destroy_tirs
;
375 for (i
= 0; i
< tt
; i
++)
376 mlx5_core_destroy_tir(hp
->func_mdev
, hp
->indir_tirn
[i
]);
380 static void mlx5e_hairpin_destroy_indirect_tirs(struct mlx5e_hairpin
*hp
)
384 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++)
385 mlx5_core_destroy_tir(hp
->func_mdev
, hp
->indir_tirn
[tt
]);
388 static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin
*hp
,
389 struct ttc_params
*ttc_params
)
391 struct mlx5_flow_table_attr
*ft_attr
= &ttc_params
->ft_attr
;
394 memset(ttc_params
, 0, sizeof(*ttc_params
));
396 ttc_params
->any_tt_tirn
= hp
->tirn
;
398 for (tt
= 0; tt
< MLX5E_NUM_INDIR_TIRS
; tt
++)
399 ttc_params
->indir_tirn
[tt
] = hp
->indir_tirn
[tt
];
401 ft_attr
->max_fte
= MLX5E_NUM_TT
;
402 ft_attr
->level
= MLX5E_TC_TTC_FT_LEVEL
;
403 ft_attr
->prio
= MLX5E_TC_PRIO
;
406 static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin
*hp
)
408 struct mlx5e_priv
*priv
= hp
->func_priv
;
409 struct ttc_params ttc_params
;
412 err
= mlx5e_hairpin_create_indirect_rqt(hp
);
416 err
= mlx5e_hairpin_create_indirect_tirs(hp
);
418 goto err_create_indirect_tirs
;
420 mlx5e_hairpin_set_ttc_params(hp
, &ttc_params
);
421 err
= mlx5e_create_ttc_table(priv
, &ttc_params
, &hp
->ttc
);
423 goto err_create_ttc_table
;
425 netdev_dbg(priv
->netdev
, "add hairpin: using %d channels rss ttc table id %x\n",
426 hp
->num_channels
, hp
->ttc
.ft
.t
->id
);
430 err_create_ttc_table
:
431 mlx5e_hairpin_destroy_indirect_tirs(hp
);
432 err_create_indirect_tirs
:
433 mlx5e_destroy_rqt(priv
, &hp
->indir_rqt
);
438 static void mlx5e_hairpin_rss_cleanup(struct mlx5e_hairpin
*hp
)
440 struct mlx5e_priv
*priv
= hp
->func_priv
;
442 mlx5e_destroy_ttc_table(priv
, &hp
->ttc
);
443 mlx5e_hairpin_destroy_indirect_tirs(hp
);
444 mlx5e_destroy_rqt(priv
, &hp
->indir_rqt
);
447 static struct mlx5e_hairpin
*
448 mlx5e_hairpin_create(struct mlx5e_priv
*priv
, struct mlx5_hairpin_params
*params
,
451 struct mlx5_core_dev
*func_mdev
, *peer_mdev
;
452 struct mlx5e_hairpin
*hp
;
453 struct mlx5_hairpin
*pair
;
456 hp
= kzalloc(sizeof(*hp
), GFP_KERNEL
);
458 return ERR_PTR(-ENOMEM
);
460 func_mdev
= priv
->mdev
;
461 peer_mdev
= mlx5e_hairpin_get_mdev(dev_net(priv
->netdev
), peer_ifindex
);
463 pair
= mlx5_core_hairpin_create(func_mdev
, peer_mdev
, params
);
466 goto create_pair_err
;
469 hp
->func_mdev
= func_mdev
;
470 hp
->func_priv
= priv
;
471 hp
->num_channels
= params
->num_channels
;
473 err
= mlx5e_hairpin_create_transport(hp
);
475 goto create_transport_err
;
477 if (hp
->num_channels
> 1) {
478 err
= mlx5e_hairpin_rss_init(hp
);
486 mlx5e_hairpin_destroy_transport(hp
);
487 create_transport_err
:
488 mlx5_core_hairpin_destroy(hp
->pair
);
494 static void mlx5e_hairpin_destroy(struct mlx5e_hairpin
*hp
)
496 if (hp
->num_channels
> 1)
497 mlx5e_hairpin_rss_cleanup(hp
);
498 mlx5e_hairpin_destroy_transport(hp
);
499 mlx5_core_hairpin_destroy(hp
->pair
);
503 static inline u32
hash_hairpin_info(u16 peer_vhca_id
, u8 prio
)
505 return (peer_vhca_id
<< 16 | prio
);
508 static struct mlx5e_hairpin_entry
*mlx5e_hairpin_get(struct mlx5e_priv
*priv
,
509 u16 peer_vhca_id
, u8 prio
)
511 struct mlx5e_hairpin_entry
*hpe
;
512 u32 hash_key
= hash_hairpin_info(peer_vhca_id
, prio
);
514 hash_for_each_possible(priv
->fs
.tc
.hairpin_tbl
, hpe
,
515 hairpin_hlist
, hash_key
) {
516 if (hpe
->peer_vhca_id
== peer_vhca_id
&& hpe
->prio
== prio
)
523 #define UNKNOWN_MATCH_PRIO 8
525 static int mlx5e_hairpin_get_prio(struct mlx5e_priv
*priv
,
526 struct mlx5_flow_spec
*spec
, u8
*match_prio
)
528 void *headers_c
, *headers_v
;
529 u8 prio_val
, prio_mask
= 0;
532 #ifdef CONFIG_MLX5_CORE_EN_DCB
533 if (priv
->dcbx_dp
.trust_state
!= MLX5_QPTS_TRUST_PCP
) {
534 netdev_warn(priv
->netdev
,
535 "only PCP trust state supported for hairpin\n");
539 headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, outer_headers
);
540 headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, outer_headers
);
542 vlan_present
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, cvlan_tag
);
544 prio_mask
= MLX5_GET(fte_match_set_lyr_2_4
, headers_c
, first_prio
);
545 prio_val
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, first_prio
);
548 if (!vlan_present
|| !prio_mask
) {
549 prio_val
= UNKNOWN_MATCH_PRIO
;
550 } else if (prio_mask
!= 0x7) {
551 netdev_warn(priv
->netdev
,
552 "masked priority match not supported for hairpin\n");
556 *match_prio
= prio_val
;
560 static int mlx5e_hairpin_flow_add(struct mlx5e_priv
*priv
,
561 struct mlx5e_tc_flow
*flow
,
562 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
564 int peer_ifindex
= parse_attr
->mirred_ifindex
;
565 struct mlx5_hairpin_params params
;
566 struct mlx5_core_dev
*peer_mdev
;
567 struct mlx5e_hairpin_entry
*hpe
;
568 struct mlx5e_hairpin
*hp
;
575 peer_mdev
= mlx5e_hairpin_get_mdev(dev_net(priv
->netdev
), peer_ifindex
);
576 if (!MLX5_CAP_GEN(priv
->mdev
, hairpin
) || !MLX5_CAP_GEN(peer_mdev
, hairpin
)) {
577 netdev_warn(priv
->netdev
, "hairpin is not supported\n");
581 peer_id
= MLX5_CAP_GEN(peer_mdev
, vhca_id
);
582 err
= mlx5e_hairpin_get_prio(priv
, &parse_attr
->spec
, &match_prio
);
585 hpe
= mlx5e_hairpin_get(priv
, peer_id
, match_prio
);
589 hpe
= kzalloc(sizeof(*hpe
), GFP_KERNEL
);
593 INIT_LIST_HEAD(&hpe
->flows
);
594 hpe
->peer_vhca_id
= peer_id
;
595 hpe
->prio
= match_prio
;
597 params
.log_data_size
= 15;
598 params
.log_data_size
= min_t(u8
, params
.log_data_size
,
599 MLX5_CAP_GEN(priv
->mdev
, log_max_hairpin_wq_data_sz
));
600 params
.log_data_size
= max_t(u8
, params
.log_data_size
,
601 MLX5_CAP_GEN(priv
->mdev
, log_min_hairpin_wq_data_sz
));
602 params
.q_counter
= priv
->q_counter
;
604 /* set hairpin pair per each 50Gbs share of the link */
605 mlx5e_get_max_linkspeed(priv
->mdev
, &link_speed
);
606 link_speed
= max_t(u32
, link_speed
, 50000);
607 link_speed64
= link_speed
;
608 do_div(link_speed64
, 50000);
609 params
.num_channels
= link_speed64
;
611 hp
= mlx5e_hairpin_create(priv
, ¶ms
, peer_ifindex
);
614 goto create_hairpin_err
;
617 netdev_dbg(priv
->netdev
, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d log data size %d\n",
618 hp
->tirn
, hp
->pair
->rqn
[0], hp
->pair
->peer_mdev
->priv
.name
,
619 hp
->pair
->sqn
[0], match_prio
, params
.log_data_size
);
622 hash_add(priv
->fs
.tc
.hairpin_tbl
, &hpe
->hairpin_hlist
,
623 hash_hairpin_info(peer_id
, match_prio
));
626 if (hpe
->hp
->num_channels
> 1) {
627 flow
->flags
|= MLX5E_TC_FLOW_HAIRPIN_RSS
;
628 flow
->nic_attr
->hairpin_ft
= hpe
->hp
->ttc
.ft
.t
;
630 flow
->nic_attr
->hairpin_tirn
= hpe
->hp
->tirn
;
632 list_add(&flow
->hairpin
, &hpe
->flows
);
641 static void mlx5e_hairpin_flow_del(struct mlx5e_priv
*priv
,
642 struct mlx5e_tc_flow
*flow
)
644 struct list_head
*next
= flow
->hairpin
.next
;
646 list_del(&flow
->hairpin
);
648 /* no more hairpin flows for us, release the hairpin pair */
649 if (list_empty(next
)) {
650 struct mlx5e_hairpin_entry
*hpe
;
652 hpe
= list_entry(next
, struct mlx5e_hairpin_entry
, flows
);
654 netdev_dbg(priv
->netdev
, "del hairpin: peer %s\n",
655 hpe
->hp
->pair
->peer_mdev
->priv
.name
);
657 mlx5e_hairpin_destroy(hpe
->hp
);
658 hash_del(&hpe
->hairpin_hlist
);
663 static struct mlx5_flow_handle
*
664 mlx5e_tc_add_nic_flow(struct mlx5e_priv
*priv
,
665 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
666 struct mlx5e_tc_flow
*flow
)
668 struct mlx5_nic_flow_attr
*attr
= flow
->nic_attr
;
669 struct mlx5_core_dev
*dev
= priv
->mdev
;
670 struct mlx5_flow_destination dest
[2] = {};
671 struct mlx5_flow_act flow_act
= {
672 .action
= attr
->action
,
673 .flow_tag
= attr
->flow_tag
,
676 struct mlx5_fc
*counter
= NULL
;
677 struct mlx5_flow_handle
*rule
;
678 bool table_created
= false;
679 int err
, dest_ix
= 0;
681 if (flow
->flags
& MLX5E_TC_FLOW_HAIRPIN
) {
682 err
= mlx5e_hairpin_flow_add(priv
, flow
, parse_attr
);
685 goto err_add_hairpin_flow
;
687 if (flow
->flags
& MLX5E_TC_FLOW_HAIRPIN_RSS
) {
688 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
689 dest
[dest_ix
].ft
= attr
->hairpin_ft
;
691 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
692 dest
[dest_ix
].tir_num
= attr
->hairpin_tirn
;
695 } else if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
696 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
697 dest
[dest_ix
].ft
= priv
->fs
.vlan
.ft
.t
;
701 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
702 counter
= mlx5_fc_create(dev
, true);
703 if (IS_ERR(counter
)) {
704 rule
= ERR_CAST(counter
);
707 dest
[dest_ix
].type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
708 dest
[dest_ix
].counter
= counter
;
712 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
) {
713 err
= mlx5e_attach_mod_hdr(priv
, flow
, parse_attr
);
714 flow_act
.modify_id
= attr
->mod_hdr_id
;
715 kfree(parse_attr
->mod_hdr_actions
);
718 goto err_create_mod_hdr_id
;
722 if (IS_ERR_OR_NULL(priv
->fs
.tc
.t
)) {
723 int tc_grp_size
, tc_tbl_size
;
724 u32 max_flow_counter
;
726 max_flow_counter
= (MLX5_CAP_GEN(dev
, max_flow_counter_31_16
) << 16) |
727 MLX5_CAP_GEN(dev
, max_flow_counter_15_0
);
729 tc_grp_size
= min_t(int, max_flow_counter
, MLX5E_TC_TABLE_MAX_GROUP_SIZE
);
731 tc_tbl_size
= min_t(int, tc_grp_size
* MLX5E_TC_TABLE_NUM_GROUPS
,
732 BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev
, log_max_ft_size
)));
735 mlx5_create_auto_grouped_flow_table(priv
->fs
.ns
,
738 MLX5E_TC_TABLE_NUM_GROUPS
,
739 MLX5E_TC_FT_LEVEL
, 0);
740 if (IS_ERR(priv
->fs
.tc
.t
)) {
741 netdev_err(priv
->netdev
,
742 "Failed to create tc offload table\n");
743 rule
= ERR_CAST(priv
->fs
.tc
.t
);
747 table_created
= true;
750 parse_attr
->spec
.match_criteria_enable
= MLX5_MATCH_OUTER_HEADERS
;
751 rule
= mlx5_add_flow_rules(priv
->fs
.tc
.t
, &parse_attr
->spec
,
752 &flow_act
, dest
, dest_ix
);
761 mlx5_destroy_flow_table(priv
->fs
.tc
.t
);
762 priv
->fs
.tc
.t
= NULL
;
765 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
766 mlx5e_detach_mod_hdr(priv
, flow
);
767 err_create_mod_hdr_id
:
768 mlx5_fc_destroy(dev
, counter
);
770 if (flow
->flags
& MLX5E_TC_FLOW_HAIRPIN
)
771 mlx5e_hairpin_flow_del(priv
, flow
);
772 err_add_hairpin_flow
:
776 static void mlx5e_tc_del_nic_flow(struct mlx5e_priv
*priv
,
777 struct mlx5e_tc_flow
*flow
)
779 struct mlx5_nic_flow_attr
*attr
= flow
->nic_attr
;
780 struct mlx5_fc
*counter
= NULL
;
782 counter
= mlx5_flow_rule_counter(flow
->rule
);
783 mlx5_del_flow_rules(flow
->rule
);
784 mlx5_fc_destroy(priv
->mdev
, counter
);
786 if (!mlx5e_tc_num_filters(priv
) && (priv
->fs
.tc
.t
)) {
787 mlx5_destroy_flow_table(priv
->fs
.tc
.t
);
788 priv
->fs
.tc
.t
= NULL
;
791 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
792 mlx5e_detach_mod_hdr(priv
, flow
);
794 if (flow
->flags
& MLX5E_TC_FLOW_HAIRPIN
)
795 mlx5e_hairpin_flow_del(priv
, flow
);
798 static void mlx5e_detach_encap(struct mlx5e_priv
*priv
,
799 struct mlx5e_tc_flow
*flow
);
801 static int mlx5e_attach_encap(struct mlx5e_priv
*priv
,
802 struct ip_tunnel_info
*tun_info
,
803 struct net_device
*mirred_dev
,
804 struct net_device
**encap_dev
,
805 struct mlx5e_tc_flow
*flow
);
807 static struct mlx5_flow_handle
*
808 mlx5e_tc_add_fdb_flow(struct mlx5e_priv
*priv
,
809 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
810 struct mlx5e_tc_flow
*flow
)
812 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
813 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
814 struct net_device
*out_dev
, *encap_dev
= NULL
;
815 struct mlx5_flow_handle
*rule
= NULL
;
816 struct mlx5e_rep_priv
*rpriv
;
817 struct mlx5e_priv
*out_priv
;
820 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_ENCAP
) {
821 out_dev
= __dev_get_by_index(dev_net(priv
->netdev
),
822 attr
->parse_attr
->mirred_ifindex
);
823 err
= mlx5e_attach_encap(priv
, &parse_attr
->tun_info
,
824 out_dev
, &encap_dev
, flow
);
828 goto err_attach_encap
;
830 out_priv
= netdev_priv(encap_dev
);
831 rpriv
= out_priv
->ppriv
;
832 attr
->out_rep
= rpriv
->rep
;
835 err
= mlx5_eswitch_add_vlan_action(esw
, attr
);
841 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
) {
842 err
= mlx5e_attach_mod_hdr(priv
, flow
, parse_attr
);
843 kfree(parse_attr
->mod_hdr_actions
);
850 /* we get here if (1) there's no error (rule being null) or when
851 * (2) there's an encap action and we're on -EAGAIN (no valid neigh)
853 if (rule
!= ERR_PTR(-EAGAIN
)) {
854 rule
= mlx5_eswitch_add_offloaded_rule(esw
, &parse_attr
->spec
, attr
);
861 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
862 mlx5e_detach_mod_hdr(priv
, flow
);
864 mlx5_eswitch_del_vlan_action(esw
, attr
);
866 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_ENCAP
)
867 mlx5e_detach_encap(priv
, flow
);
872 static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv
*priv
,
873 struct mlx5e_tc_flow
*flow
)
875 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
876 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
878 if (flow
->flags
& MLX5E_TC_FLOW_OFFLOADED
) {
879 flow
->flags
&= ~MLX5E_TC_FLOW_OFFLOADED
;
880 mlx5_eswitch_del_offloaded_rule(esw
, flow
->rule
, attr
);
883 mlx5_eswitch_del_vlan_action(esw
, attr
);
885 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_ENCAP
) {
886 mlx5e_detach_encap(priv
, flow
);
887 kvfree(attr
->parse_attr
);
890 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
891 mlx5e_detach_mod_hdr(priv
, flow
);
894 void mlx5e_tc_encap_flows_add(struct mlx5e_priv
*priv
,
895 struct mlx5e_encap_entry
*e
)
897 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
898 struct mlx5_esw_flow_attr
*esw_attr
;
899 struct mlx5e_tc_flow
*flow
;
902 err
= mlx5_encap_alloc(priv
->mdev
, e
->tunnel_type
,
903 e
->encap_size
, e
->encap_header
,
906 mlx5_core_warn(priv
->mdev
, "Failed to offload cached encapsulation header, %d\n",
910 e
->flags
|= MLX5_ENCAP_ENTRY_VALID
;
911 mlx5e_rep_queue_neigh_stats_work(priv
);
913 list_for_each_entry(flow
, &e
->flows
, encap
) {
914 esw_attr
= flow
->esw_attr
;
915 esw_attr
->encap_id
= e
->encap_id
;
916 flow
->rule
= mlx5_eswitch_add_offloaded_rule(esw
, &esw_attr
->parse_attr
->spec
, esw_attr
);
917 if (IS_ERR(flow
->rule
)) {
918 err
= PTR_ERR(flow
->rule
);
919 mlx5_core_warn(priv
->mdev
, "Failed to update cached encapsulation flow, %d\n",
923 flow
->flags
|= MLX5E_TC_FLOW_OFFLOADED
;
927 void mlx5e_tc_encap_flows_del(struct mlx5e_priv
*priv
,
928 struct mlx5e_encap_entry
*e
)
930 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
931 struct mlx5e_tc_flow
*flow
;
933 list_for_each_entry(flow
, &e
->flows
, encap
) {
934 if (flow
->flags
& MLX5E_TC_FLOW_OFFLOADED
) {
935 flow
->flags
&= ~MLX5E_TC_FLOW_OFFLOADED
;
936 mlx5_eswitch_del_offloaded_rule(esw
, flow
->rule
, flow
->esw_attr
);
940 if (e
->flags
& MLX5_ENCAP_ENTRY_VALID
) {
941 e
->flags
&= ~MLX5_ENCAP_ENTRY_VALID
;
942 mlx5_encap_dealloc(priv
->mdev
, e
->encap_id
);
946 void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry
*nhe
)
948 struct mlx5e_neigh
*m_neigh
= &nhe
->m_neigh
;
949 u64 bytes
, packets
, lastuse
= 0;
950 struct mlx5e_tc_flow
*flow
;
951 struct mlx5e_encap_entry
*e
;
952 struct mlx5_fc
*counter
;
953 struct neigh_table
*tbl
;
954 bool neigh_used
= false;
957 if (m_neigh
->family
== AF_INET
)
959 #if IS_ENABLED(CONFIG_IPV6)
960 else if (m_neigh
->family
== AF_INET6
)
961 tbl
= ipv6_stub
->nd_tbl
;
966 list_for_each_entry(e
, &nhe
->encap_list
, encap_list
) {
967 if (!(e
->flags
& MLX5_ENCAP_ENTRY_VALID
))
969 list_for_each_entry(flow
, &e
->flows
, encap
) {
970 if (flow
->flags
& MLX5E_TC_FLOW_OFFLOADED
) {
971 counter
= mlx5_flow_rule_counter(flow
->rule
);
972 mlx5_fc_query_cached(counter
, &bytes
, &packets
, &lastuse
);
973 if (time_after((unsigned long)lastuse
, nhe
->reported_lastuse
)) {
982 nhe
->reported_lastuse
= jiffies
;
984 /* find the relevant neigh according to the cached device and
987 n
= neigh_lookup(tbl
, &m_neigh
->dst_ip
, m_neigh
->dev
);
989 WARN(1, "The neighbour already freed\n");
993 neigh_event_send(n
, NULL
);
998 static void mlx5e_detach_encap(struct mlx5e_priv
*priv
,
999 struct mlx5e_tc_flow
*flow
)
1001 struct list_head
*next
= flow
->encap
.next
;
1003 list_del(&flow
->encap
);
1004 if (list_empty(next
)) {
1005 struct mlx5e_encap_entry
*e
;
1007 e
= list_entry(next
, struct mlx5e_encap_entry
, flows
);
1008 mlx5e_rep_encap_entry_detach(netdev_priv(e
->out_dev
), e
);
1010 if (e
->flags
& MLX5_ENCAP_ENTRY_VALID
)
1011 mlx5_encap_dealloc(priv
->mdev
, e
->encap_id
);
1013 hash_del_rcu(&e
->encap_hlist
);
1014 kfree(e
->encap_header
);
1019 static void mlx5e_tc_del_flow(struct mlx5e_priv
*priv
,
1020 struct mlx5e_tc_flow
*flow
)
1022 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
)
1023 mlx5e_tc_del_fdb_flow(priv
, flow
);
1025 mlx5e_tc_del_nic_flow(priv
, flow
);
1028 static void parse_vxlan_attr(struct mlx5_flow_spec
*spec
,
1029 struct tc_cls_flower_offload
*f
)
1031 void *headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1033 void *headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1035 void *misc_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1037 void *misc_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1040 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, ip_protocol
);
1041 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_protocol
, IPPROTO_UDP
);
1043 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_ENC_KEYID
)) {
1044 struct flow_dissector_key_keyid
*key
=
1045 skb_flow_dissector_target(f
->dissector
,
1046 FLOW_DISSECTOR_KEY_ENC_KEYID
,
1048 struct flow_dissector_key_keyid
*mask
=
1049 skb_flow_dissector_target(f
->dissector
,
1050 FLOW_DISSECTOR_KEY_ENC_KEYID
,
1052 MLX5_SET(fte_match_set_misc
, misc_c
, vxlan_vni
,
1053 be32_to_cpu(mask
->keyid
));
1054 MLX5_SET(fte_match_set_misc
, misc_v
, vxlan_vni
,
1055 be32_to_cpu(key
->keyid
));
1059 static int parse_tunnel_attr(struct mlx5e_priv
*priv
,
1060 struct mlx5_flow_spec
*spec
,
1061 struct tc_cls_flower_offload
*f
)
1063 void *headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1065 void *headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1068 struct flow_dissector_key_control
*enc_control
=
1069 skb_flow_dissector_target(f
->dissector
,
1070 FLOW_DISSECTOR_KEY_ENC_CONTROL
,
1073 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_ENC_PORTS
)) {
1074 struct flow_dissector_key_ports
*key
=
1075 skb_flow_dissector_target(f
->dissector
,
1076 FLOW_DISSECTOR_KEY_ENC_PORTS
,
1078 struct flow_dissector_key_ports
*mask
=
1079 skb_flow_dissector_target(f
->dissector
,
1080 FLOW_DISSECTOR_KEY_ENC_PORTS
,
1082 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
1083 struct mlx5e_rep_priv
*uplink_rpriv
= mlx5_eswitch_get_uplink_priv(esw
, REP_ETH
);
1084 struct net_device
*up_dev
= uplink_rpriv
->netdev
;
1085 struct mlx5e_priv
*up_priv
= netdev_priv(up_dev
);
1087 /* Full udp dst port must be given */
1088 if (memchr_inv(&mask
->dst
, 0xff, sizeof(mask
->dst
)))
1089 goto vxlan_match_offload_err
;
1091 if (mlx5e_vxlan_lookup_port(up_priv
, be16_to_cpu(key
->dst
)) &&
1092 MLX5_CAP_ESW(priv
->mdev
, vxlan_encap_decap
))
1093 parse_vxlan_attr(spec
, f
);
1095 netdev_warn(priv
->netdev
,
1096 "%d isn't an offloaded vxlan udp dport\n", be16_to_cpu(key
->dst
));
1100 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1101 udp_dport
, ntohs(mask
->dst
));
1102 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1103 udp_dport
, ntohs(key
->dst
));
1105 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1106 udp_sport
, ntohs(mask
->src
));
1107 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1108 udp_sport
, ntohs(key
->src
));
1109 } else { /* udp dst port must be given */
1110 vxlan_match_offload_err
:
1111 netdev_warn(priv
->netdev
,
1112 "IP tunnel decap offload supported only for vxlan, must set UDP dport\n");
1116 if (enc_control
->addr_type
== FLOW_DISSECTOR_KEY_IPV4_ADDRS
) {
1117 struct flow_dissector_key_ipv4_addrs
*key
=
1118 skb_flow_dissector_target(f
->dissector
,
1119 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
,
1121 struct flow_dissector_key_ipv4_addrs
*mask
=
1122 skb_flow_dissector_target(f
->dissector
,
1123 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
,
1125 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1126 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
,
1128 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1129 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
,
1132 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1133 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
,
1135 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1136 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
,
1139 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, ethertype
);
1140 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ethertype
, ETH_P_IP
);
1141 } else if (enc_control
->addr_type
== FLOW_DISSECTOR_KEY_IPV6_ADDRS
) {
1142 struct flow_dissector_key_ipv6_addrs
*key
=
1143 skb_flow_dissector_target(f
->dissector
,
1144 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
,
1146 struct flow_dissector_key_ipv6_addrs
*mask
=
1147 skb_flow_dissector_target(f
->dissector
,
1148 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
,
1151 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1152 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
1153 &mask
->src
, MLX5_FLD_SZ_BYTES(ipv6_layout
, ipv6
));
1154 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1155 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
1156 &key
->src
, MLX5_FLD_SZ_BYTES(ipv6_layout
, ipv6
));
1158 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1159 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
1160 &mask
->dst
, MLX5_FLD_SZ_BYTES(ipv6_layout
, ipv6
));
1161 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1162 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
1163 &key
->dst
, MLX5_FLD_SZ_BYTES(ipv6_layout
, ipv6
));
1165 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, ethertype
);
1166 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ethertype
, ETH_P_IPV6
);
1169 /* Enforce DMAC when offloading incoming tunneled flows.
1170 * Flow counters require a match on the DMAC.
1172 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, dmac_47_16
);
1173 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4
, headers_c
, dmac_15_0
);
1174 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1175 dmac_47_16
), priv
->netdev
->dev_addr
);
1177 /* let software handle IP fragments */
1178 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, frag
, 1);
1179 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, frag
, 0);
1184 static int __parse_cls_flower(struct mlx5e_priv
*priv
,
1185 struct mlx5_flow_spec
*spec
,
1186 struct tc_cls_flower_offload
*f
,
1189 void *headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1191 void *headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1196 *min_inline
= MLX5_INLINE_MODE_L2
;
1198 if (f
->dissector
->used_keys
&
1199 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL
) |
1200 BIT(FLOW_DISSECTOR_KEY_BASIC
) |
1201 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS
) |
1202 BIT(FLOW_DISSECTOR_KEY_VLAN
) |
1203 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS
) |
1204 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS
) |
1205 BIT(FLOW_DISSECTOR_KEY_PORTS
) |
1206 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID
) |
1207 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
) |
1208 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
) |
1209 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS
) |
1210 BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL
) |
1211 BIT(FLOW_DISSECTOR_KEY_TCP
) |
1212 BIT(FLOW_DISSECTOR_KEY_IP
))) {
1213 netdev_warn(priv
->netdev
, "Unsupported key used: 0x%x\n",
1214 f
->dissector
->used_keys
);
1218 if ((dissector_uses_key(f
->dissector
,
1219 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
) ||
1220 dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_ENC_KEYID
) ||
1221 dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_ENC_PORTS
)) &&
1222 dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_ENC_CONTROL
)) {
1223 struct flow_dissector_key_control
*key
=
1224 skb_flow_dissector_target(f
->dissector
,
1225 FLOW_DISSECTOR_KEY_ENC_CONTROL
,
1227 switch (key
->addr_type
) {
1228 case FLOW_DISSECTOR_KEY_IPV4_ADDRS
:
1229 case FLOW_DISSECTOR_KEY_IPV6_ADDRS
:
1230 if (parse_tunnel_attr(priv
, spec
, f
))
1237 /* In decap flow, header pointers should point to the inner
1238 * headers, outer header were already set by parse_tunnel_attr
1240 headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1242 headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1246 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_CONTROL
)) {
1247 struct flow_dissector_key_control
*key
=
1248 skb_flow_dissector_target(f
->dissector
,
1249 FLOW_DISSECTOR_KEY_CONTROL
,
1252 struct flow_dissector_key_control
*mask
=
1253 skb_flow_dissector_target(f
->dissector
,
1254 FLOW_DISSECTOR_KEY_CONTROL
,
1256 addr_type
= key
->addr_type
;
1258 if (mask
->flags
& FLOW_DIS_IS_FRAGMENT
) {
1259 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, frag
, 1);
1260 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, frag
,
1261 key
->flags
& FLOW_DIS_IS_FRAGMENT
);
1263 /* the HW doesn't need L3 inline to match on frag=no */
1264 if (key
->flags
& FLOW_DIS_IS_FRAGMENT
)
1265 *min_inline
= MLX5_INLINE_MODE_IP
;
1269 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_BASIC
)) {
1270 struct flow_dissector_key_basic
*key
=
1271 skb_flow_dissector_target(f
->dissector
,
1272 FLOW_DISSECTOR_KEY_BASIC
,
1274 struct flow_dissector_key_basic
*mask
=
1275 skb_flow_dissector_target(f
->dissector
,
1276 FLOW_DISSECTOR_KEY_BASIC
,
1278 ip_proto
= key
->ip_proto
;
1280 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ethertype
,
1281 ntohs(mask
->n_proto
));
1282 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ethertype
,
1283 ntohs(key
->n_proto
));
1285 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_protocol
,
1287 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_protocol
,
1291 *min_inline
= MLX5_INLINE_MODE_IP
;
1294 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_ETH_ADDRS
)) {
1295 struct flow_dissector_key_eth_addrs
*key
=
1296 skb_flow_dissector_target(f
->dissector
,
1297 FLOW_DISSECTOR_KEY_ETH_ADDRS
,
1299 struct flow_dissector_key_eth_addrs
*mask
=
1300 skb_flow_dissector_target(f
->dissector
,
1301 FLOW_DISSECTOR_KEY_ETH_ADDRS
,
1304 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1307 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1311 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1314 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1319 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_VLAN
)) {
1320 struct flow_dissector_key_vlan
*key
=
1321 skb_flow_dissector_target(f
->dissector
,
1322 FLOW_DISSECTOR_KEY_VLAN
,
1324 struct flow_dissector_key_vlan
*mask
=
1325 skb_flow_dissector_target(f
->dissector
,
1326 FLOW_DISSECTOR_KEY_VLAN
,
1328 if (mask
->vlan_id
|| mask
->vlan_priority
) {
1329 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, cvlan_tag
, 1);
1330 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, cvlan_tag
, 1);
1332 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, first_vid
, mask
->vlan_id
);
1333 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, first_vid
, key
->vlan_id
);
1335 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, first_prio
, mask
->vlan_priority
);
1336 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, first_prio
, key
->vlan_priority
);
1340 if (addr_type
== FLOW_DISSECTOR_KEY_IPV4_ADDRS
) {
1341 struct flow_dissector_key_ipv4_addrs
*key
=
1342 skb_flow_dissector_target(f
->dissector
,
1343 FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
1345 struct flow_dissector_key_ipv4_addrs
*mask
=
1346 skb_flow_dissector_target(f
->dissector
,
1347 FLOW_DISSECTOR_KEY_IPV4_ADDRS
,
1350 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1351 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
1352 &mask
->src
, sizeof(mask
->src
));
1353 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1354 src_ipv4_src_ipv6
.ipv4_layout
.ipv4
),
1355 &key
->src
, sizeof(key
->src
));
1356 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1357 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
1358 &mask
->dst
, sizeof(mask
->dst
));
1359 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1360 dst_ipv4_dst_ipv6
.ipv4_layout
.ipv4
),
1361 &key
->dst
, sizeof(key
->dst
));
1363 if (mask
->src
|| mask
->dst
)
1364 *min_inline
= MLX5_INLINE_MODE_IP
;
1367 if (addr_type
== FLOW_DISSECTOR_KEY_IPV6_ADDRS
) {
1368 struct flow_dissector_key_ipv6_addrs
*key
=
1369 skb_flow_dissector_target(f
->dissector
,
1370 FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
1372 struct flow_dissector_key_ipv6_addrs
*mask
=
1373 skb_flow_dissector_target(f
->dissector
,
1374 FLOW_DISSECTOR_KEY_IPV6_ADDRS
,
1377 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1378 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
1379 &mask
->src
, sizeof(mask
->src
));
1380 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1381 src_ipv4_src_ipv6
.ipv6_layout
.ipv6
),
1382 &key
->src
, sizeof(key
->src
));
1384 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_c
,
1385 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
1386 &mask
->dst
, sizeof(mask
->dst
));
1387 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4
, headers_v
,
1388 dst_ipv4_dst_ipv6
.ipv6_layout
.ipv6
),
1389 &key
->dst
, sizeof(key
->dst
));
1391 if (ipv6_addr_type(&mask
->src
) != IPV6_ADDR_ANY
||
1392 ipv6_addr_type(&mask
->dst
) != IPV6_ADDR_ANY
)
1393 *min_inline
= MLX5_INLINE_MODE_IP
;
1396 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_IP
)) {
1397 struct flow_dissector_key_ip
*key
=
1398 skb_flow_dissector_target(f
->dissector
,
1399 FLOW_DISSECTOR_KEY_IP
,
1401 struct flow_dissector_key_ip
*mask
=
1402 skb_flow_dissector_target(f
->dissector
,
1403 FLOW_DISSECTOR_KEY_IP
,
1406 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_ecn
, mask
->tos
& 0x3);
1407 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_ecn
, key
->tos
& 0x3);
1409 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ip_dscp
, mask
->tos
>> 2);
1410 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ip_dscp
, key
->tos
>> 2);
1412 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, ttl_hoplimit
, mask
->ttl
);
1413 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, ttl_hoplimit
, key
->ttl
);
1416 !MLX5_CAP_ESW_FLOWTABLE_FDB(priv
->mdev
,
1417 ft_field_support
.outer_ipv4_ttl
))
1420 if (mask
->tos
|| mask
->ttl
)
1421 *min_inline
= MLX5_INLINE_MODE_IP
;
1424 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_PORTS
)) {
1425 struct flow_dissector_key_ports
*key
=
1426 skb_flow_dissector_target(f
->dissector
,
1427 FLOW_DISSECTOR_KEY_PORTS
,
1429 struct flow_dissector_key_ports
*mask
=
1430 skb_flow_dissector_target(f
->dissector
,
1431 FLOW_DISSECTOR_KEY_PORTS
,
1435 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1436 tcp_sport
, ntohs(mask
->src
));
1437 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1438 tcp_sport
, ntohs(key
->src
));
1440 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1441 tcp_dport
, ntohs(mask
->dst
));
1442 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1443 tcp_dport
, ntohs(key
->dst
));
1447 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1448 udp_sport
, ntohs(mask
->src
));
1449 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1450 udp_sport
, ntohs(key
->src
));
1452 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
,
1453 udp_dport
, ntohs(mask
->dst
));
1454 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
,
1455 udp_dport
, ntohs(key
->dst
));
1458 netdev_err(priv
->netdev
,
1459 "Only UDP and TCP transport are supported\n");
1463 if (mask
->src
|| mask
->dst
)
1464 *min_inline
= MLX5_INLINE_MODE_TCP_UDP
;
1467 if (dissector_uses_key(f
->dissector
, FLOW_DISSECTOR_KEY_TCP
)) {
1468 struct flow_dissector_key_tcp
*key
=
1469 skb_flow_dissector_target(f
->dissector
,
1470 FLOW_DISSECTOR_KEY_TCP
,
1472 struct flow_dissector_key_tcp
*mask
=
1473 skb_flow_dissector_target(f
->dissector
,
1474 FLOW_DISSECTOR_KEY_TCP
,
1477 MLX5_SET(fte_match_set_lyr_2_4
, headers_c
, tcp_flags
,
1478 ntohs(mask
->flags
));
1479 MLX5_SET(fte_match_set_lyr_2_4
, headers_v
, tcp_flags
,
1483 *min_inline
= MLX5_INLINE_MODE_TCP_UDP
;
1489 static int parse_cls_flower(struct mlx5e_priv
*priv
,
1490 struct mlx5e_tc_flow
*flow
,
1491 struct mlx5_flow_spec
*spec
,
1492 struct tc_cls_flower_offload
*f
)
1494 struct mlx5_core_dev
*dev
= priv
->mdev
;
1495 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
1496 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
1497 struct mlx5_eswitch_rep
*rep
;
1501 err
= __parse_cls_flower(priv
, spec
, f
, &min_inline
);
1503 if (!err
&& (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
)) {
1505 if (rep
->vport
!= FDB_UPLINK_VPORT
&&
1506 (esw
->offloads
.inline_mode
!= MLX5_INLINE_MODE_NONE
&&
1507 esw
->offloads
.inline_mode
< min_inline
)) {
1508 netdev_warn(priv
->netdev
,
1509 "Flow is not offloaded due to min inline setting, required %d actual %d\n",
1510 min_inline
, esw
->offloads
.inline_mode
);
1518 struct pedit_headers
{
1526 static int pedit_header_offsets
[] = {
1527 [TCA_PEDIT_KEY_EX_HDR_TYPE_ETH
] = offsetof(struct pedit_headers
, eth
),
1528 [TCA_PEDIT_KEY_EX_HDR_TYPE_IP4
] = offsetof(struct pedit_headers
, ip4
),
1529 [TCA_PEDIT_KEY_EX_HDR_TYPE_IP6
] = offsetof(struct pedit_headers
, ip6
),
1530 [TCA_PEDIT_KEY_EX_HDR_TYPE_TCP
] = offsetof(struct pedit_headers
, tcp
),
1531 [TCA_PEDIT_KEY_EX_HDR_TYPE_UDP
] = offsetof(struct pedit_headers
, udp
),
1534 #define pedit_header(_ph, _htype) ((void *)(_ph) + pedit_header_offsets[_htype])
1536 static int set_pedit_val(u8 hdr_type
, u32 mask
, u32 val
, u32 offset
,
1537 struct pedit_headers
*masks
,
1538 struct pedit_headers
*vals
)
1540 u32
*curr_pmask
, *curr_pval
;
1542 if (hdr_type
>= __PEDIT_HDR_TYPE_MAX
)
1545 curr_pmask
= (u32
*)(pedit_header(masks
, hdr_type
) + offset
);
1546 curr_pval
= (u32
*)(pedit_header(vals
, hdr_type
) + offset
);
1548 if (*curr_pmask
& mask
) /* disallow acting twice on the same location */
1551 *curr_pmask
|= mask
;
1552 *curr_pval
|= (val
& mask
);
1560 struct mlx5_fields
{
1566 #define OFFLOAD(fw_field, size, field, off) \
1567 {MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, size, offsetof(struct pedit_headers, field) + (off)}
1569 static struct mlx5_fields fields
[] = {
1570 OFFLOAD(DMAC_47_16
, 4, eth
.h_dest
[0], 0),
1571 OFFLOAD(DMAC_47_16
, 4, eth
.h_dest
[0], 0),
1572 OFFLOAD(DMAC_15_0
, 2, eth
.h_dest
[4], 0),
1573 OFFLOAD(SMAC_47_16
, 4, eth
.h_source
[0], 0),
1574 OFFLOAD(SMAC_15_0
, 2, eth
.h_source
[4], 0),
1575 OFFLOAD(ETHERTYPE
, 2, eth
.h_proto
, 0),
1577 OFFLOAD(IP_TTL
, 1, ip4
.ttl
, 0),
1578 OFFLOAD(SIPV4
, 4, ip4
.saddr
, 0),
1579 OFFLOAD(DIPV4
, 4, ip4
.daddr
, 0),
1581 OFFLOAD(SIPV6_127_96
, 4, ip6
.saddr
.s6_addr32
[0], 0),
1582 OFFLOAD(SIPV6_95_64
, 4, ip6
.saddr
.s6_addr32
[1], 0),
1583 OFFLOAD(SIPV6_63_32
, 4, ip6
.saddr
.s6_addr32
[2], 0),
1584 OFFLOAD(SIPV6_31_0
, 4, ip6
.saddr
.s6_addr32
[3], 0),
1585 OFFLOAD(DIPV6_127_96
, 4, ip6
.daddr
.s6_addr32
[0], 0),
1586 OFFLOAD(DIPV6_95_64
, 4, ip6
.daddr
.s6_addr32
[1], 0),
1587 OFFLOAD(DIPV6_63_32
, 4, ip6
.daddr
.s6_addr32
[2], 0),
1588 OFFLOAD(DIPV6_31_0
, 4, ip6
.daddr
.s6_addr32
[3], 0),
1589 OFFLOAD(IPV6_HOPLIMIT
, 1, ip6
.hop_limit
, 0),
1591 OFFLOAD(TCP_SPORT
, 2, tcp
.source
, 0),
1592 OFFLOAD(TCP_DPORT
, 2, tcp
.dest
, 0),
1593 OFFLOAD(TCP_FLAGS
, 1, tcp
.ack_seq
, 5),
1595 OFFLOAD(UDP_SPORT
, 2, udp
.source
, 0),
1596 OFFLOAD(UDP_DPORT
, 2, udp
.dest
, 0),
1599 /* On input attr->num_mod_hdr_actions tells how many HW actions can be parsed at
1600 * max from the SW pedit action. On success, it says how many HW actions were
1603 static int offload_pedit_fields(struct pedit_headers
*masks
,
1604 struct pedit_headers
*vals
,
1605 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
1607 struct pedit_headers
*set_masks
, *add_masks
, *set_vals
, *add_vals
;
1608 int i
, action_size
, nactions
, max_actions
, first
, last
, next_z
;
1609 void *s_masks_p
, *a_masks_p
, *vals_p
;
1610 struct mlx5_fields
*f
;
1611 u8 cmd
, field_bsize
;
1618 set_masks
= &masks
[TCA_PEDIT_KEY_EX_CMD_SET
];
1619 add_masks
= &masks
[TCA_PEDIT_KEY_EX_CMD_ADD
];
1620 set_vals
= &vals
[TCA_PEDIT_KEY_EX_CMD_SET
];
1621 add_vals
= &vals
[TCA_PEDIT_KEY_EX_CMD_ADD
];
1623 action_size
= MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto
);
1624 action
= parse_attr
->mod_hdr_actions
;
1625 max_actions
= parse_attr
->num_mod_hdr_actions
;
1628 for (i
= 0; i
< ARRAY_SIZE(fields
); i
++) {
1630 /* avoid seeing bits set from previous iterations */
1634 s_masks_p
= (void *)set_masks
+ f
->offset
;
1635 a_masks_p
= (void *)add_masks
+ f
->offset
;
1637 memcpy(&s_mask
, s_masks_p
, f
->size
);
1638 memcpy(&a_mask
, a_masks_p
, f
->size
);
1640 if (!s_mask
&& !a_mask
) /* nothing to offload here */
1643 if (s_mask
&& a_mask
) {
1644 printk(KERN_WARNING
"mlx5: can't set and add to the same HW field (%x)\n", f
->field
);
1648 if (nactions
== max_actions
) {
1649 printk(KERN_WARNING
"mlx5: parsed %d pedit actions, can't do more\n", nactions
);
1654 cmd
= MLX5_ACTION_TYPE_SET
;
1656 vals_p
= (void *)set_vals
+ f
->offset
;
1657 /* clear to denote we consumed this field */
1658 memset(s_masks_p
, 0, f
->size
);
1660 cmd
= MLX5_ACTION_TYPE_ADD
;
1662 vals_p
= (void *)add_vals
+ f
->offset
;
1663 /* clear to denote we consumed this field */
1664 memset(a_masks_p
, 0, f
->size
);
1667 field_bsize
= f
->size
* BITS_PER_BYTE
;
1669 if (field_bsize
== 32) {
1670 mask_be32
= *(__be32
*)&mask
;
1671 mask
= (__force
unsigned long)cpu_to_le32(be32_to_cpu(mask_be32
));
1672 } else if (field_bsize
== 16) {
1673 mask_be16
= *(__be16
*)&mask
;
1674 mask
= (__force
unsigned long)cpu_to_le16(be16_to_cpu(mask_be16
));
1677 first
= find_first_bit(&mask
, field_bsize
);
1678 next_z
= find_next_zero_bit(&mask
, field_bsize
, first
);
1679 last
= find_last_bit(&mask
, field_bsize
);
1680 if (first
< next_z
&& next_z
< last
) {
1681 printk(KERN_WARNING
"mlx5: rewrite of few sub-fields (mask %lx) isn't offloaded\n",
1686 MLX5_SET(set_action_in
, action
, action_type
, cmd
);
1687 MLX5_SET(set_action_in
, action
, field
, f
->field
);
1689 if (cmd
== MLX5_ACTION_TYPE_SET
) {
1690 MLX5_SET(set_action_in
, action
, offset
, first
);
1691 /* length is num of bits to be written, zero means length of 32 */
1692 MLX5_SET(set_action_in
, action
, length
, (last
- first
+ 1));
1695 if (field_bsize
== 32)
1696 MLX5_SET(set_action_in
, action
, data
, ntohl(*(__be32
*)vals_p
) >> first
);
1697 else if (field_bsize
== 16)
1698 MLX5_SET(set_action_in
, action
, data
, ntohs(*(__be16
*)vals_p
) >> first
);
1699 else if (field_bsize
== 8)
1700 MLX5_SET(set_action_in
, action
, data
, *(u8
*)vals_p
>> first
);
1702 action
+= action_size
;
1706 parse_attr
->num_mod_hdr_actions
= nactions
;
1710 static int alloc_mod_hdr_actions(struct mlx5e_priv
*priv
,
1711 const struct tc_action
*a
, int namespace,
1712 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
1714 int nkeys
, action_size
, max_actions
;
1716 nkeys
= tcf_pedit_nkeys(a
);
1717 action_size
= MLX5_UN_SZ_BYTES(set_action_in_add_action_in_auto
);
1719 if (namespace == MLX5_FLOW_NAMESPACE_FDB
) /* FDB offloading */
1720 max_actions
= MLX5_CAP_ESW_FLOWTABLE_FDB(priv
->mdev
, max_modify_header_actions
);
1721 else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */
1722 max_actions
= MLX5_CAP_FLOWTABLE_NIC_RX(priv
->mdev
, max_modify_header_actions
);
1724 /* can get up to crazingly 16 HW actions in 32 bits pedit SW key */
1725 max_actions
= min(max_actions
, nkeys
* 16);
1727 parse_attr
->mod_hdr_actions
= kcalloc(max_actions
, action_size
, GFP_KERNEL
);
1728 if (!parse_attr
->mod_hdr_actions
)
1731 parse_attr
->num_mod_hdr_actions
= max_actions
;
1735 static const struct pedit_headers zero_masks
= {};
1737 static int parse_tc_pedit_action(struct mlx5e_priv
*priv
,
1738 const struct tc_action
*a
, int namespace,
1739 struct mlx5e_tc_flow_parse_attr
*parse_attr
)
1741 struct pedit_headers masks
[__PEDIT_CMD_MAX
], vals
[__PEDIT_CMD_MAX
], *cmd_masks
;
1742 int nkeys
, i
, err
= -EOPNOTSUPP
;
1743 u32 mask
, val
, offset
;
1746 nkeys
= tcf_pedit_nkeys(a
);
1748 memset(masks
, 0, sizeof(struct pedit_headers
) * __PEDIT_CMD_MAX
);
1749 memset(vals
, 0, sizeof(struct pedit_headers
) * __PEDIT_CMD_MAX
);
1751 for (i
= 0; i
< nkeys
; i
++) {
1752 htype
= tcf_pedit_htype(a
, i
);
1753 cmd
= tcf_pedit_cmd(a
, i
);
1754 err
= -EOPNOTSUPP
; /* can't be all optimistic */
1756 if (htype
== TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK
) {
1757 printk(KERN_WARNING
"mlx5: legacy pedit isn't offloaded\n");
1761 if (cmd
!= TCA_PEDIT_KEY_EX_CMD_SET
&& cmd
!= TCA_PEDIT_KEY_EX_CMD_ADD
) {
1762 printk(KERN_WARNING
"mlx5: pedit cmd %d isn't offloaded\n", cmd
);
1766 mask
= tcf_pedit_mask(a
, i
);
1767 val
= tcf_pedit_val(a
, i
);
1768 offset
= tcf_pedit_offset(a
, i
);
1770 err
= set_pedit_val(htype
, ~mask
, val
, offset
, &masks
[cmd
], &vals
[cmd
]);
1775 err
= alloc_mod_hdr_actions(priv
, a
, namespace, parse_attr
);
1779 err
= offload_pedit_fields(masks
, vals
, parse_attr
);
1781 goto out_dealloc_parsed_actions
;
1783 for (cmd
= 0; cmd
< __PEDIT_CMD_MAX
; cmd
++) {
1784 cmd_masks
= &masks
[cmd
];
1785 if (memcmp(cmd_masks
, &zero_masks
, sizeof(zero_masks
))) {
1786 printk(KERN_WARNING
"mlx5: attempt to offload an unsupported field (cmd %d)\n",
1788 print_hex_dump(KERN_WARNING
, "mask: ", DUMP_PREFIX_ADDRESS
,
1789 16, 1, cmd_masks
, sizeof(zero_masks
), true);
1791 goto out_dealloc_parsed_actions
;
1797 out_dealloc_parsed_actions
:
1798 kfree(parse_attr
->mod_hdr_actions
);
1803 static bool csum_offload_supported(struct mlx5e_priv
*priv
, u32 action
, u32 update_flags
)
1805 u32 prot_flags
= TCA_CSUM_UPDATE_FLAG_IPV4HDR
| TCA_CSUM_UPDATE_FLAG_TCP
|
1806 TCA_CSUM_UPDATE_FLAG_UDP
;
1808 /* The HW recalcs checksums only if re-writing headers */
1809 if (!(action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)) {
1810 netdev_warn(priv
->netdev
,
1811 "TC csum action is only offloaded with pedit\n");
1815 if (update_flags
& ~prot_flags
) {
1816 netdev_warn(priv
->netdev
,
1817 "can't offload TC csum action for some header/s - flags %#x\n",
1825 static bool modify_header_match_supported(struct mlx5_flow_spec
*spec
,
1826 struct tcf_exts
*exts
)
1828 const struct tc_action
*a
;
1829 bool modify_ip_header
;
1836 headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, outer_headers
);
1837 ethertype
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, ethertype
);
1839 /* for non-IP we only re-write MACs, so we're okay */
1840 if (ethertype
!= ETH_P_IP
&& ethertype
!= ETH_P_IPV6
)
1843 modify_ip_header
= false;
1844 tcf_exts_to_list(exts
, &actions
);
1845 list_for_each_entry(a
, &actions
, list
) {
1846 if (!is_tcf_pedit(a
))
1849 nkeys
= tcf_pedit_nkeys(a
);
1850 for (i
= 0; i
< nkeys
; i
++) {
1851 htype
= tcf_pedit_htype(a
, i
);
1852 if (htype
== TCA_PEDIT_KEY_EX_HDR_TYPE_IP4
||
1853 htype
== TCA_PEDIT_KEY_EX_HDR_TYPE_IP6
) {
1854 modify_ip_header
= true;
1860 ip_proto
= MLX5_GET(fte_match_set_lyr_2_4
, headers_v
, ip_protocol
);
1861 if (modify_ip_header
&& ip_proto
!= IPPROTO_TCP
&& ip_proto
!= IPPROTO_UDP
) {
1862 pr_info("can't offload re-write of ip proto %d\n", ip_proto
);
1870 static bool actions_match_supported(struct mlx5e_priv
*priv
,
1871 struct tcf_exts
*exts
,
1872 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
1873 struct mlx5e_tc_flow
*flow
)
1877 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
)
1878 actions
= flow
->esw_attr
->action
;
1880 actions
= flow
->nic_attr
->action
;
1882 if (actions
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
1883 return modify_header_match_supported(&parse_attr
->spec
, exts
);
1888 static bool same_hw_devs(struct mlx5e_priv
*priv
, struct mlx5e_priv
*peer_priv
)
1890 struct mlx5_core_dev
*fmdev
, *pmdev
;
1891 u16 func_id
, peer_id
;
1894 pmdev
= peer_priv
->mdev
;
1896 func_id
= (u16
)((fmdev
->pdev
->bus
->number
<< 8) | PCI_SLOT(fmdev
->pdev
->devfn
));
1897 peer_id
= (u16
)((pmdev
->pdev
->bus
->number
<< 8) | PCI_SLOT(pmdev
->pdev
->devfn
));
1899 return (func_id
== peer_id
);
1902 static int parse_tc_nic_actions(struct mlx5e_priv
*priv
, struct tcf_exts
*exts
,
1903 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
1904 struct mlx5e_tc_flow
*flow
)
1906 struct mlx5_nic_flow_attr
*attr
= flow
->nic_attr
;
1907 const struct tc_action
*a
;
1911 if (!tcf_exts_has_actions(exts
))
1914 attr
->flow_tag
= MLX5_FS_DEFAULT_FLOW_TAG
;
1917 tcf_exts_to_list(exts
, &actions
);
1918 list_for_each_entry(a
, &actions
, list
) {
1919 if (is_tcf_gact_shot(a
)) {
1920 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_DROP
;
1921 if (MLX5_CAP_FLOWTABLE(priv
->mdev
,
1922 flow_table_properties_nic_receive
.flow_counter
))
1923 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_COUNT
;
1927 if (is_tcf_pedit(a
)) {
1928 err
= parse_tc_pedit_action(priv
, a
, MLX5_FLOW_NAMESPACE_KERNEL
,
1933 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
|
1934 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1938 if (is_tcf_csum(a
)) {
1939 if (csum_offload_supported(priv
, attr
->action
,
1940 tcf_csum_update_flags(a
)))
1946 if (is_tcf_mirred_egress_redirect(a
)) {
1947 struct net_device
*peer_dev
= tcf_mirred_dev(a
);
1949 if (priv
->netdev
->netdev_ops
== peer_dev
->netdev_ops
&&
1950 same_hw_devs(priv
, netdev_priv(peer_dev
))) {
1951 parse_attr
->mirred_ifindex
= peer_dev
->ifindex
;
1952 flow
->flags
|= MLX5E_TC_FLOW_HAIRPIN
;
1953 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
1954 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
1956 netdev_warn(priv
->netdev
, "device %s not on same HW, can't offload\n",
1963 if (is_tcf_skbedit_mark(a
)) {
1964 u32 mark
= tcf_skbedit_mark(a
);
1966 if (mark
& ~MLX5E_TC_FLOW_ID_MASK
) {
1967 netdev_warn(priv
->netdev
, "Bad flow mark - only 16 bit is supported: 0x%x\n",
1972 attr
->flow_tag
= mark
;
1973 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1980 if (!actions_match_supported(priv
, exts
, parse_attr
, flow
))
1986 static inline int cmp_encap_info(struct ip_tunnel_key
*a
,
1987 struct ip_tunnel_key
*b
)
1989 return memcmp(a
, b
, sizeof(*a
));
1992 static inline int hash_encap_info(struct ip_tunnel_key
*key
)
1994 return jhash(key
, sizeof(*key
), 0);
1997 static int mlx5e_route_lookup_ipv4(struct mlx5e_priv
*priv
,
1998 struct net_device
*mirred_dev
,
1999 struct net_device
**out_dev
,
2001 struct neighbour
**out_n
,
2004 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
2005 struct mlx5e_rep_priv
*uplink_rpriv
;
2007 struct neighbour
*n
= NULL
;
2009 #if IS_ENABLED(CONFIG_INET)
2012 rt
= ip_route_output_key(dev_net(mirred_dev
), fl4
);
2013 ret
= PTR_ERR_OR_ZERO(rt
);
2019 uplink_rpriv
= mlx5_eswitch_get_uplink_priv(esw
, REP_ETH
);
2020 /* if the egress device isn't on the same HW e-switch, we use the uplink */
2021 if (!switchdev_port_same_parent_id(priv
->netdev
, rt
->dst
.dev
))
2022 *out_dev
= uplink_rpriv
->netdev
;
2024 *out_dev
= rt
->dst
.dev
;
2026 *out_ttl
= ip4_dst_hoplimit(&rt
->dst
);
2027 n
= dst_neigh_lookup(&rt
->dst
, &fl4
->daddr
);
2036 static int mlx5e_route_lookup_ipv6(struct mlx5e_priv
*priv
,
2037 struct net_device
*mirred_dev
,
2038 struct net_device
**out_dev
,
2040 struct neighbour
**out_n
,
2043 struct neighbour
*n
= NULL
;
2044 struct dst_entry
*dst
;
2046 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
2047 struct mlx5e_rep_priv
*uplink_rpriv
;
2048 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
2051 ret
= ipv6_stub
->ipv6_dst_lookup(dev_net(mirred_dev
), NULL
, &dst
,
2056 *out_ttl
= ip6_dst_hoplimit(dst
);
2058 uplink_rpriv
= mlx5_eswitch_get_uplink_priv(esw
, REP_ETH
);
2059 /* if the egress device isn't on the same HW e-switch, we use the uplink */
2060 if (!switchdev_port_same_parent_id(priv
->netdev
, dst
->dev
))
2061 *out_dev
= uplink_rpriv
->netdev
;
2063 *out_dev
= dst
->dev
;
2068 n
= dst_neigh_lookup(dst
, &fl6
->daddr
);
2077 static void gen_vxlan_header_ipv4(struct net_device
*out_dev
,
2078 char buf
[], int encap_size
,
2079 unsigned char h_dest
[ETH_ALEN
],
2083 __be16 udp_dst_port
,
2086 struct ethhdr
*eth
= (struct ethhdr
*)buf
;
2087 struct iphdr
*ip
= (struct iphdr
*)((char *)eth
+ sizeof(struct ethhdr
));
2088 struct udphdr
*udp
= (struct udphdr
*)((char *)ip
+ sizeof(struct iphdr
));
2089 struct vxlanhdr
*vxh
= (struct vxlanhdr
*)((char *)udp
+ sizeof(struct udphdr
));
2091 memset(buf
, 0, encap_size
);
2093 ether_addr_copy(eth
->h_dest
, h_dest
);
2094 ether_addr_copy(eth
->h_source
, out_dev
->dev_addr
);
2095 eth
->h_proto
= htons(ETH_P_IP
);
2101 ip
->protocol
= IPPROTO_UDP
;
2105 udp
->dest
= udp_dst_port
;
2106 vxh
->vx_flags
= VXLAN_HF_VNI
;
2107 vxh
->vx_vni
= vxlan_vni_field(vx_vni
);
2110 static void gen_vxlan_header_ipv6(struct net_device
*out_dev
,
2111 char buf
[], int encap_size
,
2112 unsigned char h_dest
[ETH_ALEN
],
2114 struct in6_addr
*daddr
,
2115 struct in6_addr
*saddr
,
2116 __be16 udp_dst_port
,
2119 struct ethhdr
*eth
= (struct ethhdr
*)buf
;
2120 struct ipv6hdr
*ip6h
= (struct ipv6hdr
*)((char *)eth
+ sizeof(struct ethhdr
));
2121 struct udphdr
*udp
= (struct udphdr
*)((char *)ip6h
+ sizeof(struct ipv6hdr
));
2122 struct vxlanhdr
*vxh
= (struct vxlanhdr
*)((char *)udp
+ sizeof(struct udphdr
));
2124 memset(buf
, 0, encap_size
);
2126 ether_addr_copy(eth
->h_dest
, h_dest
);
2127 ether_addr_copy(eth
->h_source
, out_dev
->dev_addr
);
2128 eth
->h_proto
= htons(ETH_P_IPV6
);
2130 ip6_flow_hdr(ip6h
, 0, 0);
2131 /* the HW fills up ipv6 payload len */
2132 ip6h
->nexthdr
= IPPROTO_UDP
;
2133 ip6h
->hop_limit
= ttl
;
2134 ip6h
->daddr
= *daddr
;
2135 ip6h
->saddr
= *saddr
;
2137 udp
->dest
= udp_dst_port
;
2138 vxh
->vx_flags
= VXLAN_HF_VNI
;
2139 vxh
->vx_vni
= vxlan_vni_field(vx_vni
);
2142 static int mlx5e_create_encap_header_ipv4(struct mlx5e_priv
*priv
,
2143 struct net_device
*mirred_dev
,
2144 struct mlx5e_encap_entry
*e
)
2146 int max_encap_size
= MLX5_CAP_ESW(priv
->mdev
, max_encap_header_size
);
2147 int ipv4_encap_size
= ETH_HLEN
+ sizeof(struct iphdr
) + VXLAN_HLEN
;
2148 struct ip_tunnel_key
*tun_key
= &e
->tun_info
.key
;
2149 struct net_device
*out_dev
;
2150 struct neighbour
*n
= NULL
;
2151 struct flowi4 fl4
= {};
2156 if (max_encap_size
< ipv4_encap_size
) {
2157 mlx5_core_warn(priv
->mdev
, "encap size %d too big, max supported is %d\n",
2158 ipv4_encap_size
, max_encap_size
);
2162 encap_header
= kzalloc(ipv4_encap_size
, GFP_KERNEL
);
2166 switch (e
->tunnel_type
) {
2167 case MLX5_HEADER_TYPE_VXLAN
:
2168 fl4
.flowi4_proto
= IPPROTO_UDP
;
2169 fl4
.fl4_dport
= tun_key
->tp_dst
;
2175 fl4
.flowi4_tos
= tun_key
->tos
;
2176 fl4
.daddr
= tun_key
->u
.ipv4
.dst
;
2177 fl4
.saddr
= tun_key
->u
.ipv4
.src
;
2179 err
= mlx5e_route_lookup_ipv4(priv
, mirred_dev
, &out_dev
,
2184 /* used by mlx5e_detach_encap to lookup a neigh hash table
2185 * entry in the neigh hash table when a user deletes a rule
2187 e
->m_neigh
.dev
= n
->dev
;
2188 e
->m_neigh
.family
= n
->ops
->family
;
2189 memcpy(&e
->m_neigh
.dst_ip
, n
->primary_key
, n
->tbl
->key_len
);
2190 e
->out_dev
= out_dev
;
2192 /* It's importent to add the neigh to the hash table before checking
2193 * the neigh validity state. So if we'll get a notification, in case the
2194 * neigh changes it's validity state, we would find the relevant neigh
2197 err
= mlx5e_rep_encap_entry_attach(netdev_priv(out_dev
), e
);
2201 read_lock_bh(&n
->lock
);
2202 nud_state
= n
->nud_state
;
2203 ether_addr_copy(e
->h_dest
, n
->ha
);
2204 read_unlock_bh(&n
->lock
);
2206 switch (e
->tunnel_type
) {
2207 case MLX5_HEADER_TYPE_VXLAN
:
2208 gen_vxlan_header_ipv4(out_dev
, encap_header
,
2209 ipv4_encap_size
, e
->h_dest
, ttl
,
2211 fl4
.saddr
, tun_key
->tp_dst
,
2212 tunnel_id_to_key32(tun_key
->tun_id
));
2216 goto destroy_neigh_entry
;
2218 e
->encap_size
= ipv4_encap_size
;
2219 e
->encap_header
= encap_header
;
2221 if (!(nud_state
& NUD_VALID
)) {
2222 neigh_event_send(n
, NULL
);
2227 err
= mlx5_encap_alloc(priv
->mdev
, e
->tunnel_type
,
2228 ipv4_encap_size
, encap_header
, &e
->encap_id
);
2230 goto destroy_neigh_entry
;
2232 e
->flags
|= MLX5_ENCAP_ENTRY_VALID
;
2233 mlx5e_rep_queue_neigh_stats_work(netdev_priv(out_dev
));
2237 destroy_neigh_entry
:
2238 mlx5e_rep_encap_entry_detach(netdev_priv(e
->out_dev
), e
);
2240 kfree(encap_header
);
2247 static int mlx5e_create_encap_header_ipv6(struct mlx5e_priv
*priv
,
2248 struct net_device
*mirred_dev
,
2249 struct mlx5e_encap_entry
*e
)
2251 int max_encap_size
= MLX5_CAP_ESW(priv
->mdev
, max_encap_header_size
);
2252 int ipv6_encap_size
= ETH_HLEN
+ sizeof(struct ipv6hdr
) + VXLAN_HLEN
;
2253 struct ip_tunnel_key
*tun_key
= &e
->tun_info
.key
;
2254 struct net_device
*out_dev
;
2255 struct neighbour
*n
= NULL
;
2256 struct flowi6 fl6
= {};
2261 if (max_encap_size
< ipv6_encap_size
) {
2262 mlx5_core_warn(priv
->mdev
, "encap size %d too big, max supported is %d\n",
2263 ipv6_encap_size
, max_encap_size
);
2267 encap_header
= kzalloc(ipv6_encap_size
, GFP_KERNEL
);
2271 switch (e
->tunnel_type
) {
2272 case MLX5_HEADER_TYPE_VXLAN
:
2273 fl6
.flowi6_proto
= IPPROTO_UDP
;
2274 fl6
.fl6_dport
= tun_key
->tp_dst
;
2281 fl6
.flowlabel
= ip6_make_flowinfo(RT_TOS(tun_key
->tos
), tun_key
->label
);
2282 fl6
.daddr
= tun_key
->u
.ipv6
.dst
;
2283 fl6
.saddr
= tun_key
->u
.ipv6
.src
;
2285 err
= mlx5e_route_lookup_ipv6(priv
, mirred_dev
, &out_dev
,
2290 /* used by mlx5e_detach_encap to lookup a neigh hash table
2291 * entry in the neigh hash table when a user deletes a rule
2293 e
->m_neigh
.dev
= n
->dev
;
2294 e
->m_neigh
.family
= n
->ops
->family
;
2295 memcpy(&e
->m_neigh
.dst_ip
, n
->primary_key
, n
->tbl
->key_len
);
2296 e
->out_dev
= out_dev
;
2298 /* It's importent to add the neigh to the hash table before checking
2299 * the neigh validity state. So if we'll get a notification, in case the
2300 * neigh changes it's validity state, we would find the relevant neigh
2303 err
= mlx5e_rep_encap_entry_attach(netdev_priv(out_dev
), e
);
2307 read_lock_bh(&n
->lock
);
2308 nud_state
= n
->nud_state
;
2309 ether_addr_copy(e
->h_dest
, n
->ha
);
2310 read_unlock_bh(&n
->lock
);
2312 switch (e
->tunnel_type
) {
2313 case MLX5_HEADER_TYPE_VXLAN
:
2314 gen_vxlan_header_ipv6(out_dev
, encap_header
,
2315 ipv6_encap_size
, e
->h_dest
, ttl
,
2317 &fl6
.saddr
, tun_key
->tp_dst
,
2318 tunnel_id_to_key32(tun_key
->tun_id
));
2322 goto destroy_neigh_entry
;
2325 e
->encap_size
= ipv6_encap_size
;
2326 e
->encap_header
= encap_header
;
2328 if (!(nud_state
& NUD_VALID
)) {
2329 neigh_event_send(n
, NULL
);
2334 err
= mlx5_encap_alloc(priv
->mdev
, e
->tunnel_type
,
2335 ipv6_encap_size
, encap_header
, &e
->encap_id
);
2337 goto destroy_neigh_entry
;
2339 e
->flags
|= MLX5_ENCAP_ENTRY_VALID
;
2340 mlx5e_rep_queue_neigh_stats_work(netdev_priv(out_dev
));
2344 destroy_neigh_entry
:
2345 mlx5e_rep_encap_entry_detach(netdev_priv(e
->out_dev
), e
);
2347 kfree(encap_header
);
2354 static int mlx5e_attach_encap(struct mlx5e_priv
*priv
,
2355 struct ip_tunnel_info
*tun_info
,
2356 struct net_device
*mirred_dev
,
2357 struct net_device
**encap_dev
,
2358 struct mlx5e_tc_flow
*flow
)
2360 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
2361 struct mlx5e_rep_priv
*uplink_rpriv
= mlx5_eswitch_get_uplink_priv(esw
,
2363 struct net_device
*up_dev
= uplink_rpriv
->netdev
;
2364 unsigned short family
= ip_tunnel_info_af(tun_info
);
2365 struct mlx5e_priv
*up_priv
= netdev_priv(up_dev
);
2366 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
2367 struct ip_tunnel_key
*key
= &tun_info
->key
;
2368 struct mlx5e_encap_entry
*e
;
2369 int tunnel_type
, err
= 0;
2373 /* udp dst port must be set */
2374 if (!memchr_inv(&key
->tp_dst
, 0, sizeof(key
->tp_dst
)))
2375 goto vxlan_encap_offload_err
;
2377 /* setting udp src port isn't supported */
2378 if (memchr_inv(&key
->tp_src
, 0, sizeof(key
->tp_src
))) {
2379 vxlan_encap_offload_err
:
2380 netdev_warn(priv
->netdev
,
2381 "must set udp dst port and not set udp src port\n");
2385 if (mlx5e_vxlan_lookup_port(up_priv
, be16_to_cpu(key
->tp_dst
)) &&
2386 MLX5_CAP_ESW(priv
->mdev
, vxlan_encap_decap
)) {
2387 tunnel_type
= MLX5_HEADER_TYPE_VXLAN
;
2389 netdev_warn(priv
->netdev
,
2390 "%d isn't an offloaded vxlan udp dport\n", be16_to_cpu(key
->tp_dst
));
2394 hash_key
= hash_encap_info(key
);
2396 hash_for_each_possible_rcu(esw
->offloads
.encap_tbl
, e
,
2397 encap_hlist
, hash_key
) {
2398 if (!cmp_encap_info(&e
->tun_info
.key
, key
)) {
2404 /* must verify if encap is valid or not */
2408 e
= kzalloc(sizeof(*e
), GFP_KERNEL
);
2412 e
->tun_info
= *tun_info
;
2413 e
->tunnel_type
= tunnel_type
;
2414 INIT_LIST_HEAD(&e
->flows
);
2416 if (family
== AF_INET
)
2417 err
= mlx5e_create_encap_header_ipv4(priv
, mirred_dev
, e
);
2418 else if (family
== AF_INET6
)
2419 err
= mlx5e_create_encap_header_ipv6(priv
, mirred_dev
, e
);
2421 if (err
&& err
!= -EAGAIN
)
2424 hash_add_rcu(esw
->offloads
.encap_tbl
, &e
->encap_hlist
, hash_key
);
2427 list_add(&flow
->encap
, &e
->flows
);
2428 *encap_dev
= e
->out_dev
;
2429 if (e
->flags
& MLX5_ENCAP_ENTRY_VALID
)
2430 attr
->encap_id
= e
->encap_id
;
2441 static int parse_tc_fdb_actions(struct mlx5e_priv
*priv
, struct tcf_exts
*exts
,
2442 struct mlx5e_tc_flow_parse_attr
*parse_attr
,
2443 struct mlx5e_tc_flow
*flow
)
2445 struct mlx5_esw_flow_attr
*attr
= flow
->esw_attr
;
2446 struct mlx5e_rep_priv
*rpriv
= priv
->ppriv
;
2447 struct ip_tunnel_info
*info
= NULL
;
2448 const struct tc_action
*a
;
2453 if (!tcf_exts_has_actions(exts
))
2456 memset(attr
, 0, sizeof(*attr
));
2457 attr
->in_rep
= rpriv
->rep
;
2459 tcf_exts_to_list(exts
, &actions
);
2460 list_for_each_entry(a
, &actions
, list
) {
2461 if (is_tcf_gact_shot(a
)) {
2462 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_DROP
|
2463 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
2467 if (is_tcf_pedit(a
)) {
2468 err
= parse_tc_pedit_action(priv
, a
, MLX5_FLOW_NAMESPACE_FDB
,
2473 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
2477 if (is_tcf_csum(a
)) {
2478 if (csum_offload_supported(priv
, attr
->action
,
2479 tcf_csum_update_flags(a
)))
2485 if (is_tcf_mirred_egress_redirect(a
)) {
2486 struct net_device
*out_dev
;
2487 struct mlx5e_priv
*out_priv
;
2489 out_dev
= tcf_mirred_dev(a
);
2491 if (switchdev_port_same_parent_id(priv
->netdev
,
2493 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
2494 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
2495 out_priv
= netdev_priv(out_dev
);
2496 rpriv
= out_priv
->ppriv
;
2497 attr
->out_rep
= rpriv
->rep
;
2499 parse_attr
->mirred_ifindex
= out_dev
->ifindex
;
2500 parse_attr
->tun_info
= *info
;
2501 attr
->parse_attr
= parse_attr
;
2502 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_ENCAP
|
2503 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
2504 MLX5_FLOW_CONTEXT_ACTION_COUNT
;
2505 /* attr->out_rep is resolved when we handle encap */
2507 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
2508 priv
->netdev
->name
, out_dev
->name
);
2514 if (is_tcf_tunnel_set(a
)) {
2515 info
= tcf_tunnel_info(a
);
2523 if (is_tcf_vlan(a
)) {
2524 if (tcf_vlan_action(a
) == TCA_VLAN_ACT_POP
) {
2525 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
;
2526 } else if (tcf_vlan_action(a
) == TCA_VLAN_ACT_PUSH
) {
2527 if (tcf_vlan_push_proto(a
) != htons(ETH_P_8021Q
))
2530 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
;
2531 attr
->vlan
= tcf_vlan_push_vid(a
);
2532 } else { /* action is TCA_VLAN_ACT_MODIFY */
2538 if (is_tcf_tunnel_release(a
)) {
2539 attr
->action
|= MLX5_FLOW_CONTEXT_ACTION_DECAP
;
2546 if (!actions_match_supported(priv
, exts
, parse_attr
, flow
))
2552 int mlx5e_configure_flower(struct mlx5e_priv
*priv
,
2553 struct tc_cls_flower_offload
*f
)
2555 struct mlx5_eswitch
*esw
= priv
->mdev
->priv
.eswitch
;
2556 struct mlx5e_tc_flow_parse_attr
*parse_attr
;
2557 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
2558 struct mlx5e_tc_flow
*flow
;
2559 int attr_size
, err
= 0;
2562 if (esw
&& esw
->mode
== SRIOV_OFFLOADS
) {
2563 flow_flags
= MLX5E_TC_FLOW_ESWITCH
;
2564 attr_size
= sizeof(struct mlx5_esw_flow_attr
);
2566 flow_flags
= MLX5E_TC_FLOW_NIC
;
2567 attr_size
= sizeof(struct mlx5_nic_flow_attr
);
2570 flow
= kzalloc(sizeof(*flow
) + attr_size
, GFP_KERNEL
);
2571 parse_attr
= kvzalloc(sizeof(*parse_attr
), GFP_KERNEL
);
2572 if (!parse_attr
|| !flow
) {
2577 flow
->cookie
= f
->cookie
;
2578 flow
->flags
= flow_flags
;
2580 err
= parse_cls_flower(priv
, flow
, &parse_attr
->spec
, f
);
2584 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
) {
2585 err
= parse_tc_fdb_actions(priv
, f
->exts
, parse_attr
, flow
);
2588 flow
->rule
= mlx5e_tc_add_fdb_flow(priv
, parse_attr
, flow
);
2590 err
= parse_tc_nic_actions(priv
, f
->exts
, parse_attr
, flow
);
2593 flow
->rule
= mlx5e_tc_add_nic_flow(priv
, parse_attr
, flow
);
2596 if (IS_ERR(flow
->rule
)) {
2597 err
= PTR_ERR(flow
->rule
);
2603 flow
->flags
|= MLX5E_TC_FLOW_OFFLOADED
;
2605 err
= rhashtable_insert_fast(&tc
->ht
, &flow
->node
,
2610 if (flow
->flags
& MLX5E_TC_FLOW_ESWITCH
&&
2611 !(flow
->esw_attr
->action
& MLX5_FLOW_CONTEXT_ACTION_ENCAP
))
2616 mlx5e_tc_del_flow(priv
, flow
);
2624 int mlx5e_delete_flower(struct mlx5e_priv
*priv
,
2625 struct tc_cls_flower_offload
*f
)
2627 struct mlx5e_tc_flow
*flow
;
2628 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
2630 flow
= rhashtable_lookup_fast(&tc
->ht
, &f
->cookie
,
2635 rhashtable_remove_fast(&tc
->ht
, &flow
->node
, tc
->ht_params
);
2637 mlx5e_tc_del_flow(priv
, flow
);
2644 int mlx5e_stats_flower(struct mlx5e_priv
*priv
,
2645 struct tc_cls_flower_offload
*f
)
2647 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
2648 struct mlx5e_tc_flow
*flow
;
2649 struct mlx5_fc
*counter
;
2654 flow
= rhashtable_lookup_fast(&tc
->ht
, &f
->cookie
,
2659 if (!(flow
->flags
& MLX5E_TC_FLOW_OFFLOADED
))
2662 counter
= mlx5_flow_rule_counter(flow
->rule
);
2666 mlx5_fc_query_cached(counter
, &bytes
, &packets
, &lastuse
);
2668 tcf_exts_stats_update(f
->exts
, bytes
, packets
, lastuse
);
2673 static const struct rhashtable_params mlx5e_tc_flow_ht_params
= {
2674 .head_offset
= offsetof(struct mlx5e_tc_flow
, node
),
2675 .key_offset
= offsetof(struct mlx5e_tc_flow
, cookie
),
2676 .key_len
= sizeof(((struct mlx5e_tc_flow
*)0)->cookie
),
2677 .automatic_shrinking
= true,
2680 int mlx5e_tc_init(struct mlx5e_priv
*priv
)
2682 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
2684 hash_init(tc
->mod_hdr_tbl
);
2685 hash_init(tc
->hairpin_tbl
);
2687 tc
->ht_params
= mlx5e_tc_flow_ht_params
;
2688 return rhashtable_init(&tc
->ht
, &tc
->ht_params
);
2691 static void _mlx5e_tc_del_flow(void *ptr
, void *arg
)
2693 struct mlx5e_tc_flow
*flow
= ptr
;
2694 struct mlx5e_priv
*priv
= arg
;
2696 mlx5e_tc_del_flow(priv
, flow
);
2700 void mlx5e_tc_cleanup(struct mlx5e_priv
*priv
)
2702 struct mlx5e_tc_table
*tc
= &priv
->fs
.tc
;
2704 rhashtable_free_and_destroy(&tc
->ht
, _mlx5e_tc_del_flow
, priv
);
2706 if (!IS_ERR_OR_NULL(tc
->t
)) {
2707 mlx5_destroy_flow_table(tc
->t
);