2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
46 #include "lib/devcom.h"
48 #include "lib/fs_chains.h"
50 #include "en/mapping.h"
53 #define mlx5_esw_for_each_rep(esw, i, rep) \
54 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
56 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \
57 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF)
59 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \
60 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \
61 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF)
63 /* There are two match-all miss flows, one for unicast dst mac and
66 #define MLX5_ESW_MISS_FLOWS (2)
67 #define UPLINK_REP_INDEX 0
69 #define MLX5_ESW_VPORT_TBL_SIZE 128
70 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
72 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns
= {
73 .max_fte
= MLX5_ESW_VPORT_TBL_SIZE
,
74 .max_num_groups
= MLX5_ESW_VPORT_TBL_NUM_GROUPS
,
78 static struct mlx5_eswitch_rep
*mlx5_eswitch_get_rep(struct mlx5_eswitch
*esw
,
81 return xa_load(&esw
->offloads
.vport_reps
, vport_num
);
85 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch
*esw
,
86 struct mlx5_flow_spec
*spec
,
87 struct mlx5_esw_flow_attr
*attr
)
89 if (MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, flow_source
) &&
91 spec
->flow_context
.flow_source
=
92 attr
->in_rep
->vport
== MLX5_VPORT_UPLINK
?
93 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK
:
94 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT
;
97 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
98 * are not needed as well in the following process. So clear them all for simplicity.
101 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch
*esw
, struct mlx5_flow_spec
*spec
)
103 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
106 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
107 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
, 0);
109 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
110 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
, 0);
112 if (!memchr_inv(misc2
, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2
)))
113 spec
->match_criteria_enable
&= ~MLX5_MATCH_MISC_PARAMETERS_2
;
118 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch
*esw
,
119 struct mlx5_flow_spec
*spec
,
120 struct mlx5_flow_attr
*attr
,
121 struct mlx5_eswitch
*src_esw
,
127 /* Use metadata matching because vport is not represented by single
128 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
130 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
131 if (mlx5_esw_indir_table_decap_vport(attr
))
132 vport
= mlx5_esw_indir_table_decap_vport(attr
);
133 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
134 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
,
135 mlx5_eswitch_get_vport_metadata_for_match(src_esw
,
138 misc2
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
139 MLX5_SET(fte_match_set_misc2
, misc2
, metadata_reg_c_0
,
140 mlx5_eswitch_get_vport_metadata_mask());
142 spec
->match_criteria_enable
|= MLX5_MATCH_MISC_PARAMETERS_2
;
144 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
145 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
147 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
148 MLX5_SET(fte_match_set_misc
, misc
,
149 source_eswitch_owner_vhca_id
,
150 MLX5_CAP_GEN(src_esw
->dev
, vhca_id
));
152 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
153 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
154 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
155 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
156 source_eswitch_owner_vhca_id
);
158 spec
->match_criteria_enable
|= MLX5_MATCH_MISC_PARAMETERS
;
163 esw_setup_decap_indir(struct mlx5_eswitch
*esw
,
164 struct mlx5_flow_attr
*attr
,
165 struct mlx5_flow_spec
*spec
)
167 struct mlx5_flow_table
*ft
;
169 if (!(attr
->flags
& MLX5_ESW_ATTR_FLAG_SRC_REWRITE
))
172 ft
= mlx5_esw_indir_table_get(esw
, attr
, spec
,
173 mlx5_esw_indir_table_decap_vport(attr
), true);
174 return PTR_ERR_OR_ZERO(ft
);
178 esw_cleanup_decap_indir(struct mlx5_eswitch
*esw
,
179 struct mlx5_flow_attr
*attr
)
181 if (mlx5_esw_indir_table_decap_vport(attr
))
182 mlx5_esw_indir_table_put(esw
, attr
,
183 mlx5_esw_indir_table_decap_vport(attr
),
188 esw_setup_sampler_dest(struct mlx5_flow_destination
*dest
,
189 struct mlx5_flow_act
*flow_act
,
190 struct mlx5_flow_attr
*attr
,
193 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
194 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER
;
195 dest
[i
].sampler_id
= attr
->sample_attr
->sampler_id
;
201 esw_setup_ft_dest(struct mlx5_flow_destination
*dest
,
202 struct mlx5_flow_act
*flow_act
,
203 struct mlx5_eswitch
*esw
,
204 struct mlx5_flow_attr
*attr
,
205 struct mlx5_flow_spec
*spec
,
208 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
209 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
210 dest
[i
].ft
= attr
->dest_ft
;
212 if (mlx5_esw_indir_table_decap_vport(attr
))
213 return esw_setup_decap_indir(esw
, attr
, spec
);
218 esw_setup_slow_path_dest(struct mlx5_flow_destination
*dest
,
219 struct mlx5_flow_act
*flow_act
,
220 struct mlx5_fs_chains
*chains
,
223 if (mlx5_chains_ignore_flow_level_supported(chains
))
224 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
225 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
226 dest
[i
].ft
= mlx5_chains_get_tc_end_ft(chains
);
230 esw_setup_chain_dest(struct mlx5_flow_destination
*dest
,
231 struct mlx5_flow_act
*flow_act
,
232 struct mlx5_fs_chains
*chains
,
233 u32 chain
, u32 prio
, u32 level
,
236 struct mlx5_flow_table
*ft
;
238 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
239 ft
= mlx5_chains_get_table(chains
, chain
, prio
, level
);
243 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
248 static void esw_put_dest_tables_loop(struct mlx5_eswitch
*esw
, struct mlx5_flow_attr
*attr
,
251 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
252 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
255 for (i
= from
; i
< to
; i
++)
256 if (esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE
)
257 mlx5_chains_put_table(chains
, 0, 1, 0);
258 else if (mlx5_esw_indir_table_needed(esw
, attr
, esw_attr
->dests
[i
].rep
->vport
,
259 esw_attr
->dests
[i
].mdev
))
260 mlx5_esw_indir_table_put(esw
, attr
, esw_attr
->dests
[i
].rep
->vport
,
265 esw_is_chain_src_port_rewrite(struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
)
269 for (i
= esw_attr
->split_count
; i
< esw_attr
->out_count
; i
++)
270 if (esw_attr
->dests
[i
].flags
& MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE
)
276 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination
*dest
,
277 struct mlx5_flow_act
*flow_act
,
278 struct mlx5_eswitch
*esw
,
279 struct mlx5_fs_chains
*chains
,
280 struct mlx5_flow_attr
*attr
,
283 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
286 if (!(attr
->flags
& MLX5_ESW_ATTR_FLAG_SRC_REWRITE
))
289 for (j
= esw_attr
->split_count
; j
< esw_attr
->out_count
; j
++, (*i
)++) {
290 err
= esw_setup_chain_dest(dest
, flow_act
, chains
, attr
->dest_chain
, 1, 0, *i
);
292 goto err_setup_chain
;
293 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
;
294 flow_act
->pkt_reformat
= esw_attr
->dests
[j
].pkt_reformat
;
299 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, j
);
303 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch
*esw
,
304 struct mlx5_flow_attr
*attr
)
306 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
308 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, esw_attr
->out_count
);
312 esw_is_indir_table(struct mlx5_eswitch
*esw
, struct mlx5_flow_attr
*attr
)
314 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
317 for (i
= esw_attr
->split_count
; i
< esw_attr
->out_count
; i
++)
318 if (mlx5_esw_indir_table_needed(esw
, attr
, esw_attr
->dests
[i
].rep
->vport
,
319 esw_attr
->dests
[i
].mdev
))
325 esw_setup_indir_table(struct mlx5_flow_destination
*dest
,
326 struct mlx5_flow_act
*flow_act
,
327 struct mlx5_eswitch
*esw
,
328 struct mlx5_flow_attr
*attr
,
329 struct mlx5_flow_spec
*spec
,
330 bool ignore_flow_lvl
,
333 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
336 if (!(attr
->flags
& MLX5_ESW_ATTR_FLAG_SRC_REWRITE
))
339 for (j
= esw_attr
->split_count
; j
< esw_attr
->out_count
; j
++, (*i
)++) {
341 flow_act
->flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
342 dest
[*i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
344 dest
[*i
].ft
= mlx5_esw_indir_table_get(esw
, attr
, spec
,
345 esw_attr
->dests
[j
].rep
->vport
, false);
346 if (IS_ERR(dest
[*i
].ft
)) {
347 err
= PTR_ERR(dest
[*i
].ft
);
348 goto err_indir_tbl_get
;
352 if (mlx5_esw_indir_table_decap_vport(attr
)) {
353 err
= esw_setup_decap_indir(esw
, attr
, spec
);
355 goto err_indir_tbl_get
;
361 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, j
);
365 static void esw_cleanup_indir_table(struct mlx5_eswitch
*esw
, struct mlx5_flow_attr
*attr
)
367 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
369 esw_put_dest_tables_loop(esw
, attr
, esw_attr
->split_count
, esw_attr
->out_count
);
370 esw_cleanup_decap_indir(esw
, attr
);
374 esw_cleanup_chain_dest(struct mlx5_fs_chains
*chains
, u32 chain
, u32 prio
, u32 level
)
376 mlx5_chains_put_table(chains
, chain
, prio
, level
);
380 esw_setup_vport_dest(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
381 struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
,
382 int attr_idx
, int dest_idx
, bool pkt_reformat
)
384 dest
[dest_idx
].type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
385 dest
[dest_idx
].vport
.num
= esw_attr
->dests
[attr_idx
].rep
->vport
;
386 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
)) {
387 dest
[dest_idx
].vport
.vhca_id
=
388 MLX5_CAP_GEN(esw_attr
->dests
[attr_idx
].mdev
, vhca_id
);
389 dest
[dest_idx
].vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
391 if (esw_attr
->dests
[attr_idx
].flags
& MLX5_ESW_DEST_ENCAP
) {
393 flow_act
->action
|= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
;
394 flow_act
->pkt_reformat
= esw_attr
->dests
[attr_idx
].pkt_reformat
;
396 dest
[dest_idx
].vport
.flags
|= MLX5_FLOW_DEST_VPORT_REFORMAT_ID
;
397 dest
[dest_idx
].vport
.pkt_reformat
= esw_attr
->dests
[attr_idx
].pkt_reformat
;
402 esw_setup_vport_dests(struct mlx5_flow_destination
*dest
, struct mlx5_flow_act
*flow_act
,
403 struct mlx5_eswitch
*esw
, struct mlx5_esw_flow_attr
*esw_attr
,
408 for (j
= esw_attr
->split_count
; j
< esw_attr
->out_count
; j
++, i
++)
409 esw_setup_vport_dest(dest
, flow_act
, esw
, esw_attr
, j
, i
, true);
414 esw_src_port_rewrite_supported(struct mlx5_eswitch
*esw
)
416 return MLX5_CAP_GEN(esw
->dev
, reg_c_preserve
) &&
417 mlx5_eswitch_vport_match_metadata_enabled(esw
) &&
418 MLX5_CAP_ESW_FLOWTABLE_FDB(esw
->dev
, ignore_flow_level
);
422 esw_setup_dests(struct mlx5_flow_destination
*dest
,
423 struct mlx5_flow_act
*flow_act
,
424 struct mlx5_eswitch
*esw
,
425 struct mlx5_flow_attr
*attr
,
426 struct mlx5_flow_spec
*spec
,
429 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
430 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
433 if (!mlx5_eswitch_termtbl_required(esw
, attr
, flow_act
, spec
) &&
434 esw_src_port_rewrite_supported(esw
))
435 attr
->flags
|= MLX5_ESW_ATTR_FLAG_SRC_REWRITE
;
437 if (attr
->flags
& MLX5_ESW_ATTR_FLAG_SAMPLE
) {
438 esw_setup_sampler_dest(dest
, flow_act
, attr
, *i
);
440 } else if (attr
->dest_ft
) {
441 esw_setup_ft_dest(dest
, flow_act
, esw
, attr
, spec
, *i
);
443 } else if (mlx5_esw_attr_flags_skip(attr
->flags
)) {
444 esw_setup_slow_path_dest(dest
, flow_act
, chains
, *i
);
446 } else if (attr
->dest_chain
) {
447 err
= esw_setup_chain_dest(dest
, flow_act
, chains
, attr
->dest_chain
,
450 } else if (esw_is_indir_table(esw
, attr
)) {
451 err
= esw_setup_indir_table(dest
, flow_act
, esw
, attr
, spec
, true, i
);
452 } else if (esw_is_chain_src_port_rewrite(esw
, esw_attr
)) {
453 err
= esw_setup_chain_src_port_rewrite(dest
, flow_act
, esw
, chains
, attr
, i
);
455 *i
= esw_setup_vport_dests(dest
, flow_act
, esw
, esw_attr
, *i
);
462 esw_cleanup_dests(struct mlx5_eswitch
*esw
,
463 struct mlx5_flow_attr
*attr
)
465 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
466 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
469 esw_cleanup_decap_indir(esw
, attr
);
470 } else if (!mlx5_esw_attr_flags_skip(attr
->flags
)) {
471 if (attr
->dest_chain
)
472 esw_cleanup_chain_dest(chains
, attr
->dest_chain
, 1, 0);
473 else if (esw_is_indir_table(esw
, attr
))
474 esw_cleanup_indir_table(esw
, attr
);
475 else if (esw_is_chain_src_port_rewrite(esw
, esw_attr
))
476 esw_cleanup_chain_src_port_rewrite(esw
, attr
);
480 struct mlx5_flow_handle
*
481 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch
*esw
,
482 struct mlx5_flow_spec
*spec
,
483 struct mlx5_flow_attr
*attr
)
485 struct mlx5_flow_destination dest
[MLX5_MAX_FLOW_FWD_VPORTS
+ 1] = {};
486 struct mlx5_flow_act flow_act
= { .flags
= FLOW_ACT_NO_APPEND
, };
487 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
488 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
489 bool split
= !!(esw_attr
->split_count
);
490 struct mlx5_vport_tbl_attr fwd_attr
;
491 struct mlx5_flow_handle
*rule
;
492 struct mlx5_flow_table
*fdb
;
495 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
496 return ERR_PTR(-EOPNOTSUPP
);
498 flow_act
.action
= attr
->action
;
499 /* if per flow vlan pop/push is emulated, don't set that into the firmware */
500 if (!mlx5_eswitch_vlan_actions_supported(esw
->dev
, 1))
501 flow_act
.action
&= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
|
502 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
);
503 else if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
) {
504 flow_act
.vlan
[0].ethtype
= ntohs(esw_attr
->vlan_proto
[0]);
505 flow_act
.vlan
[0].vid
= esw_attr
->vlan_vid
[0];
506 flow_act
.vlan
[0].prio
= esw_attr
->vlan_prio
[0];
507 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2
) {
508 flow_act
.vlan
[1].ethtype
= ntohs(esw_attr
->vlan_proto
[1]);
509 flow_act
.vlan
[1].vid
= esw_attr
->vlan_vid
[1];
510 flow_act
.vlan
[1].prio
= esw_attr
->vlan_prio
[1];
514 mlx5_eswitch_set_rule_flow_source(esw
, spec
, esw_attr
);
516 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
519 err
= esw_setup_dests(dest
, &flow_act
, esw
, attr
, spec
, &i
);
522 goto err_create_goto_table
;
526 if (esw_attr
->decap_pkt_reformat
)
527 flow_act
.pkt_reformat
= esw_attr
->decap_pkt_reformat
;
529 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
530 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
531 dest
[i
].counter_id
= mlx5_fc_id(attr
->counter
);
535 if (attr
->outer_match_level
!= MLX5_MATCH_NONE
)
536 spec
->match_criteria_enable
|= MLX5_MATCH_OUTER_HEADERS
;
537 if (attr
->inner_match_level
!= MLX5_MATCH_NONE
)
538 spec
->match_criteria_enable
|= MLX5_MATCH_INNER_HEADERS
;
540 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
541 flow_act
.modify_hdr
= attr
->modify_hdr
;
544 fwd_attr
.chain
= attr
->chain
;
545 fwd_attr
.prio
= attr
->prio
;
546 fwd_attr
.vport
= esw_attr
->in_rep
->vport
;
547 fwd_attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
549 fdb
= mlx5_esw_vporttbl_get(esw
, &fwd_attr
);
551 if (attr
->chain
|| attr
->prio
)
552 fdb
= mlx5_chains_get_table(chains
, attr
->chain
,
557 if (!(attr
->flags
& MLX5_ESW_ATTR_FLAG_NO_IN_PORT
))
558 mlx5_eswitch_set_rule_source_port(esw
, spec
, attr
,
559 esw_attr
->in_mdev
->priv
.eswitch
,
560 esw_attr
->in_rep
->vport
);
563 rule
= ERR_CAST(fdb
);
567 if (mlx5_eswitch_termtbl_required(esw
, attr
, &flow_act
, spec
))
568 rule
= mlx5_eswitch_add_termtbl_rule(esw
, fdb
, spec
, esw_attr
,
571 rule
= mlx5_add_flow_rules(fdb
, spec
, &flow_act
, dest
, i
);
575 atomic64_inc(&esw
->offloads
.num_flows
);
581 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
582 else if (attr
->chain
|| attr
->prio
)
583 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
585 esw_cleanup_dests(esw
, attr
);
586 err_create_goto_table
:
590 struct mlx5_flow_handle
*
591 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch
*esw
,
592 struct mlx5_flow_spec
*spec
,
593 struct mlx5_flow_attr
*attr
)
595 struct mlx5_flow_destination dest
[MLX5_MAX_FLOW_FWD_VPORTS
+ 1] = {};
596 struct mlx5_flow_act flow_act
= { .flags
= FLOW_ACT_NO_APPEND
, };
597 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
598 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
599 struct mlx5_vport_tbl_attr fwd_attr
;
600 struct mlx5_flow_table
*fast_fdb
;
601 struct mlx5_flow_table
*fwd_fdb
;
602 struct mlx5_flow_handle
*rule
;
605 fast_fdb
= mlx5_chains_get_table(chains
, attr
->chain
, attr
->prio
, 0);
606 if (IS_ERR(fast_fdb
)) {
607 rule
= ERR_CAST(fast_fdb
);
611 fwd_attr
.chain
= attr
->chain
;
612 fwd_attr
.prio
= attr
->prio
;
613 fwd_attr
.vport
= esw_attr
->in_rep
->vport
;
614 fwd_attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
615 fwd_fdb
= mlx5_esw_vporttbl_get(esw
, &fwd_attr
);
616 if (IS_ERR(fwd_fdb
)) {
617 rule
= ERR_CAST(fwd_fdb
);
621 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
622 for (i
= 0; i
< esw_attr
->split_count
; i
++) {
623 if (esw_is_indir_table(esw
, attr
))
624 err
= esw_setup_indir_table(dest
, &flow_act
, esw
, attr
, spec
, false, &i
);
625 else if (esw_is_chain_src_port_rewrite(esw
, esw_attr
))
626 err
= esw_setup_chain_src_port_rewrite(dest
, &flow_act
, esw
, chains
, attr
,
629 esw_setup_vport_dest(dest
, &flow_act
, esw
, esw_attr
, i
, i
, false);
633 goto err_chain_src_rewrite
;
636 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
637 dest
[i
].ft
= fwd_fdb
;
640 mlx5_eswitch_set_rule_source_port(esw
, spec
, attr
,
641 esw_attr
->in_mdev
->priv
.eswitch
,
642 esw_attr
->in_rep
->vport
);
644 if (attr
->outer_match_level
!= MLX5_MATCH_NONE
)
645 spec
->match_criteria_enable
|= MLX5_MATCH_OUTER_HEADERS
;
647 flow_act
.flags
|= FLOW_ACT_IGNORE_FLOW_LEVEL
;
648 rule
= mlx5_add_flow_rules(fast_fdb
, spec
, &flow_act
, dest
, i
);
651 i
= esw_attr
->split_count
;
652 goto err_chain_src_rewrite
;
655 atomic64_inc(&esw
->offloads
.num_flows
);
658 err_chain_src_rewrite
:
659 esw_put_dest_tables_loop(esw
, attr
, 0, i
);
660 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
662 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
668 __mlx5_eswitch_del_rule(struct mlx5_eswitch
*esw
,
669 struct mlx5_flow_handle
*rule
,
670 struct mlx5_flow_attr
*attr
,
673 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
674 struct mlx5_fs_chains
*chains
= esw_chains(esw
);
675 bool split
= (esw_attr
->split_count
> 0);
676 struct mlx5_vport_tbl_attr fwd_attr
;
679 mlx5_del_flow_rules(rule
);
681 if (!mlx5_esw_attr_flags_skip(attr
->flags
)) {
682 /* unref the term table */
683 for (i
= 0; i
< MLX5_MAX_FLOW_FWD_VPORTS
; i
++) {
684 if (esw_attr
->dests
[i
].termtbl
)
685 mlx5_eswitch_termtbl_put(esw
, esw_attr
->dests
[i
].termtbl
);
689 atomic64_dec(&esw
->offloads
.num_flows
);
691 if (fwd_rule
|| split
) {
692 fwd_attr
.chain
= attr
->chain
;
693 fwd_attr
.prio
= attr
->prio
;
694 fwd_attr
.vport
= esw_attr
->in_rep
->vport
;
695 fwd_attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
699 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
700 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
701 esw_put_dest_tables_loop(esw
, attr
, 0, esw_attr
->split_count
);
704 mlx5_esw_vporttbl_put(esw
, &fwd_attr
);
705 else if (attr
->chain
|| attr
->prio
)
706 mlx5_chains_put_table(chains
, attr
->chain
, attr
->prio
, 0);
707 esw_cleanup_dests(esw
, attr
);
712 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch
*esw
,
713 struct mlx5_flow_handle
*rule
,
714 struct mlx5_flow_attr
*attr
)
716 __mlx5_eswitch_del_rule(esw
, rule
, attr
, false);
720 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch
*esw
,
721 struct mlx5_flow_handle
*rule
,
722 struct mlx5_flow_attr
*attr
)
724 __mlx5_eswitch_del_rule(esw
, rule
, attr
, true);
727 static int esw_set_global_vlan_pop(struct mlx5_eswitch
*esw
, u8 val
)
729 struct mlx5_eswitch_rep
*rep
;
733 esw_debug(esw
->dev
, "%s applying global %s policy\n", __func__
, val
? "pop" : "none");
734 mlx5_esw_for_each_host_func_vport(esw
, i
, rep
, esw
->esw_funcs
.num_vfs
) {
735 if (atomic_read(&rep
->rep_data
[REP_ETH
].state
) != REP_LOADED
)
738 err
= __mlx5_eswitch_set_vport_vlan(esw
, rep
->vport
, 0, 0, val
);
747 static struct mlx5_eswitch_rep
*
748 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr
*attr
, bool push
, bool pop
)
750 struct mlx5_eswitch_rep
*in_rep
, *out_rep
, *vport
= NULL
;
752 in_rep
= attr
->in_rep
;
753 out_rep
= attr
->dests
[0].rep
;
765 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr
*attr
,
766 bool push
, bool pop
, bool fwd
)
768 struct mlx5_eswitch_rep
*in_rep
, *out_rep
;
770 if ((push
|| pop
) && !fwd
)
773 in_rep
= attr
->in_rep
;
774 out_rep
= attr
->dests
[0].rep
;
776 if (push
&& in_rep
->vport
== MLX5_VPORT_UPLINK
)
779 if (pop
&& out_rep
->vport
== MLX5_VPORT_UPLINK
)
782 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
783 if (!push
&& !pop
&& fwd
)
784 if (in_rep
->vlan
&& out_rep
->vport
== MLX5_VPORT_UPLINK
)
787 /* protects against (1) setting rules with different vlans to push and
788 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
790 if (push
&& in_rep
->vlan_refcount
&& (in_rep
->vlan
!= attr
->vlan_vid
[0]))
799 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch
*esw
,
800 struct mlx5_flow_attr
*attr
)
802 struct offloads_fdb
*offloads
= &esw
->fdb_table
.offloads
;
803 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
804 struct mlx5_eswitch_rep
*vport
= NULL
;
808 /* nop if we're on the vlan push/pop non emulation mode */
809 if (mlx5_eswitch_vlan_actions_supported(esw
->dev
, 1))
812 push
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
);
813 pop
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
);
814 fwd
= !!((attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) &&
817 mutex_lock(&esw
->state_lock
);
819 err
= esw_add_vlan_action_check(esw_attr
, push
, pop
, fwd
);
823 attr
->flags
&= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED
;
825 vport
= esw_vlan_action_get_vport(esw_attr
, push
, pop
);
827 if (!push
&& !pop
&& fwd
) {
828 /* tracks VF --> wire rules without vlan push action */
829 if (esw_attr
->dests
[0].rep
->vport
== MLX5_VPORT_UPLINK
) {
830 vport
->vlan_refcount
++;
831 attr
->flags
|= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED
;
840 if (!(offloads
->vlan_push_pop_refcount
)) {
841 /* it's the 1st vlan rule, apply global vlan pop policy */
842 err
= esw_set_global_vlan_pop(esw
, SET_VLAN_STRIP
);
846 offloads
->vlan_push_pop_refcount
++;
849 if (vport
->vlan_refcount
)
852 err
= __mlx5_eswitch_set_vport_vlan(esw
, vport
->vport
, esw_attr
->vlan_vid
[0],
853 0, SET_VLAN_INSERT
| SET_VLAN_STRIP
);
856 vport
->vlan
= esw_attr
->vlan_vid
[0];
858 vport
->vlan_refcount
++;
862 attr
->flags
|= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED
;
864 mutex_unlock(&esw
->state_lock
);
868 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch
*esw
,
869 struct mlx5_flow_attr
*attr
)
871 struct offloads_fdb
*offloads
= &esw
->fdb_table
.offloads
;
872 struct mlx5_esw_flow_attr
*esw_attr
= attr
->esw_attr
;
873 struct mlx5_eswitch_rep
*vport
= NULL
;
877 /* nop if we're on the vlan push/pop non emulation mode */
878 if (mlx5_eswitch_vlan_actions_supported(esw
->dev
, 1))
881 if (!(attr
->flags
& MLX5_ESW_ATTR_FLAG_VLAN_HANDLED
))
884 push
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
);
885 pop
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
);
886 fwd
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
);
888 mutex_lock(&esw
->state_lock
);
890 vport
= esw_vlan_action_get_vport(esw_attr
, push
, pop
);
892 if (!push
&& !pop
&& fwd
) {
893 /* tracks VF --> wire rules without vlan push action */
894 if (esw_attr
->dests
[0].rep
->vport
== MLX5_VPORT_UPLINK
)
895 vport
->vlan_refcount
--;
901 vport
->vlan_refcount
--;
902 if (vport
->vlan_refcount
)
903 goto skip_unset_push
;
906 err
= __mlx5_eswitch_set_vport_vlan(esw
, vport
->vport
,
907 0, 0, SET_VLAN_STRIP
);
913 offloads
->vlan_push_pop_refcount
--;
914 if (offloads
->vlan_push_pop_refcount
)
917 /* no more vlan rules, stop global vlan pop policy */
918 err
= esw_set_global_vlan_pop(esw
, 0);
921 mutex_unlock(&esw
->state_lock
);
925 struct mlx5_flow_handle
*
926 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch
*on_esw
,
927 struct mlx5_eswitch
*from_esw
,
928 struct mlx5_eswitch_rep
*rep
,
931 struct mlx5_flow_act flow_act
= {0};
932 struct mlx5_flow_destination dest
= {};
933 struct mlx5_flow_handle
*flow_rule
;
934 struct mlx5_flow_spec
*spec
;
937 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
939 flow_rule
= ERR_PTR(-ENOMEM
);
943 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
944 MLX5_SET(fte_match_set_misc
, misc
, source_sqn
, sqn
);
945 /* source vport is the esw manager */
946 MLX5_SET(fte_match_set_misc
, misc
, source_port
, from_esw
->manager_vport
);
947 if (MLX5_CAP_ESW(on_esw
->dev
, merged_eswitch
))
948 MLX5_SET(fte_match_set_misc
, misc
, source_eswitch_owner_vhca_id
,
949 MLX5_CAP_GEN(from_esw
->dev
, vhca_id
));
951 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
952 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_sqn
);
953 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
954 if (MLX5_CAP_ESW(on_esw
->dev
, merged_eswitch
))
955 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
956 source_eswitch_owner_vhca_id
);
958 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
959 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
960 dest
.vport
.num
= rep
->vport
;
961 dest
.vport
.vhca_id
= MLX5_CAP_GEN(rep
->esw
->dev
, vhca_id
);
962 dest
.vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
963 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
965 if (rep
->vport
== MLX5_VPORT_UPLINK
)
966 spec
->flow_context
.flow_source
= MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT
;
968 flow_rule
= mlx5_add_flow_rules(on_esw
->fdb_table
.offloads
.slow_fdb
,
969 spec
, &flow_act
, &dest
, 1);
970 if (IS_ERR(flow_rule
))
971 esw_warn(on_esw
->dev
, "FDB: Failed to add send to vport rule err %ld\n",
977 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule
);
979 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle
*rule
)
981 mlx5_del_flow_rules(rule
);
984 static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch
*esw
)
986 struct mlx5_flow_handle
**flows
= esw
->fdb_table
.offloads
.send_to_vport_meta_rules
;
987 int i
= 0, num_vfs
= esw
->esw_funcs
.num_vfs
;
989 if (!num_vfs
|| !flows
)
992 for (i
= 0; i
< num_vfs
; i
++)
993 mlx5_del_flow_rules(flows
[i
]);
999 mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch
*esw
)
1001 struct mlx5_flow_destination dest
= {};
1002 struct mlx5_flow_act flow_act
= {0};
1003 int num_vfs
, rule_idx
= 0, err
= 0;
1004 struct mlx5_flow_handle
*flow_rule
;
1005 struct mlx5_flow_handle
**flows
;
1006 struct mlx5_flow_spec
*spec
;
1007 struct mlx5_vport
*vport
;
1011 num_vfs
= esw
->esw_funcs
.num_vfs
;
1012 flows
= kvzalloc(num_vfs
* sizeof(*flows
), GFP_KERNEL
);
1016 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1022 MLX5_SET(fte_match_param
, spec
->match_criteria
,
1023 misc_parameters_2
.metadata_reg_c_0
, mlx5_eswitch_get_vport_metadata_mask());
1024 MLX5_SET(fte_match_param
, spec
->match_criteria
,
1025 misc_parameters_2
.metadata_reg_c_1
, ESW_TUN_MASK
);
1026 MLX5_SET(fte_match_param
, spec
->match_value
, misc_parameters_2
.metadata_reg_c_1
,
1027 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK
);
1029 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1030 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
1031 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1033 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, num_vfs
) {
1034 vport_num
= vport
->vport
;
1035 MLX5_SET(fte_match_param
, spec
->match_value
, misc_parameters_2
.metadata_reg_c_0
,
1036 mlx5_eswitch_get_vport_metadata_for_match(esw
, vport_num
));
1037 dest
.vport
.num
= vport_num
;
1039 flow_rule
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1040 spec
, &flow_act
, &dest
, 1);
1041 if (IS_ERR(flow_rule
)) {
1042 err
= PTR_ERR(flow_rule
);
1043 esw_warn(esw
->dev
, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n",
1044 rule_idx
, PTR_ERR(flow_rule
));
1047 flows
[rule_idx
++] = flow_rule
;
1050 esw
->fdb_table
.offloads
.send_to_vport_meta_rules
= flows
;
1055 while (--rule_idx
>= 0)
1056 mlx5_del_flow_rules(flows
[rule_idx
]);
1063 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch
*esw
)
1065 return MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, fdb_to_vport_reg_c_id
) &
1066 MLX5_FDB_TO_VPORT_REG_C_1
;
1069 static int esw_set_passing_vport_metadata(struct mlx5_eswitch
*esw
, bool enable
)
1071 u32 out
[MLX5_ST_SZ_DW(query_esw_vport_context_out
)] = {};
1072 u32 min
[MLX5_ST_SZ_DW(modify_esw_vport_context_in
)] = {};
1073 u32 in
[MLX5_ST_SZ_DW(query_esw_vport_context_in
)] = {};
1077 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
) &&
1078 !mlx5_eswitch_vport_match_metadata_enabled(esw
))
1081 MLX5_SET(query_esw_vport_context_in
, in
, opcode
,
1082 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT
);
1083 err
= mlx5_cmd_exec_inout(esw
->dev
, query_esw_vport_context
, in
, out
);
1087 curr
= MLX5_GET(query_esw_vport_context_out
, out
,
1088 esw_vport_context
.fdb_to_vport_reg_c_id
);
1089 wanted
= MLX5_FDB_TO_VPORT_REG_C_0
;
1090 if (mlx5_eswitch_reg_c1_loopback_supported(esw
))
1091 wanted
|= MLX5_FDB_TO_VPORT_REG_C_1
;
1098 MLX5_SET(modify_esw_vport_context_in
, min
,
1099 esw_vport_context
.fdb_to_vport_reg_c_id
, curr
);
1100 MLX5_SET(modify_esw_vport_context_in
, min
,
1101 field_select
.fdb_to_vport_reg_c_id
, 1);
1103 err
= mlx5_eswitch_modify_esw_vport_context(esw
->dev
, 0, false, min
);
1105 if (enable
&& (curr
& MLX5_FDB_TO_VPORT_REG_C_1
))
1106 esw
->flags
|= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED
;
1108 esw
->flags
&= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED
;
1114 static void peer_miss_rules_setup(struct mlx5_eswitch
*esw
,
1115 struct mlx5_core_dev
*peer_dev
,
1116 struct mlx5_flow_spec
*spec
,
1117 struct mlx5_flow_destination
*dest
)
1121 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1122 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1124 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1125 mlx5_eswitch_get_vport_metadata_mask());
1127 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1129 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1132 MLX5_SET(fte_match_set_misc
, misc
, source_eswitch_owner_vhca_id
,
1133 MLX5_CAP_GEN(peer_dev
, vhca_id
));
1135 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
1137 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1139 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
1140 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
1141 source_eswitch_owner_vhca_id
);
1144 dest
->type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
1145 dest
->vport
.num
= peer_dev
->priv
.eswitch
->manager_vport
;
1146 dest
->vport
.vhca_id
= MLX5_CAP_GEN(peer_dev
, vhca_id
);
1147 dest
->vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
1150 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch
*esw
,
1151 struct mlx5_eswitch
*peer_esw
,
1152 struct mlx5_flow_spec
*spec
,
1157 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1158 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1160 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1161 mlx5_eswitch_get_vport_metadata_for_match(peer_esw
,
1164 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1166 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
1170 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch
*esw
,
1171 struct mlx5_core_dev
*peer_dev
)
1173 struct mlx5_flow_destination dest
= {};
1174 struct mlx5_flow_act flow_act
= {0};
1175 struct mlx5_flow_handle
**flows
;
1176 /* total vports is the same for both e-switches */
1177 int nvports
= esw
->total_vports
;
1178 struct mlx5_flow_handle
*flow
;
1179 struct mlx5_flow_spec
*spec
;
1180 struct mlx5_vport
*vport
;
1185 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1189 peer_miss_rules_setup(esw
, peer_dev
, spec
, &dest
);
1191 flows
= kvzalloc(nvports
* sizeof(*flows
), GFP_KERNEL
);
1194 goto alloc_flows_err
;
1197 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1198 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1201 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
1202 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_PF
);
1203 esw_set_peer_miss_rule_source_port(esw
, peer_dev
->priv
.eswitch
,
1204 spec
, MLX5_VPORT_PF
);
1206 flow
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1207 spec
, &flow_act
, &dest
, 1);
1209 err
= PTR_ERR(flow
);
1210 goto add_pf_flow_err
;
1212 flows
[vport
->index
] = flow
;
1215 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
1216 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_ECPF
);
1217 MLX5_SET(fte_match_set_misc
, misc
, source_port
, MLX5_VPORT_ECPF
);
1218 flow
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1219 spec
, &flow_act
, &dest
, 1);
1221 err
= PTR_ERR(flow
);
1222 goto add_ecpf_flow_err
;
1224 flows
[vport
->index
] = flow
;
1227 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, mlx5_core_max_vfs(esw
->dev
)) {
1228 esw_set_peer_miss_rule_source_port(esw
,
1229 peer_dev
->priv
.eswitch
,
1230 spec
, vport
->vport
);
1232 flow
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1233 spec
, &flow_act
, &dest
, 1);
1235 err
= PTR_ERR(flow
);
1236 goto add_vf_flow_err
;
1238 flows
[vport
->index
] = flow
;
1241 esw
->fdb_table
.offloads
.peer_miss_rules
= flows
;
1247 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, mlx5_core_max_vfs(esw
->dev
)) {
1248 if (!flows
[vport
->index
])
1250 mlx5_del_flow_rules(flows
[vport
->index
]);
1252 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
1253 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_ECPF
);
1254 mlx5_del_flow_rules(flows
[vport
->index
]);
1257 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
1258 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_PF
);
1259 mlx5_del_flow_rules(flows
[vport
->index
]);
1262 esw_warn(esw
->dev
, "FDB: Failed to add peer miss flow rule err %d\n", err
);
1269 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch
*esw
)
1271 struct mlx5_flow_handle
**flows
;
1272 struct mlx5_vport
*vport
;
1275 flows
= esw
->fdb_table
.offloads
.peer_miss_rules
;
1277 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, mlx5_core_max_vfs(esw
->dev
))
1278 mlx5_del_flow_rules(flows
[vport
->index
]);
1280 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
1281 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_ECPF
);
1282 mlx5_del_flow_rules(flows
[vport
->index
]);
1285 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
1286 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_PF
);
1287 mlx5_del_flow_rules(flows
[vport
->index
]);
1292 static int esw_add_fdb_miss_rule(struct mlx5_eswitch
*esw
)
1294 struct mlx5_flow_act flow_act
= {0};
1295 struct mlx5_flow_destination dest
= {};
1296 struct mlx5_flow_handle
*flow_rule
= NULL
;
1297 struct mlx5_flow_spec
*spec
;
1304 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1310 spec
->match_criteria_enable
= MLX5_MATCH_OUTER_HEADERS
;
1311 headers_c
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1313 dmac_c
= MLX5_ADDR_OF(fte_match_param
, headers_c
,
1314 outer_headers
.dmac_47_16
);
1317 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
1318 dest
.vport
.num
= esw
->manager_vport
;
1319 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1321 flow_rule
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1322 spec
, &flow_act
, &dest
, 1);
1323 if (IS_ERR(flow_rule
)) {
1324 err
= PTR_ERR(flow_rule
);
1325 esw_warn(esw
->dev
, "FDB: Failed to add unicast miss flow rule err %d\n", err
);
1329 esw
->fdb_table
.offloads
.miss_rule_uni
= flow_rule
;
1331 headers_v
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1333 dmac_v
= MLX5_ADDR_OF(fte_match_param
, headers_v
,
1334 outer_headers
.dmac_47_16
);
1336 flow_rule
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.slow_fdb
,
1337 spec
, &flow_act
, &dest
, 1);
1338 if (IS_ERR(flow_rule
)) {
1339 err
= PTR_ERR(flow_rule
);
1340 esw_warn(esw
->dev
, "FDB: Failed to add multicast miss flow rule err %d\n", err
);
1341 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule_uni
);
1345 esw
->fdb_table
.offloads
.miss_rule_multi
= flow_rule
;
1352 struct mlx5_flow_handle
*
1353 esw_add_restore_rule(struct mlx5_eswitch
*esw
, u32 tag
)
1355 struct mlx5_flow_act flow_act
= { .flags
= FLOW_ACT_NO_APPEND
, };
1356 struct mlx5_flow_table
*ft
= esw
->offloads
.ft_offloads_restore
;
1357 struct mlx5_flow_context
*flow_context
;
1358 struct mlx5_flow_handle
*flow_rule
;
1359 struct mlx5_flow_destination dest
;
1360 struct mlx5_flow_spec
*spec
;
1363 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
))
1364 return ERR_PTR(-EOPNOTSUPP
);
1366 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1368 return ERR_PTR(-ENOMEM
);
1370 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
,
1372 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1373 ESW_REG_C0_USER_DATA_METADATA_MASK
);
1374 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
1376 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
, tag
);
1377 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1378 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
|
1379 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
;
1380 flow_act
.modify_hdr
= esw
->offloads
.restore_copy_hdr_id
;
1382 flow_context
= &spec
->flow_context
;
1383 flow_context
->flags
|= FLOW_CONTEXT_HAS_TAG
;
1384 flow_context
->flow_tag
= tag
;
1385 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
1386 dest
.ft
= esw
->offloads
.ft_offloads
;
1388 flow_rule
= mlx5_add_flow_rules(ft
, spec
, &flow_act
, &dest
, 1);
1391 if (IS_ERR(flow_rule
))
1393 "Failed to create restore rule for tag: %d, err(%d)\n",
1394 tag
, (int)PTR_ERR(flow_rule
));
1399 #define MAX_PF_SQ 256
1400 #define MAX_SQ_NVPORTS 32
1402 static void esw_set_flow_group_source_port(struct mlx5_eswitch
*esw
,
1405 void *match_criteria
= MLX5_ADDR_OF(create_flow_group_in
,
1409 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1410 MLX5_SET(create_flow_group_in
, flow_group_in
,
1411 match_criteria_enable
,
1412 MLX5_MATCH_MISC_PARAMETERS_2
);
1414 MLX5_SET(fte_match_param
, match_criteria
,
1415 misc_parameters_2
.metadata_reg_c_0
,
1416 mlx5_eswitch_get_vport_metadata_mask());
1418 MLX5_SET(create_flow_group_in
, flow_group_in
,
1419 match_criteria_enable
,
1420 MLX5_MATCH_MISC_PARAMETERS
);
1422 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
1423 misc_parameters
.source_port
);
1427 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
1428 static void esw_vport_tbl_put(struct mlx5_eswitch
*esw
)
1430 struct mlx5_vport_tbl_attr attr
;
1431 struct mlx5_vport
*vport
;
1436 mlx5_esw_for_each_vport(esw
, i
, vport
) {
1437 attr
.vport
= vport
->vport
;
1438 attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
1439 mlx5_esw_vporttbl_put(esw
, &attr
);
1443 static int esw_vport_tbl_get(struct mlx5_eswitch
*esw
)
1445 struct mlx5_vport_tbl_attr attr
;
1446 struct mlx5_flow_table
*fdb
;
1447 struct mlx5_vport
*vport
;
1452 mlx5_esw_for_each_vport(esw
, i
, vport
) {
1453 attr
.vport
= vport
->vport
;
1454 attr
.vport_ns
= &mlx5_esw_vport_tbl_mirror_ns
;
1455 fdb
= mlx5_esw_vporttbl_get(esw
, &attr
);
1462 esw_vport_tbl_put(esw
);
1463 return PTR_ERR(fdb
);
1466 #define fdb_modify_header_fwd_to_table_supported(esw) \
1467 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
1468 static void esw_init_chains_offload_flags(struct mlx5_eswitch
*esw
, u32
*flags
)
1470 struct mlx5_core_dev
*dev
= esw
->dev
;
1472 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, ignore_flow_level
))
1473 *flags
|= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED
;
1475 if (!MLX5_CAP_ESW_FLOWTABLE(dev
, multi_fdb_encap
) &&
1476 esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
) {
1477 *flags
&= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1478 esw_warn(dev
, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1479 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw
)) {
1480 *flags
&= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1481 esw_warn(dev
, "Tc chains and priorities offload aren't supported\n");
1482 } else if (!fdb_modify_header_fwd_to_table_supported(esw
)) {
1483 /* Disabled when ttl workaround is needed, e.g
1484 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1487 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1488 *flags
&= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1490 *flags
|= MLX5_CHAINS_AND_PRIOS_SUPPORTED
;
1491 esw_info(dev
, "Supported tc chains and prios offload\n");
1494 if (esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
)
1495 *flags
|= MLX5_CHAINS_FT_TUNNEL_SUPPORTED
;
1499 esw_chains_create(struct mlx5_eswitch
*esw
, struct mlx5_flow_table
*miss_fdb
)
1501 struct mlx5_core_dev
*dev
= esw
->dev
;
1502 struct mlx5_flow_table
*nf_ft
, *ft
;
1503 struct mlx5_chains_attr attr
= {};
1504 struct mlx5_fs_chains
*chains
;
1508 fdb_max
= 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, log_max_ft_size
);
1510 esw_init_chains_offload_flags(esw
, &attr
.flags
);
1511 attr
.ns
= MLX5_FLOW_NAMESPACE_FDB
;
1512 attr
.max_ft_sz
= fdb_max
;
1513 attr
.max_grp_num
= esw
->params
.large_group_num
;
1514 attr
.default_ft
= miss_fdb
;
1515 attr
.mapping
= esw
->offloads
.reg_c0_obj_pool
;
1517 chains
= mlx5_chains_create(dev
, &attr
);
1518 if (IS_ERR(chains
)) {
1519 err
= PTR_ERR(chains
);
1520 esw_warn(dev
, "Failed to create fdb chains err(%d)\n", err
);
1524 esw
->fdb_table
.offloads
.esw_chains_priv
= chains
;
1526 /* Create tc_end_ft which is the always created ft chain */
1527 nf_ft
= mlx5_chains_get_table(chains
, mlx5_chains_get_nf_ft_chain(chains
),
1529 if (IS_ERR(nf_ft
)) {
1530 err
= PTR_ERR(nf_ft
);
1534 /* Always open the root for fast path */
1535 ft
= mlx5_chains_get_table(chains
, 0, 1, 0);
1541 /* Open level 1 for split fdb rules now if prios isn't supported */
1542 if (!mlx5_chains_prios_supported(chains
)) {
1543 err
= esw_vport_tbl_get(esw
);
1548 mlx5_chains_set_end_ft(chains
, nf_ft
);
1553 mlx5_chains_put_table(chains
, 0, 1, 0);
1555 mlx5_chains_put_table(chains
, mlx5_chains_get_nf_ft_chain(chains
), 1, 0);
1557 mlx5_chains_destroy(chains
);
1558 esw
->fdb_table
.offloads
.esw_chains_priv
= NULL
;
1564 esw_chains_destroy(struct mlx5_eswitch
*esw
, struct mlx5_fs_chains
*chains
)
1566 if (!mlx5_chains_prios_supported(chains
))
1567 esw_vport_tbl_put(esw
);
1568 mlx5_chains_put_table(chains
, 0, 1, 0);
1569 mlx5_chains_put_table(chains
, mlx5_chains_get_nf_ft_chain(chains
), 1, 0);
1570 mlx5_chains_destroy(chains
);
1573 #else /* CONFIG_MLX5_CLS_ACT */
1576 esw_chains_create(struct mlx5_eswitch
*esw
, struct mlx5_flow_table
*miss_fdb
)
1580 esw_chains_destroy(struct mlx5_eswitch
*esw
, struct mlx5_fs_chains
*chains
)
1585 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch
*esw
)
1587 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1588 struct mlx5_flow_table_attr ft_attr
= {};
1589 int num_vfs
, table_size
, ix
, err
= 0;
1590 struct mlx5_core_dev
*dev
= esw
->dev
;
1591 struct mlx5_flow_namespace
*root_ns
;
1592 struct mlx5_flow_table
*fdb
= NULL
;
1593 u32 flags
= 0, *flow_group_in
;
1594 struct mlx5_flow_group
*g
;
1595 void *match_criteria
;
1598 esw_debug(esw
->dev
, "Create offloads FDB Tables\n");
1600 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
1604 root_ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_FDB
);
1606 esw_warn(dev
, "Failed to get FDB flow namespace\n");
1610 esw
->fdb_table
.offloads
.ns
= root_ns
;
1611 err
= mlx5_flow_namespace_set_mode(root_ns
,
1612 esw
->dev
->priv
.steering
->mode
);
1614 esw_warn(dev
, "Failed to set FDB namespace steering mode\n");
1618 /* To be strictly correct:
1619 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1621 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1622 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1623 * but as the peer device might not be in switchdev mode it's not
1624 * possible. We use the fact that by default FW sets max vfs and max sfs
1625 * to the same value on both devices. If it needs to be changed in the future note
1626 * the peer miss group should also be created based on the number of
1627 * total vports of the peer (currently is also uses esw->total_vports).
1629 table_size
= MLX5_MAX_PORTS
* (esw
->total_vports
* MAX_SQ_NVPORTS
+ MAX_PF_SQ
) +
1630 MLX5_ESW_MISS_FLOWS
+ esw
->total_vports
+ esw
->esw_funcs
.num_vfs
;
1632 /* create the slow path fdb with encap set, so further table instances
1633 * can be created at run time while VFs are probed if the FW allows that.
1635 if (esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
)
1636 flags
|= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT
|
1637 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP
);
1639 ft_attr
.flags
= flags
;
1640 ft_attr
.max_fte
= table_size
;
1641 ft_attr
.prio
= FDB_SLOW_PATH
;
1643 fdb
= mlx5_create_flow_table(root_ns
, &ft_attr
);
1646 esw_warn(dev
, "Failed to create slow path FDB Table err %d\n", err
);
1649 esw
->fdb_table
.offloads
.slow_fdb
= fdb
;
1651 /* Create empty TC-miss managed table. This allows plugging in following
1652 * priorities without directly exposing their level 0 table to
1653 * eswitch_offloads and passing it as miss_fdb to following call to
1654 * esw_chains_create().
1656 memset(&ft_attr
, 0, sizeof(ft_attr
));
1657 ft_attr
.prio
= FDB_TC_MISS
;
1658 esw
->fdb_table
.offloads
.tc_miss_table
= mlx5_create_flow_table(root_ns
, &ft_attr
);
1659 if (IS_ERR(esw
->fdb_table
.offloads
.tc_miss_table
)) {
1660 err
= PTR_ERR(esw
->fdb_table
.offloads
.tc_miss_table
);
1661 esw_warn(dev
, "Failed to create TC miss FDB Table err %d\n", err
);
1662 goto tc_miss_table_err
;
1665 err
= esw_chains_create(esw
, esw
->fdb_table
.offloads
.tc_miss_table
);
1667 esw_warn(dev
, "Failed to open fdb chains err(%d)\n", err
);
1668 goto fdb_chains_err
;
1671 /* create send-to-vport group */
1672 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
1673 MLX5_MATCH_MISC_PARAMETERS
);
1675 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
1677 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_sqn
);
1678 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_port
);
1679 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
)) {
1680 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
1681 misc_parameters
.source_eswitch_owner_vhca_id
);
1682 MLX5_SET(create_flow_group_in
, flow_group_in
,
1683 source_eswitch_owner_vhca_id_valid
, 1);
1686 /* See comment above table_size calculation */
1687 ix
= MLX5_MAX_PORTS
* (esw
->total_vports
* MAX_SQ_NVPORTS
+ MAX_PF_SQ
);
1688 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
1689 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, ix
- 1);
1691 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1694 esw_warn(dev
, "Failed to create send-to-vport flow group err(%d)\n", err
);
1695 goto send_vport_err
;
1697 esw
->fdb_table
.offloads
.send_to_vport_grp
= g
;
1699 if (esw_src_port_rewrite_supported(esw
)) {
1700 /* meta send to vport */
1701 memset(flow_group_in
, 0, inlen
);
1702 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
1703 MLX5_MATCH_MISC_PARAMETERS_2
);
1705 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
1707 MLX5_SET(fte_match_param
, match_criteria
,
1708 misc_parameters_2
.metadata_reg_c_0
,
1709 mlx5_eswitch_get_vport_metadata_mask());
1710 MLX5_SET(fte_match_param
, match_criteria
,
1711 misc_parameters_2
.metadata_reg_c_1
, ESW_TUN_MASK
);
1713 num_vfs
= esw
->esw_funcs
.num_vfs
;
1715 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, ix
);
1716 MLX5_SET(create_flow_group_in
, flow_group_in
,
1717 end_flow_index
, ix
+ num_vfs
- 1);
1720 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1723 esw_warn(dev
, "Failed to create send-to-vport meta flow group err(%d)\n",
1725 goto send_vport_meta_err
;
1727 esw
->fdb_table
.offloads
.send_to_vport_meta_grp
= g
;
1729 err
= mlx5_eswitch_add_send_to_vport_meta_rules(esw
);
1735 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
)) {
1736 /* create peer esw miss group */
1737 memset(flow_group_in
, 0, inlen
);
1739 esw_set_flow_group_source_port(esw
, flow_group_in
);
1741 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1742 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
,
1746 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
1747 misc_parameters
.source_eswitch_owner_vhca_id
);
1749 MLX5_SET(create_flow_group_in
, flow_group_in
,
1750 source_eswitch_owner_vhca_id_valid
, 1);
1753 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, ix
);
1754 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
,
1755 ix
+ esw
->total_vports
- 1);
1756 ix
+= esw
->total_vports
;
1758 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1761 esw_warn(dev
, "Failed to create peer miss flow group err(%d)\n", err
);
1764 esw
->fdb_table
.offloads
.peer_miss_grp
= g
;
1767 /* create miss group */
1768 memset(flow_group_in
, 0, inlen
);
1769 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
1770 MLX5_MATCH_OUTER_HEADERS
);
1771 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
,
1773 dmac
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
1774 outer_headers
.dmac_47_16
);
1777 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, ix
);
1778 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
,
1779 ix
+ MLX5_ESW_MISS_FLOWS
);
1781 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
1784 esw_warn(dev
, "Failed to create miss flow group err(%d)\n", err
);
1787 esw
->fdb_table
.offloads
.miss_grp
= g
;
1789 err
= esw_add_fdb_miss_rule(esw
);
1793 kvfree(flow_group_in
);
1797 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
1799 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
1800 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.peer_miss_grp
);
1802 mlx5_eswitch_del_send_to_vport_meta_rules(esw
);
1804 if (esw
->fdb_table
.offloads
.send_to_vport_meta_grp
)
1805 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_meta_grp
);
1806 send_vport_meta_err
:
1807 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
1809 esw_chains_destroy(esw
, esw_chains(esw
));
1811 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.tc_miss_table
);
1813 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.slow_fdb
);
1815 /* Holds true only as long as DMFS is the default */
1816 mlx5_flow_namespace_set_mode(root_ns
, MLX5_FLOW_STEERING_MODE_DMFS
);
1818 kvfree(flow_group_in
);
1822 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch
*esw
)
1824 if (!esw
->fdb_table
.offloads
.slow_fdb
)
1827 esw_debug(esw
->dev
, "Destroy offloads FDB Tables\n");
1828 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule_multi
);
1829 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule_uni
);
1830 mlx5_eswitch_del_send_to_vport_meta_rules(esw
);
1831 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
1832 if (esw
->fdb_table
.offloads
.send_to_vport_meta_grp
)
1833 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_meta_grp
);
1834 if (MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
1835 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.peer_miss_grp
);
1836 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
1838 esw_chains_destroy(esw
, esw_chains(esw
));
1840 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.tc_miss_table
);
1841 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.slow_fdb
);
1842 /* Holds true only as long as DMFS is the default */
1843 mlx5_flow_namespace_set_mode(esw
->fdb_table
.offloads
.ns
,
1844 MLX5_FLOW_STEERING_MODE_DMFS
);
1845 atomic64_set(&esw
->user_count
, 0);
1848 static int esw_get_offloads_ft_size(struct mlx5_eswitch
*esw
)
1852 nvports
= esw
->total_vports
+ MLX5_ESW_MISS_FLOWS
;
1853 if (mlx5e_tc_int_port_supported(esw
))
1854 nvports
+= MLX5E_TC_MAX_INT_PORT_NUM
;
1859 static int esw_create_offloads_table(struct mlx5_eswitch
*esw
)
1861 struct mlx5_flow_table_attr ft_attr
= {};
1862 struct mlx5_core_dev
*dev
= esw
->dev
;
1863 struct mlx5_flow_table
*ft_offloads
;
1864 struct mlx5_flow_namespace
*ns
;
1867 ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
1869 esw_warn(esw
->dev
, "Failed to get offloads flow namespace\n");
1873 ft_attr
.max_fte
= esw_get_offloads_ft_size(esw
);
1876 ft_offloads
= mlx5_create_flow_table(ns
, &ft_attr
);
1877 if (IS_ERR(ft_offloads
)) {
1878 err
= PTR_ERR(ft_offloads
);
1879 esw_warn(esw
->dev
, "Failed to create offloads table, err %d\n", err
);
1883 esw
->offloads
.ft_offloads
= ft_offloads
;
1887 static void esw_destroy_offloads_table(struct mlx5_eswitch
*esw
)
1889 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
1891 mlx5_destroy_flow_table(offloads
->ft_offloads
);
1894 static int esw_create_vport_rx_group(struct mlx5_eswitch
*esw
)
1896 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1897 struct mlx5_flow_group
*g
;
1902 nvports
= esw_get_offloads_ft_size(esw
);
1903 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
1907 /* create vport rx group */
1908 esw_set_flow_group_source_port(esw
, flow_group_in
);
1910 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
1911 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, nvports
- 1);
1913 g
= mlx5_create_flow_group(esw
->offloads
.ft_offloads
, flow_group_in
);
1917 mlx5_core_warn(esw
->dev
, "Failed to create vport rx group err %d\n", err
);
1921 esw
->offloads
.vport_rx_group
= g
;
1923 kvfree(flow_group_in
);
1927 static void esw_destroy_vport_rx_group(struct mlx5_eswitch
*esw
)
1929 mlx5_destroy_flow_group(esw
->offloads
.vport_rx_group
);
1932 struct mlx5_flow_handle
*
1933 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch
*esw
, u16 vport
,
1934 struct mlx5_flow_destination
*dest
)
1936 struct mlx5_flow_act flow_act
= {0};
1937 struct mlx5_flow_handle
*flow_rule
;
1938 struct mlx5_flow_spec
*spec
;
1941 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
1943 flow_rule
= ERR_PTR(-ENOMEM
);
1947 if (mlx5_eswitch_vport_match_metadata_enabled(esw
)) {
1948 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters_2
);
1949 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1950 mlx5_eswitch_get_vport_metadata_for_match(esw
, vport
));
1952 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters_2
);
1953 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
1954 mlx5_eswitch_get_vport_metadata_mask());
1956 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS_2
;
1958 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
1959 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
1961 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
1962 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
1964 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
1967 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1968 flow_rule
= mlx5_add_flow_rules(esw
->offloads
.ft_offloads
, spec
,
1969 &flow_act
, dest
, 1);
1970 if (IS_ERR(flow_rule
)) {
1971 esw_warn(esw
->dev
, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule
));
1980 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch
*esw
, u8
*mode
)
1982 u8 prev_mlx5_mode
, mlx5_mode
= MLX5_INLINE_MODE_L2
;
1983 struct mlx5_core_dev
*dev
= esw
->dev
;
1984 struct mlx5_vport
*vport
;
1987 if (!MLX5_CAP_GEN(dev
, vport_group_manager
))
1990 if (esw
->mode
== MLX5_ESWITCH_NONE
)
1993 switch (MLX5_CAP_ETH(dev
, wqe_inline_mode
)) {
1994 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED
:
1995 mlx5_mode
= MLX5_INLINE_MODE_NONE
;
1997 case MLX5_CAP_INLINE_MODE_L2
:
1998 mlx5_mode
= MLX5_INLINE_MODE_L2
;
2000 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
:
2005 mlx5_query_nic_vport_min_inline(dev
, esw
->first_host_vport
, &prev_mlx5_mode
);
2006 mlx5_esw_for_each_host_func_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
) {
2007 mlx5_query_nic_vport_min_inline(dev
, vport
->vport
, &mlx5_mode
);
2008 if (prev_mlx5_mode
!= mlx5_mode
)
2010 prev_mlx5_mode
= mlx5_mode
;
2018 static void esw_destroy_restore_table(struct mlx5_eswitch
*esw
)
2020 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
2022 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
))
2025 mlx5_modify_header_dealloc(esw
->dev
, offloads
->restore_copy_hdr_id
);
2026 mlx5_destroy_flow_group(offloads
->restore_group
);
2027 mlx5_destroy_flow_table(offloads
->ft_offloads_restore
);
2030 static int esw_create_restore_table(struct mlx5_eswitch
*esw
)
2032 u8 modact
[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto
)] = {};
2033 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2034 struct mlx5_flow_table_attr ft_attr
= {};
2035 struct mlx5_core_dev
*dev
= esw
->dev
;
2036 struct mlx5_flow_namespace
*ns
;
2037 struct mlx5_modify_hdr
*mod_hdr
;
2038 void *match_criteria
, *misc
;
2039 struct mlx5_flow_table
*ft
;
2040 struct mlx5_flow_group
*g
;
2044 if (!mlx5_eswitch_reg_c1_loopback_supported(esw
))
2047 ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
2049 esw_warn(esw
->dev
, "Failed to get offloads flow namespace\n");
2053 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2054 if (!flow_group_in
) {
2059 ft_attr
.max_fte
= 1 << ESW_REG_C0_USER_DATA_METADATA_BITS
;
2060 ft
= mlx5_create_flow_table(ns
, &ft_attr
);
2063 esw_warn(esw
->dev
, "Failed to create restore table, err %d\n",
2068 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
,
2070 misc
= MLX5_ADDR_OF(fte_match_param
, match_criteria
,
2073 MLX5_SET(fte_match_set_misc2
, misc
, metadata_reg_c_0
,
2074 ESW_REG_C0_USER_DATA_METADATA_MASK
);
2075 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
2076 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
,
2077 ft_attr
.max_fte
- 1);
2078 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
2079 MLX5_MATCH_MISC_PARAMETERS_2
);
2080 g
= mlx5_create_flow_group(ft
, flow_group_in
);
2083 esw_warn(dev
, "Failed to create restore flow group, err: %d\n",
2088 MLX5_SET(copy_action_in
, modact
, action_type
, MLX5_ACTION_TYPE_COPY
);
2089 MLX5_SET(copy_action_in
, modact
, src_field
,
2090 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1
);
2091 MLX5_SET(copy_action_in
, modact
, dst_field
,
2092 MLX5_ACTION_IN_FIELD_METADATA_REG_B
);
2093 mod_hdr
= mlx5_modify_header_alloc(esw
->dev
,
2094 MLX5_FLOW_NAMESPACE_KERNEL
, 1,
2096 if (IS_ERR(mod_hdr
)) {
2097 err
= PTR_ERR(mod_hdr
);
2098 esw_warn(dev
, "Failed to create restore mod header, err: %d\n",
2103 esw
->offloads
.ft_offloads_restore
= ft
;
2104 esw
->offloads
.restore_group
= g
;
2105 esw
->offloads
.restore_copy_hdr_id
= mod_hdr
;
2107 kvfree(flow_group_in
);
2112 mlx5_destroy_flow_group(g
);
2114 mlx5_destroy_flow_table(ft
);
2116 kvfree(flow_group_in
);
2121 static int esw_offloads_start(struct mlx5_eswitch
*esw
,
2122 struct netlink_ext_ack
*extack
)
2126 mlx5_eswitch_disable_locked(esw
, false);
2127 err
= mlx5_eswitch_enable_locked(esw
, MLX5_ESWITCH_OFFLOADS
,
2128 esw
->dev
->priv
.sriov
.num_vfs
);
2130 NL_SET_ERR_MSG_MOD(extack
,
2131 "Failed setting eswitch to offloads");
2132 err1
= mlx5_eswitch_enable_locked(esw
, MLX5_ESWITCH_LEGACY
,
2133 MLX5_ESWITCH_IGNORE_NUM_VFS
);
2135 NL_SET_ERR_MSG_MOD(extack
,
2136 "Failed setting eswitch back to legacy");
2139 if (esw
->offloads
.inline_mode
== MLX5_INLINE_MODE_NONE
) {
2140 if (mlx5_eswitch_inline_mode_get(esw
,
2141 &esw
->offloads
.inline_mode
)) {
2142 esw
->offloads
.inline_mode
= MLX5_INLINE_MODE_L2
;
2143 NL_SET_ERR_MSG_MOD(extack
,
2144 "Inline mode is different between vports");
2150 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch
*esw
,
2151 struct mlx5_eswitch_rep
*rep
,
2156 /* Copy the mark from vport to its rep */
2157 mark_set
= xa_get_mark(&esw
->vports
, rep
->vport
, mark
);
2159 xa_set_mark(&esw
->offloads
.vport_reps
, rep
->vport
, mark
);
2162 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch
*esw
, const struct mlx5_vport
*vport
)
2164 struct mlx5_eswitch_rep
*rep
;
2168 rep
= kzalloc(sizeof(*rep
), GFP_KERNEL
);
2172 rep
->vport
= vport
->vport
;
2173 rep
->vport_index
= vport
->index
;
2174 for (rep_type
= 0; rep_type
< NUM_REP_TYPES
; rep_type
++)
2175 atomic_set(&rep
->rep_data
[rep_type
].state
, REP_UNREGISTERED
);
2177 err
= xa_insert(&esw
->offloads
.vport_reps
, rep
->vport
, rep
, GFP_KERNEL
);
2181 mlx5_esw_offloads_rep_mark_set(esw
, rep
, MLX5_ESW_VPT_HOST_FN
);
2182 mlx5_esw_offloads_rep_mark_set(esw
, rep
, MLX5_ESW_VPT_VF
);
2183 mlx5_esw_offloads_rep_mark_set(esw
, rep
, MLX5_ESW_VPT_SF
);
2191 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch
*esw
,
2192 struct mlx5_eswitch_rep
*rep
)
2194 xa_erase(&esw
->offloads
.vport_reps
, rep
->vport
);
2198 void esw_offloads_cleanup_reps(struct mlx5_eswitch
*esw
)
2200 struct mlx5_eswitch_rep
*rep
;
2203 mlx5_esw_for_each_rep(esw
, i
, rep
)
2204 mlx5_esw_offloads_rep_cleanup(esw
, rep
);
2205 xa_destroy(&esw
->offloads
.vport_reps
);
2208 int esw_offloads_init_reps(struct mlx5_eswitch
*esw
)
2210 struct mlx5_vport
*vport
;
2214 xa_init(&esw
->offloads
.vport_reps
);
2216 mlx5_esw_for_each_vport(esw
, i
, vport
) {
2217 err
= mlx5_esw_offloads_rep_init(esw
, vport
);
2224 esw_offloads_cleanup_reps(esw
);
2228 static void __esw_offloads_unload_rep(struct mlx5_eswitch
*esw
,
2229 struct mlx5_eswitch_rep
*rep
, u8 rep_type
)
2231 if (atomic_cmpxchg(&rep
->rep_data
[rep_type
].state
,
2232 REP_LOADED
, REP_REGISTERED
) == REP_LOADED
)
2233 esw
->offloads
.rep_ops
[rep_type
]->unload(rep
);
2236 static void __unload_reps_sf_vport(struct mlx5_eswitch
*esw
, u8 rep_type
)
2238 struct mlx5_eswitch_rep
*rep
;
2241 mlx5_esw_for_each_sf_rep(esw
, i
, rep
)
2242 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2245 static void __unload_reps_all_vport(struct mlx5_eswitch
*esw
, u8 rep_type
)
2247 struct mlx5_eswitch_rep
*rep
;
2250 __unload_reps_sf_vport(esw
, rep_type
);
2252 mlx5_esw_for_each_vf_rep(esw
, i
, rep
)
2253 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2255 if (mlx5_ecpf_vport_exists(esw
->dev
)) {
2256 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_ECPF
);
2257 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2260 if (mlx5_core_is_ecpf_esw_manager(esw
->dev
)) {
2261 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_PF
);
2262 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2265 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_UPLINK
);
2266 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2269 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch
*esw
, u16 vport_num
)
2271 struct mlx5_eswitch_rep
*rep
;
2275 rep
= mlx5_eswitch_get_rep(esw
, vport_num
);
2276 for (rep_type
= 0; rep_type
< NUM_REP_TYPES
; rep_type
++)
2277 if (atomic_cmpxchg(&rep
->rep_data
[rep_type
].state
,
2278 REP_REGISTERED
, REP_LOADED
) == REP_REGISTERED
) {
2279 err
= esw
->offloads
.rep_ops
[rep_type
]->load(esw
->dev
, rep
);
2287 atomic_set(&rep
->rep_data
[rep_type
].state
, REP_REGISTERED
);
2288 for (--rep_type
; rep_type
>= 0; rep_type
--)
2289 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2293 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch
*esw
, u16 vport_num
)
2295 struct mlx5_eswitch_rep
*rep
;
2298 rep
= mlx5_eswitch_get_rep(esw
, vport_num
);
2299 for (rep_type
= NUM_REP_TYPES
- 1; rep_type
>= 0; rep_type
--)
2300 __esw_offloads_unload_rep(esw
, rep
, rep_type
);
2303 int esw_offloads_load_rep(struct mlx5_eswitch
*esw
, u16 vport_num
)
2307 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2310 if (vport_num
!= MLX5_VPORT_UPLINK
) {
2311 err
= mlx5_esw_offloads_devlink_port_register(esw
, vport_num
);
2316 err
= mlx5_esw_offloads_rep_load(esw
, vport_num
);
2322 if (vport_num
!= MLX5_VPORT_UPLINK
)
2323 mlx5_esw_offloads_devlink_port_unregister(esw
, vport_num
);
2327 void esw_offloads_unload_rep(struct mlx5_eswitch
*esw
, u16 vport_num
)
2329 if (esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2332 mlx5_esw_offloads_rep_unload(esw
, vport_num
);
2334 if (vport_num
!= MLX5_VPORT_UPLINK
)
2335 mlx5_esw_offloads_devlink_port_unregister(esw
, vport_num
);
2338 static int esw_set_uplink_slave_ingress_root(struct mlx5_core_dev
*master
,
2339 struct mlx5_core_dev
*slave
)
2341 u32 in
[MLX5_ST_SZ_DW(set_flow_table_root_in
)] = {};
2342 u32 out
[MLX5_ST_SZ_DW(set_flow_table_root_out
)] = {};
2343 struct mlx5_eswitch
*esw
;
2344 struct mlx5_flow_root_namespace
*root
;
2345 struct mlx5_flow_namespace
*ns
;
2346 struct mlx5_vport
*vport
;
2349 MLX5_SET(set_flow_table_root_in
, in
, opcode
,
2350 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT
);
2351 MLX5_SET(set_flow_table_root_in
, in
, table_type
, FS_FT_ESW_INGRESS_ACL
);
2352 MLX5_SET(set_flow_table_root_in
, in
, other_vport
, 1);
2353 MLX5_SET(set_flow_table_root_in
, in
, vport_number
, MLX5_VPORT_UPLINK
);
2356 esw
= master
->priv
.eswitch
;
2357 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_UPLINK
);
2358 MLX5_SET(set_flow_table_root_in
, in
, table_of_other_vport
, 1);
2359 MLX5_SET(set_flow_table_root_in
, in
, table_vport_number
,
2362 ns
= mlx5_get_flow_vport_acl_namespace(master
,
2363 MLX5_FLOW_NAMESPACE_ESW_INGRESS
,
2365 root
= find_root(&ns
->node
);
2366 mutex_lock(&root
->chain_lock
);
2368 MLX5_SET(set_flow_table_root_in
, in
,
2369 table_eswitch_owner_vhca_id_valid
, 1);
2370 MLX5_SET(set_flow_table_root_in
, in
,
2371 table_eswitch_owner_vhca_id
,
2372 MLX5_CAP_GEN(master
, vhca_id
));
2373 MLX5_SET(set_flow_table_root_in
, in
, table_id
,
2376 esw
= slave
->priv
.eswitch
;
2377 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_UPLINK
);
2378 ns
= mlx5_get_flow_vport_acl_namespace(slave
,
2379 MLX5_FLOW_NAMESPACE_ESW_INGRESS
,
2381 root
= find_root(&ns
->node
);
2382 mutex_lock(&root
->chain_lock
);
2383 MLX5_SET(set_flow_table_root_in
, in
, table_id
, root
->root_ft
->id
);
2386 err
= mlx5_cmd_exec(slave
, in
, sizeof(in
), out
, sizeof(out
));
2387 mutex_unlock(&root
->chain_lock
);
2392 static int esw_set_slave_root_fdb(struct mlx5_core_dev
*master
,
2393 struct mlx5_core_dev
*slave
)
2395 u32 in
[MLX5_ST_SZ_DW(set_flow_table_root_in
)] = {};
2396 u32 out
[MLX5_ST_SZ_DW(set_flow_table_root_out
)] = {};
2397 struct mlx5_flow_root_namespace
*root
;
2398 struct mlx5_flow_namespace
*ns
;
2401 MLX5_SET(set_flow_table_root_in
, in
, opcode
,
2402 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT
);
2403 MLX5_SET(set_flow_table_root_in
, in
, table_type
,
2407 ns
= mlx5_get_flow_namespace(master
,
2408 MLX5_FLOW_NAMESPACE_FDB
);
2409 root
= find_root(&ns
->node
);
2410 mutex_lock(&root
->chain_lock
);
2411 MLX5_SET(set_flow_table_root_in
, in
,
2412 table_eswitch_owner_vhca_id_valid
, 1);
2413 MLX5_SET(set_flow_table_root_in
, in
,
2414 table_eswitch_owner_vhca_id
,
2415 MLX5_CAP_GEN(master
, vhca_id
));
2416 MLX5_SET(set_flow_table_root_in
, in
, table_id
,
2419 ns
= mlx5_get_flow_namespace(slave
,
2420 MLX5_FLOW_NAMESPACE_FDB
);
2421 root
= find_root(&ns
->node
);
2422 mutex_lock(&root
->chain_lock
);
2423 MLX5_SET(set_flow_table_root_in
, in
, table_id
,
2427 err
= mlx5_cmd_exec(slave
, in
, sizeof(in
), out
, sizeof(out
));
2428 mutex_unlock(&root
->chain_lock
);
2433 static int __esw_set_master_egress_rule(struct mlx5_core_dev
*master
,
2434 struct mlx5_core_dev
*slave
,
2435 struct mlx5_vport
*vport
,
2436 struct mlx5_flow_table
*acl
)
2438 struct mlx5_flow_handle
*flow_rule
= NULL
;
2439 struct mlx5_flow_destination dest
= {};
2440 struct mlx5_flow_act flow_act
= {};
2441 struct mlx5_flow_spec
*spec
;
2445 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
2449 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
2450 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
,
2452 MLX5_SET(fte_match_set_misc
, misc
, source_port
, MLX5_VPORT_UPLINK
);
2453 MLX5_SET(fte_match_set_misc
, misc
, source_eswitch_owner_vhca_id
,
2454 MLX5_CAP_GEN(slave
, vhca_id
));
2456 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
2457 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
2458 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
,
2459 source_eswitch_owner_vhca_id
);
2461 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
2462 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
2463 dest
.vport
.num
= slave
->priv
.eswitch
->manager_vport
;
2464 dest
.vport
.vhca_id
= MLX5_CAP_GEN(slave
, vhca_id
);
2465 dest
.vport
.flags
|= MLX5_FLOW_DEST_VPORT_VHCA_ID
;
2467 flow_rule
= mlx5_add_flow_rules(acl
, spec
, &flow_act
,
2469 if (IS_ERR(flow_rule
))
2470 err
= PTR_ERR(flow_rule
);
2472 vport
->egress
.offloads
.bounce_rule
= flow_rule
;
2478 static int esw_set_master_egress_rule(struct mlx5_core_dev
*master
,
2479 struct mlx5_core_dev
*slave
)
2481 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
2482 struct mlx5_eswitch
*esw
= master
->priv
.eswitch
;
2483 struct mlx5_flow_table_attr ft_attr
= {
2484 .max_fte
= 1, .prio
= 0, .level
= 0,
2485 .flags
= MLX5_FLOW_TABLE_OTHER_VPORT
,
2487 struct mlx5_flow_namespace
*egress_ns
;
2488 struct mlx5_flow_table
*acl
;
2489 struct mlx5_flow_group
*g
;
2490 struct mlx5_vport
*vport
;
2491 void *match_criteria
;
2495 vport
= mlx5_eswitch_get_vport(esw
, esw
->manager_vport
);
2497 return PTR_ERR(vport
);
2499 egress_ns
= mlx5_get_flow_vport_acl_namespace(master
,
2500 MLX5_FLOW_NAMESPACE_ESW_EGRESS
,
2505 if (vport
->egress
.acl
)
2508 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
2512 acl
= mlx5_create_vport_flow_table(egress_ns
, &ft_attr
, vport
->vport
);
2518 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
,
2520 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
2521 misc_parameters
.source_port
);
2522 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
,
2523 misc_parameters
.source_eswitch_owner_vhca_id
);
2524 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
2525 MLX5_MATCH_MISC_PARAMETERS
);
2527 MLX5_SET(create_flow_group_in
, flow_group_in
,
2528 source_eswitch_owner_vhca_id_valid
, 1);
2529 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
2530 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, 0);
2532 g
= mlx5_create_flow_group(acl
, flow_group_in
);
2538 err
= __esw_set_master_egress_rule(master
, slave
, vport
, acl
);
2542 vport
->egress
.acl
= acl
;
2543 vport
->egress
.offloads
.bounce_grp
= g
;
2545 kvfree(flow_group_in
);
2550 mlx5_destroy_flow_group(g
);
2552 mlx5_destroy_flow_table(acl
);
2554 kvfree(flow_group_in
);
2558 static void esw_unset_master_egress_rule(struct mlx5_core_dev
*dev
)
2560 struct mlx5_vport
*vport
;
2562 vport
= mlx5_eswitch_get_vport(dev
->priv
.eswitch
,
2563 dev
->priv
.eswitch
->manager_vport
);
2565 esw_acl_egress_ofld_cleanup(vport
);
2568 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch
*master_esw
,
2569 struct mlx5_eswitch
*slave_esw
)
2573 err
= esw_set_uplink_slave_ingress_root(master_esw
->dev
,
2578 err
= esw_set_slave_root_fdb(master_esw
->dev
,
2583 err
= esw_set_master_egress_rule(master_esw
->dev
,
2591 esw_set_slave_root_fdb(NULL
, slave_esw
->dev
);
2594 esw_set_uplink_slave_ingress_root(NULL
, slave_esw
->dev
);
2599 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch
*master_esw
,
2600 struct mlx5_eswitch
*slave_esw
)
2602 esw_unset_master_egress_rule(master_esw
->dev
);
2603 esw_set_slave_root_fdb(NULL
, slave_esw
->dev
);
2604 esw_set_uplink_slave_ingress_root(NULL
, slave_esw
->dev
);
2607 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2608 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2610 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch
*esw
)
2612 const struct mlx5_eswitch_rep_ops
*ops
;
2613 struct mlx5_eswitch_rep
*rep
;
2617 mlx5_esw_for_each_rep(esw
, i
, rep
) {
2618 rep_type
= NUM_REP_TYPES
;
2619 while (rep_type
--) {
2620 ops
= esw
->offloads
.rep_ops
[rep_type
];
2621 if (atomic_read(&rep
->rep_data
[rep_type
].state
) == REP_LOADED
&&
2623 ops
->event(esw
, rep
, MLX5_SWITCHDEV_EVENT_UNPAIR
, NULL
);
2628 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch
*esw
)
2630 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2631 mlx5e_tc_clean_fdb_peer_flows(esw
);
2633 mlx5_esw_offloads_rep_event_unpair(esw
);
2634 esw_del_fdb_peer_miss_rules(esw
);
2637 static int mlx5_esw_offloads_pair(struct mlx5_eswitch
*esw
,
2638 struct mlx5_eswitch
*peer_esw
)
2640 const struct mlx5_eswitch_rep_ops
*ops
;
2641 struct mlx5_eswitch_rep
*rep
;
2646 err
= esw_add_fdb_peer_miss_rules(esw
, peer_esw
->dev
);
2650 mlx5_esw_for_each_rep(esw
, i
, rep
) {
2651 for (rep_type
= 0; rep_type
< NUM_REP_TYPES
; rep_type
++) {
2652 ops
= esw
->offloads
.rep_ops
[rep_type
];
2653 if (atomic_read(&rep
->rep_data
[rep_type
].state
) == REP_LOADED
&&
2655 err
= ops
->event(esw
, rep
, MLX5_SWITCHDEV_EVENT_PAIR
, peer_esw
);
2665 mlx5_esw_offloads_unpair(esw
);
2669 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch
*esw
,
2670 struct mlx5_eswitch
*peer_esw
,
2673 struct mlx5_flow_root_namespace
*peer_ns
;
2674 struct mlx5_flow_root_namespace
*ns
;
2677 peer_ns
= peer_esw
->dev
->priv
.steering
->fdb_root_ns
;
2678 ns
= esw
->dev
->priv
.steering
->fdb_root_ns
;
2681 err
= mlx5_flow_namespace_set_peer(ns
, peer_ns
);
2685 err
= mlx5_flow_namespace_set_peer(peer_ns
, ns
);
2687 mlx5_flow_namespace_set_peer(ns
, NULL
);
2691 mlx5_flow_namespace_set_peer(ns
, NULL
);
2692 mlx5_flow_namespace_set_peer(peer_ns
, NULL
);
2698 static int mlx5_esw_offloads_devcom_event(int event
,
2702 struct mlx5_eswitch
*esw
= my_data
;
2703 struct mlx5_devcom
*devcom
= esw
->dev
->priv
.devcom
;
2704 struct mlx5_eswitch
*peer_esw
= event_data
;
2708 case ESW_OFFLOADS_DEVCOM_PAIR
:
2709 if (mlx5_get_next_phys_dev(esw
->dev
) != peer_esw
->dev
)
2712 if (mlx5_eswitch_vport_match_metadata_enabled(esw
) !=
2713 mlx5_eswitch_vport_match_metadata_enabled(peer_esw
))
2716 err
= mlx5_esw_offloads_set_ns_peer(esw
, peer_esw
, true);
2719 err
= mlx5_esw_offloads_pair(esw
, peer_esw
);
2723 err
= mlx5_esw_offloads_pair(peer_esw
, esw
);
2727 mlx5_devcom_set_paired(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
, true);
2730 case ESW_OFFLOADS_DEVCOM_UNPAIR
:
2731 if (!mlx5_devcom_is_paired(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
))
2734 mlx5_devcom_set_paired(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
, false);
2735 mlx5_esw_offloads_unpair(peer_esw
);
2736 mlx5_esw_offloads_unpair(esw
);
2737 mlx5_esw_offloads_set_ns_peer(esw
, peer_esw
, false);
2744 mlx5_esw_offloads_unpair(esw
);
2746 mlx5_esw_offloads_set_ns_peer(esw
, peer_esw
, false);
2748 mlx5_core_err(esw
->dev
, "esw offloads devcom event failure, event %u err %d",
2753 static void esw_offloads_devcom_init(struct mlx5_eswitch
*esw
)
2755 struct mlx5_devcom
*devcom
= esw
->dev
->priv
.devcom
;
2757 INIT_LIST_HEAD(&esw
->offloads
.peer_flows
);
2758 mutex_init(&esw
->offloads
.peer_mutex
);
2760 if (!MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
2763 mlx5_devcom_register_component(devcom
,
2764 MLX5_DEVCOM_ESW_OFFLOADS
,
2765 mlx5_esw_offloads_devcom_event
,
2768 mlx5_devcom_send_event(devcom
,
2769 MLX5_DEVCOM_ESW_OFFLOADS
,
2770 ESW_OFFLOADS_DEVCOM_PAIR
, esw
);
2773 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch
*esw
)
2775 struct mlx5_devcom
*devcom
= esw
->dev
->priv
.devcom
;
2777 if (!MLX5_CAP_ESW(esw
->dev
, merged_eswitch
))
2780 mlx5_devcom_send_event(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
,
2781 ESW_OFFLOADS_DEVCOM_UNPAIR
, esw
);
2783 mlx5_devcom_unregister_component(devcom
, MLX5_DEVCOM_ESW_OFFLOADS
);
2786 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch
*esw
)
2788 if (!MLX5_CAP_ESW(esw
->dev
, esw_uplink_ingress_acl
))
2791 if (!(MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, fdb_to_vport_reg_c_id
) &
2792 MLX5_FDB_TO_VPORT_REG_C_0
))
2795 if (!MLX5_CAP_ESW_FLOWTABLE(esw
->dev
, flow_source
))
2801 u32
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch
*esw
)
2803 u32 vport_end_ida
= (1 << ESW_VPORT_BITS
) - 1;
2804 /* Reserve 0xf for internal port offload */
2805 u32 max_pf_num
= (1 << ESW_PFNUM_BITS
) - 2;
2809 /* Only 4 bits of pf_num */
2810 pf_num
= PCI_FUNC(esw
->dev
->pdev
->devfn
);
2811 if (pf_num
> max_pf_num
)
2814 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
2815 /* Use only non-zero vport_id (1-4095) for all PF's */
2816 id
= ida_alloc_range(&esw
->offloads
.vport_metadata_ida
, 1, vport_end_ida
, GFP_KERNEL
);
2819 id
= (pf_num
<< ESW_VPORT_BITS
) | id
;
2823 void mlx5_esw_match_metadata_free(struct mlx5_eswitch
*esw
, u32 metadata
)
2825 u32 vport_bit_mask
= (1 << ESW_VPORT_BITS
) - 1;
2827 /* Metadata contains only 12 bits of actual ida id */
2828 ida_free(&esw
->offloads
.vport_metadata_ida
, metadata
& vport_bit_mask
);
2831 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch
*esw
,
2832 struct mlx5_vport
*vport
)
2834 vport
->default_metadata
= mlx5_esw_match_metadata_alloc(esw
);
2835 vport
->metadata
= vport
->default_metadata
;
2836 return vport
->metadata
? 0 : -ENOSPC
;
2839 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch
*esw
,
2840 struct mlx5_vport
*vport
)
2842 if (!vport
->default_metadata
)
2845 WARN_ON(vport
->metadata
!= vport
->default_metadata
);
2846 mlx5_esw_match_metadata_free(esw
, vport
->default_metadata
);
2849 static void esw_offloads_metadata_uninit(struct mlx5_eswitch
*esw
)
2851 struct mlx5_vport
*vport
;
2854 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
))
2857 mlx5_esw_for_each_vport(esw
, i
, vport
)
2858 esw_offloads_vport_metadata_cleanup(esw
, vport
);
2861 static int esw_offloads_metadata_init(struct mlx5_eswitch
*esw
)
2863 struct mlx5_vport
*vport
;
2867 if (!mlx5_eswitch_vport_match_metadata_enabled(esw
))
2870 mlx5_esw_for_each_vport(esw
, i
, vport
) {
2871 err
= esw_offloads_vport_metadata_setup(esw
, vport
);
2879 esw_offloads_metadata_uninit(esw
);
2883 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch
*esw
, bool enable
)
2887 down_write(&esw
->mode_lock
);
2888 if (esw
->mode
!= MLX5_ESWITCH_NONE
) {
2892 if (!mlx5_esw_vport_match_metadata_supported(esw
)) {
2897 esw
->flags
|= MLX5_ESWITCH_VPORT_MATCH_METADATA
;
2899 esw
->flags
&= ~MLX5_ESWITCH_VPORT_MATCH_METADATA
;
2901 up_write(&esw
->mode_lock
);
2906 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch
*esw
,
2907 struct mlx5_vport
*vport
)
2911 err
= esw_acl_ingress_ofld_setup(esw
, vport
);
2915 err
= esw_acl_egress_ofld_setup(esw
, vport
);
2922 esw_acl_ingress_ofld_cleanup(esw
, vport
);
2927 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch
*esw
,
2928 struct mlx5_vport
*vport
)
2930 esw_acl_egress_ofld_cleanup(vport
);
2931 esw_acl_ingress_ofld_cleanup(esw
, vport
);
2934 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch
*esw
)
2936 struct mlx5_vport
*vport
;
2938 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_UPLINK
);
2940 return PTR_ERR(vport
);
2942 return esw_vport_create_offloads_acl_tables(esw
, vport
);
2945 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch
*esw
)
2947 struct mlx5_vport
*vport
;
2949 vport
= mlx5_eswitch_get_vport(esw
, MLX5_VPORT_UPLINK
);
2953 esw_vport_destroy_offloads_acl_tables(esw
, vport
);
2956 int mlx5_eswitch_reload_reps(struct mlx5_eswitch
*esw
)
2958 struct mlx5_eswitch_rep
*rep
;
2962 if (!esw
|| esw
->mode
!= MLX5_ESWITCH_OFFLOADS
)
2965 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_UPLINK
);
2966 if (atomic_read(&rep
->rep_data
[REP_ETH
].state
) != REP_LOADED
)
2969 ret
= mlx5_esw_offloads_rep_load(esw
, MLX5_VPORT_UPLINK
);
2973 mlx5_esw_for_each_rep(esw
, i
, rep
) {
2974 if (atomic_read(&rep
->rep_data
[REP_ETH
].state
) == REP_LOADED
)
2975 mlx5_esw_offloads_rep_load(esw
, rep
->vport
);
2981 static int esw_offloads_steering_init(struct mlx5_eswitch
*esw
)
2983 struct mlx5_esw_indir_table
*indir
;
2986 memset(&esw
->fdb_table
.offloads
, 0, sizeof(struct offloads_fdb
));
2987 mutex_init(&esw
->fdb_table
.offloads
.vports
.lock
);
2988 hash_init(esw
->fdb_table
.offloads
.vports
.table
);
2989 atomic64_set(&esw
->user_count
, 0);
2991 indir
= mlx5_esw_indir_table_init();
2992 if (IS_ERR(indir
)) {
2993 err
= PTR_ERR(indir
);
2994 goto create_indir_err
;
2996 esw
->fdb_table
.offloads
.indir
= indir
;
2998 err
= esw_create_uplink_offloads_acl_tables(esw
);
3000 goto create_acl_err
;
3002 err
= esw_create_offloads_table(esw
);
3004 goto create_offloads_err
;
3006 err
= esw_create_restore_table(esw
);
3008 goto create_restore_err
;
3010 err
= esw_create_offloads_fdb_tables(esw
);
3012 goto create_fdb_err
;
3014 err
= esw_create_vport_rx_group(esw
);
3021 esw_destroy_offloads_fdb_tables(esw
);
3023 esw_destroy_restore_table(esw
);
3025 esw_destroy_offloads_table(esw
);
3026 create_offloads_err
:
3027 esw_destroy_uplink_offloads_acl_tables(esw
);
3029 mlx5_esw_indir_table_destroy(esw
->fdb_table
.offloads
.indir
);
3031 mutex_destroy(&esw
->fdb_table
.offloads
.vports
.lock
);
3035 static void esw_offloads_steering_cleanup(struct mlx5_eswitch
*esw
)
3037 esw_destroy_vport_rx_group(esw
);
3038 esw_destroy_offloads_fdb_tables(esw
);
3039 esw_destroy_restore_table(esw
);
3040 esw_destroy_offloads_table(esw
);
3041 esw_destroy_uplink_offloads_acl_tables(esw
);
3042 mlx5_esw_indir_table_destroy(esw
->fdb_table
.offloads
.indir
);
3043 mutex_destroy(&esw
->fdb_table
.offloads
.vports
.lock
);
3047 esw_vfs_changed_event_handler(struct mlx5_eswitch
*esw
, const u32
*out
)
3049 bool host_pf_disabled
;
3052 new_num_vfs
= MLX5_GET(query_esw_functions_out
, out
,
3053 host_params_context
.host_num_of_vfs
);
3054 host_pf_disabled
= MLX5_GET(query_esw_functions_out
, out
,
3055 host_params_context
.host_pf_disabled
);
3057 if (new_num_vfs
== esw
->esw_funcs
.num_vfs
|| host_pf_disabled
)
3060 /* Number of VFs can only change from "0 to x" or "x to 0". */
3061 if (esw
->esw_funcs
.num_vfs
> 0) {
3062 mlx5_eswitch_unload_vf_vports(esw
, esw
->esw_funcs
.num_vfs
);
3066 err
= mlx5_eswitch_load_vf_vports(esw
, new_num_vfs
,
3067 MLX5_VPORT_UC_ADDR_CHANGE
);
3071 esw
->esw_funcs
.num_vfs
= new_num_vfs
;
3074 static void esw_functions_changed_event_handler(struct work_struct
*work
)
3076 struct mlx5_host_work
*host_work
;
3077 struct mlx5_eswitch
*esw
;
3080 host_work
= container_of(work
, struct mlx5_host_work
, work
);
3081 esw
= host_work
->esw
;
3083 out
= mlx5_esw_query_functions(esw
->dev
);
3087 esw_vfs_changed_event_handler(esw
, out
);
3093 int mlx5_esw_funcs_changed_handler(struct notifier_block
*nb
, unsigned long type
, void *data
)
3095 struct mlx5_esw_functions
*esw_funcs
;
3096 struct mlx5_host_work
*host_work
;
3097 struct mlx5_eswitch
*esw
;
3099 host_work
= kzalloc(sizeof(*host_work
), GFP_ATOMIC
);
3103 esw_funcs
= mlx5_nb_cof(nb
, struct mlx5_esw_functions
, nb
);
3104 esw
= container_of(esw_funcs
, struct mlx5_eswitch
, esw_funcs
);
3106 host_work
->esw
= esw
;
3108 INIT_WORK(&host_work
->work
, esw_functions_changed_event_handler
);
3109 queue_work(esw
->work_queue
, &host_work
->work
);
3114 static int mlx5_esw_host_number_init(struct mlx5_eswitch
*esw
)
3116 const u32
*query_host_out
;
3118 if (!mlx5_core_is_ecpf_esw_manager(esw
->dev
))
3121 query_host_out
= mlx5_esw_query_functions(esw
->dev
);
3122 if (IS_ERR(query_host_out
))
3123 return PTR_ERR(query_host_out
);
3125 /* Mark non local controller with non zero controller number. */
3126 esw
->offloads
.host_number
= MLX5_GET(query_esw_functions_out
, query_host_out
,
3127 host_params_context
.host_number
);
3128 kvfree(query_host_out
);
3132 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch
*esw
, u32 controller
)
3134 /* Local controller is always valid */
3135 if (controller
== 0)
3138 if (!mlx5_core_is_ecpf_esw_manager(esw
->dev
))
3141 /* External host number starts with zero in device */
3142 return (controller
== esw
->offloads
.host_number
+ 1);
3145 int esw_offloads_enable(struct mlx5_eswitch
*esw
)
3147 struct mapping_ctx
*reg_c0_obj_pool
;
3148 struct mlx5_vport
*vport
;
3153 mutex_init(&esw
->offloads
.termtbl_mutex
);
3154 mlx5_rdma_enable_roce(esw
->dev
);
3156 err
= mlx5_esw_host_number_init(esw
);
3160 err
= esw_offloads_metadata_init(esw
);
3164 err
= esw_set_passing_vport_metadata(esw
, true);
3166 goto err_vport_metadata
;
3168 mapping_id
= mlx5_query_nic_system_image_guid(esw
->dev
);
3170 reg_c0_obj_pool
= mapping_create_for_id(mapping_id
, MAPPING_TYPE_CHAIN
,
3171 sizeof(struct mlx5_mapped_obj
),
3172 ESW_REG_C0_USER_DATA_METADATA_MASK
,
3175 if (IS_ERR(reg_c0_obj_pool
)) {
3176 err
= PTR_ERR(reg_c0_obj_pool
);
3179 esw
->offloads
.reg_c0_obj_pool
= reg_c0_obj_pool
;
3181 err
= esw_offloads_steering_init(esw
);
3183 goto err_steering_init
;
3185 /* Representor will control the vport link state */
3186 mlx5_esw_for_each_vf_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
)
3187 vport
->info
.link_state
= MLX5_VPORT_ADMIN_STATE_DOWN
;
3189 /* Uplink vport rep must load first. */
3190 err
= esw_offloads_load_rep(esw
, MLX5_VPORT_UPLINK
);
3194 err
= mlx5_eswitch_enable_pf_vf_vports(esw
, MLX5_VPORT_UC_ADDR_CHANGE
);
3198 esw_offloads_devcom_init(esw
);
3203 esw_offloads_unload_rep(esw
, MLX5_VPORT_UPLINK
);
3205 esw_offloads_steering_cleanup(esw
);
3207 mapping_destroy(reg_c0_obj_pool
);
3209 esw_set_passing_vport_metadata(esw
, false);
3211 esw_offloads_metadata_uninit(esw
);
3213 mlx5_rdma_disable_roce(esw
->dev
);
3214 mutex_destroy(&esw
->offloads
.termtbl_mutex
);
3218 static int esw_offloads_stop(struct mlx5_eswitch
*esw
,
3219 struct netlink_ext_ack
*extack
)
3223 mlx5_eswitch_disable_locked(esw
, false);
3224 err
= mlx5_eswitch_enable_locked(esw
, MLX5_ESWITCH_LEGACY
,
3225 MLX5_ESWITCH_IGNORE_NUM_VFS
);
3227 NL_SET_ERR_MSG_MOD(extack
, "Failed setting eswitch to legacy");
3228 err1
= mlx5_eswitch_enable_locked(esw
, MLX5_ESWITCH_OFFLOADS
,
3229 MLX5_ESWITCH_IGNORE_NUM_VFS
);
3231 NL_SET_ERR_MSG_MOD(extack
,
3232 "Failed setting eswitch back to offloads");
3239 void esw_offloads_disable(struct mlx5_eswitch
*esw
)
3241 esw_offloads_devcom_cleanup(esw
);
3242 mlx5_eswitch_disable_pf_vf_vports(esw
);
3243 esw_offloads_unload_rep(esw
, MLX5_VPORT_UPLINK
);
3244 esw_set_passing_vport_metadata(esw
, false);
3245 esw_offloads_steering_cleanup(esw
);
3246 mapping_destroy(esw
->offloads
.reg_c0_obj_pool
);
3247 esw_offloads_metadata_uninit(esw
);
3248 mlx5_rdma_disable_roce(esw
->dev
);
3249 mutex_destroy(&esw
->offloads
.termtbl_mutex
);
3252 static int esw_mode_from_devlink(u16 mode
, u16
*mlx5_mode
)
3255 case DEVLINK_ESWITCH_MODE_LEGACY
:
3256 *mlx5_mode
= MLX5_ESWITCH_LEGACY
;
3258 case DEVLINK_ESWITCH_MODE_SWITCHDEV
:
3259 *mlx5_mode
= MLX5_ESWITCH_OFFLOADS
;
3268 static int esw_mode_to_devlink(u16 mlx5_mode
, u16
*mode
)
3270 switch (mlx5_mode
) {
3271 case MLX5_ESWITCH_LEGACY
:
3272 *mode
= DEVLINK_ESWITCH_MODE_LEGACY
;
3274 case MLX5_ESWITCH_OFFLOADS
:
3275 *mode
= DEVLINK_ESWITCH_MODE_SWITCHDEV
;
3284 static int esw_inline_mode_from_devlink(u8 mode
, u8
*mlx5_mode
)
3287 case DEVLINK_ESWITCH_INLINE_MODE_NONE
:
3288 *mlx5_mode
= MLX5_INLINE_MODE_NONE
;
3290 case DEVLINK_ESWITCH_INLINE_MODE_LINK
:
3291 *mlx5_mode
= MLX5_INLINE_MODE_L2
;
3293 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK
:
3294 *mlx5_mode
= MLX5_INLINE_MODE_IP
;
3296 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT
:
3297 *mlx5_mode
= MLX5_INLINE_MODE_TCP_UDP
;
3306 static int esw_inline_mode_to_devlink(u8 mlx5_mode
, u8
*mode
)
3308 switch (mlx5_mode
) {
3309 case MLX5_INLINE_MODE_NONE
:
3310 *mode
= DEVLINK_ESWITCH_INLINE_MODE_NONE
;
3312 case MLX5_INLINE_MODE_L2
:
3313 *mode
= DEVLINK_ESWITCH_INLINE_MODE_LINK
;
3315 case MLX5_INLINE_MODE_IP
:
3316 *mode
= DEVLINK_ESWITCH_INLINE_MODE_NETWORK
;
3318 case MLX5_INLINE_MODE_TCP_UDP
:
3319 *mode
= DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT
;
3328 static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch
*esw
)
3330 /* devlink commands in NONE eswitch mode are currently supported only
3333 return (esw
->mode
== MLX5_ESWITCH_NONE
&&
3334 !mlx5_core_is_ecpf_esw_manager(esw
->dev
)) ? -EOPNOTSUPP
: 0;
3337 int mlx5_devlink_eswitch_mode_set(struct devlink
*devlink
, u16 mode
,
3338 struct netlink_ext_ack
*extack
)
3340 u16 cur_mlx5_mode
, mlx5_mode
= 0;
3341 struct mlx5_eswitch
*esw
;
3344 esw
= mlx5_devlink_eswitch_get(devlink
);
3346 return PTR_ERR(esw
);
3348 if (esw_mode_from_devlink(mode
, &mlx5_mode
))
3351 mlx5_lag_disable_change(esw
->dev
);
3352 err
= mlx5_esw_try_lock(esw
);
3354 NL_SET_ERR_MSG_MOD(extack
, "Can't change mode, E-Switch is busy");
3357 cur_mlx5_mode
= err
;
3360 if (cur_mlx5_mode
== mlx5_mode
)
3363 if (mode
== DEVLINK_ESWITCH_MODE_SWITCHDEV
) {
3364 if (mlx5_devlink_trap_get_num_active(esw
->dev
)) {
3365 NL_SET_ERR_MSG_MOD(extack
,
3366 "Can't change mode while devlink traps are active");
3370 err
= esw_offloads_start(esw
, extack
);
3371 } else if (mode
== DEVLINK_ESWITCH_MODE_LEGACY
) {
3372 err
= esw_offloads_stop(esw
, extack
);
3378 mlx5_esw_unlock(esw
);
3380 mlx5_lag_enable_change(esw
->dev
);
3384 int mlx5_devlink_eswitch_mode_get(struct devlink
*devlink
, u16
*mode
)
3386 struct mlx5_eswitch
*esw
;
3389 esw
= mlx5_devlink_eswitch_get(devlink
);
3391 return PTR_ERR(esw
);
3393 down_write(&esw
->mode_lock
);
3394 err
= eswitch_devlink_esw_mode_check(esw
);
3398 err
= esw_mode_to_devlink(esw
->mode
, mode
);
3400 up_write(&esw
->mode_lock
);
3404 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch
*esw
, u8 mlx5_mode
,
3405 struct netlink_ext_ack
*extack
)
3407 struct mlx5_core_dev
*dev
= esw
->dev
;
3408 struct mlx5_vport
*vport
;
3409 u16 err_vport_num
= 0;
3413 mlx5_esw_for_each_host_func_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
) {
3414 err
= mlx5_modify_nic_vport_min_inline(dev
, vport
->vport
, mlx5_mode
);
3416 err_vport_num
= vport
->vport
;
3417 NL_SET_ERR_MSG_MOD(extack
,
3418 "Failed to set min inline on vport");
3419 goto revert_inline_mode
;
3425 mlx5_esw_for_each_host_func_vport(esw
, i
, vport
, esw
->esw_funcs
.num_vfs
) {
3426 if (vport
->vport
== err_vport_num
)
3428 mlx5_modify_nic_vport_min_inline(dev
,
3430 esw
->offloads
.inline_mode
);
3435 int mlx5_devlink_eswitch_inline_mode_set(struct devlink
*devlink
, u8 mode
,
3436 struct netlink_ext_ack
*extack
)
3438 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
3439 struct mlx5_eswitch
*esw
;
3443 esw
= mlx5_devlink_eswitch_get(devlink
);
3445 return PTR_ERR(esw
);
3447 down_write(&esw
->mode_lock
);
3448 err
= eswitch_devlink_esw_mode_check(esw
);
3452 switch (MLX5_CAP_ETH(dev
, wqe_inline_mode
)) {
3453 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED
:
3454 if (mode
== DEVLINK_ESWITCH_INLINE_MODE_NONE
) {
3460 case MLX5_CAP_INLINE_MODE_L2
:
3461 NL_SET_ERR_MSG_MOD(extack
, "Inline mode can't be set");
3464 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
:
3468 if (atomic64_read(&esw
->offloads
.num_flows
) > 0) {
3469 NL_SET_ERR_MSG_MOD(extack
,
3470 "Can't set inline mode when flows are configured");
3475 err
= esw_inline_mode_from_devlink(mode
, &mlx5_mode
);
3479 err
= mlx5_esw_vports_inline_set(esw
, mlx5_mode
, extack
);
3483 esw
->offloads
.inline_mode
= mlx5_mode
;
3484 up_write(&esw
->mode_lock
);
3488 up_write(&esw
->mode_lock
);
3492 int mlx5_devlink_eswitch_inline_mode_get(struct devlink
*devlink
, u8
*mode
)
3494 struct mlx5_eswitch
*esw
;
3497 esw
= mlx5_devlink_eswitch_get(devlink
);
3499 return PTR_ERR(esw
);
3501 down_write(&esw
->mode_lock
);
3502 err
= eswitch_devlink_esw_mode_check(esw
);
3506 err
= esw_inline_mode_to_devlink(esw
->offloads
.inline_mode
, mode
);
3508 up_write(&esw
->mode_lock
);
3512 int mlx5_devlink_eswitch_encap_mode_set(struct devlink
*devlink
,
3513 enum devlink_eswitch_encap_mode encap
,
3514 struct netlink_ext_ack
*extack
)
3516 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
3517 struct mlx5_eswitch
*esw
;
3520 esw
= mlx5_devlink_eswitch_get(devlink
);
3522 return PTR_ERR(esw
);
3524 down_write(&esw
->mode_lock
);
3525 err
= eswitch_devlink_esw_mode_check(esw
);
3529 if (encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
&&
3530 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, reformat
) ||
3531 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, decap
))) {
3536 if (encap
&& encap
!= DEVLINK_ESWITCH_ENCAP_MODE_BASIC
) {
3541 if (esw
->mode
== MLX5_ESWITCH_LEGACY
) {
3542 esw
->offloads
.encap
= encap
;
3546 if (esw
->offloads
.encap
== encap
)
3549 if (atomic64_read(&esw
->offloads
.num_flows
) > 0) {
3550 NL_SET_ERR_MSG_MOD(extack
,
3551 "Can't set encapsulation when flows are configured");
3556 esw_destroy_offloads_fdb_tables(esw
);
3558 esw
->offloads
.encap
= encap
;
3560 err
= esw_create_offloads_fdb_tables(esw
);
3563 NL_SET_ERR_MSG_MOD(extack
,
3564 "Failed re-creating fast FDB table");
3565 esw
->offloads
.encap
= !encap
;
3566 (void)esw_create_offloads_fdb_tables(esw
);
3570 up_write(&esw
->mode_lock
);
3574 int mlx5_devlink_eswitch_encap_mode_get(struct devlink
*devlink
,
3575 enum devlink_eswitch_encap_mode
*encap
)
3577 struct mlx5_eswitch
*esw
;
3580 esw
= mlx5_devlink_eswitch_get(devlink
);
3582 return PTR_ERR(esw
);
3585 down_write(&esw
->mode_lock
);
3586 err
= eswitch_devlink_esw_mode_check(esw
);
3590 *encap
= esw
->offloads
.encap
;
3592 up_write(&esw
->mode_lock
);
3597 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch
*esw
, u16 vport_num
)
3599 /* Currently, only ECPF based device has representor for host PF. */
3600 if (vport_num
== MLX5_VPORT_PF
&&
3601 !mlx5_core_is_ecpf_esw_manager(esw
->dev
))
3604 if (vport_num
== MLX5_VPORT_ECPF
&&
3605 !mlx5_ecpf_vport_exists(esw
->dev
))
3611 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch
*esw
,
3612 const struct mlx5_eswitch_rep_ops
*ops
,
3615 struct mlx5_eswitch_rep_data
*rep_data
;
3616 struct mlx5_eswitch_rep
*rep
;
3619 esw
->offloads
.rep_ops
[rep_type
] = ops
;
3620 mlx5_esw_for_each_rep(esw
, i
, rep
) {
3621 if (likely(mlx5_eswitch_vport_has_rep(esw
, rep
->vport
))) {
3623 rep_data
= &rep
->rep_data
[rep_type
];
3624 atomic_set(&rep_data
->state
, REP_REGISTERED
);
3628 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps
);
3630 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch
*esw
, u8 rep_type
)
3632 struct mlx5_eswitch_rep
*rep
;
3635 if (esw
->mode
== MLX5_ESWITCH_OFFLOADS
)
3636 __unload_reps_all_vport(esw
, rep_type
);
3638 mlx5_esw_for_each_rep(esw
, i
, rep
)
3639 atomic_set(&rep
->rep_data
[rep_type
].state
, REP_UNREGISTERED
);
3641 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps
);
3643 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch
*esw
, u8 rep_type
)
3645 struct mlx5_eswitch_rep
*rep
;
3647 rep
= mlx5_eswitch_get_rep(esw
, MLX5_VPORT_UPLINK
);
3648 return rep
->rep_data
[rep_type
].priv
;
3651 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch
*esw
,
3655 struct mlx5_eswitch_rep
*rep
;
3657 rep
= mlx5_eswitch_get_rep(esw
, vport
);
3659 if (atomic_read(&rep
->rep_data
[rep_type
].state
) == REP_LOADED
&&
3660 esw
->offloads
.rep_ops
[rep_type
]->get_proto_dev
)
3661 return esw
->offloads
.rep_ops
[rep_type
]->get_proto_dev(rep
);
3664 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev
);
3666 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch
*esw
, u8 rep_type
)
3668 return mlx5_eswitch_get_proto_dev(esw
, MLX5_VPORT_UPLINK
, rep_type
);
3670 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev
);
3672 struct mlx5_eswitch_rep
*mlx5_eswitch_vport_rep(struct mlx5_eswitch
*esw
,
3675 return mlx5_eswitch_get_rep(esw
, vport
);
3677 EXPORT_SYMBOL(mlx5_eswitch_vport_rep
);
3679 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch
*esw
)
3681 return !!(esw
->flags
& MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED
);
3683 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled
);
3685 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch
*esw
)
3687 return !!(esw
->flags
& MLX5_ESWITCH_VPORT_MATCH_METADATA
);
3689 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled
);
3691 u32
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch
*esw
,
3694 struct mlx5_vport
*vport
= mlx5_eswitch_get_vport(esw
, vport_num
);
3696 if (WARN_ON_ONCE(IS_ERR(vport
)))
3699 return vport
->metadata
<< (32 - ESW_SOURCE_PORT_METADATA_BITS
);
3701 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match
);
3703 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch
*esw
, struct devlink_port
*dl_port
,
3704 u16 vport_num
, u32 controller
, u32 sfnum
)
3708 err
= mlx5_esw_vport_enable(esw
, vport_num
, MLX5_VPORT_UC_ADDR_CHANGE
);
3712 err
= mlx5_esw_devlink_sf_port_register(esw
, dl_port
, vport_num
, controller
, sfnum
);
3716 err
= mlx5_esw_offloads_rep_load(esw
, vport_num
);
3722 mlx5_esw_devlink_sf_port_unregister(esw
, vport_num
);
3724 mlx5_esw_vport_disable(esw
, vport_num
);
3728 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch
*esw
, u16 vport_num
)
3730 mlx5_esw_offloads_rep_unload(esw
, vport_num
);
3731 mlx5_esw_devlink_sf_port_unregister(esw
, vport_num
);
3732 mlx5_esw_vport_disable(esw
, vport_num
);
3735 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch
*esw
, u16 vport_num
, u16
*vhca_id
)
3737 int query_out_sz
= MLX5_ST_SZ_BYTES(query_hca_cap_out
);
3743 if (mlx5_esw_is_manager_vport(esw
, vport_num
) ||
3744 !MLX5_CAP_GEN(esw
->dev
, vhca_resource_manager
))
3747 query_ctx
= kzalloc(query_out_sz
, GFP_KERNEL
);
3751 err
= mlx5_vport_get_other_func_cap(esw
->dev
, vport_num
, query_ctx
);
3755 hca_caps
= MLX5_ADDR_OF(query_hca_cap_out
, query_ctx
, capability
);
3756 *vhca_id
= MLX5_GET(cmd_hca_cap
, hca_caps
, vhca_id
);
3763 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch
*esw
, u16 vport_num
)
3765 u16
*old_entry
, *vhca_map_entry
, vhca_id
;
3768 err
= mlx5_esw_query_vport_vhca_id(esw
, vport_num
, &vhca_id
);
3770 esw_warn(esw
->dev
, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
3775 vhca_map_entry
= kmalloc(sizeof(*vhca_map_entry
), GFP_KERNEL
);
3776 if (!vhca_map_entry
)
3779 *vhca_map_entry
= vport_num
;
3780 old_entry
= xa_store(&esw
->offloads
.vhca_map
, vhca_id
, vhca_map_entry
, GFP_KERNEL
);
3781 if (xa_is_err(old_entry
)) {
3782 kfree(vhca_map_entry
);
3783 return xa_err(old_entry
);
3789 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch
*esw
, u16 vport_num
)
3791 u16
*vhca_map_entry
, vhca_id
;
3794 err
= mlx5_esw_query_vport_vhca_id(esw
, vport_num
, &vhca_id
);
3796 esw_warn(esw
->dev
, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
3799 vhca_map_entry
= xa_erase(&esw
->offloads
.vhca_map
, vhca_id
);
3800 kfree(vhca_map_entry
);
3803 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch
*esw
, u16 vhca_id
, u16
*vport_num
)
3805 u16
*res
= xa_load(&esw
->offloads
.vhca_map
, vhca_id
);
3814 u32
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch
*esw
,
3817 struct mlx5_vport
*vport
= mlx5_eswitch_get_vport(esw
, vport_num
);
3819 if (WARN_ON_ONCE(IS_ERR(vport
)))
3822 return vport
->metadata
;
3824 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set
);