2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/etherdevice.h>
34 #include <linux/mlx5/driver.h>
35 #include <linux/mlx5/mlx5_ifc.h>
36 #include <linux/mlx5/vport.h>
37 #include <linux/mlx5/fs.h>
38 #include "mlx5_core.h"
46 struct mlx5_flow_handle
*
47 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch
*esw
,
48 struct mlx5_flow_spec
*spec
,
49 struct mlx5_esw_flow_attr
*attr
)
51 struct mlx5_flow_destination dest
[2] = {};
52 struct mlx5_flow_act flow_act
= {0};
53 struct mlx5_fc
*counter
= NULL
;
54 struct mlx5_flow_handle
*rule
;
58 if (esw
->mode
!= SRIOV_OFFLOADS
)
59 return ERR_PTR(-EOPNOTSUPP
);
61 /* per flow vlan pop/push is emulated, don't set that into the firmware */
62 flow_act
.action
= attr
->action
& ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
| MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
);
64 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) {
65 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
66 dest
[i
].vport_num
= attr
->out_rep
->vport
;
69 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_COUNT
) {
70 counter
= mlx5_fc_create(esw
->dev
, true);
71 if (IS_ERR(counter
)) {
72 rule
= ERR_CAST(counter
);
73 goto err_counter_alloc
;
75 dest
[i
].type
= MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
76 dest
[i
].counter
= counter
;
80 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
81 MLX5_SET(fte_match_set_misc
, misc
, source_port
, attr
->in_rep
->vport
);
83 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
84 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
86 spec
->match_criteria_enable
= MLX5_MATCH_OUTER_HEADERS
|
87 MLX5_MATCH_MISC_PARAMETERS
;
88 if (flow_act
.action
& MLX5_FLOW_CONTEXT_ACTION_DECAP
)
89 spec
->match_criteria_enable
|= MLX5_MATCH_INNER_HEADERS
;
91 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
)
92 flow_act
.modify_id
= attr
->mod_hdr_id
;
94 if (attr
->action
& MLX5_FLOW_CONTEXT_ACTION_ENCAP
)
95 flow_act
.encap_id
= attr
->encap_id
;
97 rule
= mlx5_add_flow_rules((struct mlx5_flow_table
*)esw
->fdb_table
.fdb
,
98 spec
, &flow_act
, dest
, i
);
102 esw
->offloads
.num_flows
++;
107 mlx5_fc_destroy(esw
->dev
, counter
);
113 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch
*esw
,
114 struct mlx5_flow_handle
*rule
,
115 struct mlx5_esw_flow_attr
*attr
)
117 struct mlx5_fc
*counter
= NULL
;
119 counter
= mlx5_flow_rule_counter(rule
);
120 mlx5_del_flow_rules(rule
);
121 mlx5_fc_destroy(esw
->dev
, counter
);
122 esw
->offloads
.num_flows
--;
125 static int esw_set_global_vlan_pop(struct mlx5_eswitch
*esw
, u8 val
)
127 struct mlx5_eswitch_rep
*rep
;
128 int vf_vport
, err
= 0;
130 esw_debug(esw
->dev
, "%s applying global %s policy\n", __func__
, val
? "pop" : "none");
131 for (vf_vport
= 1; vf_vport
< esw
->enabled_vports
; vf_vport
++) {
132 rep
= &esw
->offloads
.vport_reps
[vf_vport
];
136 err
= __mlx5_eswitch_set_vport_vlan(esw
, rep
->vport
, 0, 0, val
);
145 static struct mlx5_eswitch_rep
*
146 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr
*attr
, bool push
, bool pop
)
148 struct mlx5_eswitch_rep
*in_rep
, *out_rep
, *vport
= NULL
;
150 in_rep
= attr
->in_rep
;
151 out_rep
= attr
->out_rep
;
163 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr
*attr
,
164 bool push
, bool pop
, bool fwd
)
166 struct mlx5_eswitch_rep
*in_rep
, *out_rep
;
168 if ((push
|| pop
) && !fwd
)
171 in_rep
= attr
->in_rep
;
172 out_rep
= attr
->out_rep
;
174 if (push
&& in_rep
->vport
== FDB_UPLINK_VPORT
)
177 if (pop
&& out_rep
->vport
== FDB_UPLINK_VPORT
)
180 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
181 if (!push
&& !pop
&& fwd
)
182 if (in_rep
->vlan
&& out_rep
->vport
== FDB_UPLINK_VPORT
)
185 /* protects against (1) setting rules with different vlans to push and
186 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
188 if (push
&& in_rep
->vlan_refcount
&& (in_rep
->vlan
!= attr
->vlan
))
197 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch
*esw
,
198 struct mlx5_esw_flow_attr
*attr
)
200 struct offloads_fdb
*offloads
= &esw
->fdb_table
.offloads
;
201 struct mlx5_eswitch_rep
*vport
= NULL
;
205 push
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
);
206 pop
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
);
207 fwd
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
);
209 err
= esw_add_vlan_action_check(attr
, push
, pop
, fwd
);
213 attr
->vlan_handled
= false;
215 vport
= esw_vlan_action_get_vport(attr
, push
, pop
);
217 if (!push
&& !pop
&& fwd
) {
218 /* tracks VF --> wire rules without vlan push action */
219 if (attr
->out_rep
->vport
== FDB_UPLINK_VPORT
) {
220 vport
->vlan_refcount
++;
221 attr
->vlan_handled
= true;
230 if (!(offloads
->vlan_push_pop_refcount
)) {
231 /* it's the 1st vlan rule, apply global vlan pop policy */
232 err
= esw_set_global_vlan_pop(esw
, SET_VLAN_STRIP
);
236 offloads
->vlan_push_pop_refcount
++;
239 if (vport
->vlan_refcount
)
242 err
= __mlx5_eswitch_set_vport_vlan(esw
, vport
->vport
, attr
->vlan
, 0,
243 SET_VLAN_INSERT
| SET_VLAN_STRIP
);
246 vport
->vlan
= attr
->vlan
;
248 vport
->vlan_refcount
++;
252 attr
->vlan_handled
= true;
256 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch
*esw
,
257 struct mlx5_esw_flow_attr
*attr
)
259 struct offloads_fdb
*offloads
= &esw
->fdb_table
.offloads
;
260 struct mlx5_eswitch_rep
*vport
= NULL
;
264 if (!attr
->vlan_handled
)
267 push
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
);
268 pop
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
);
269 fwd
= !!(attr
->action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
);
271 vport
= esw_vlan_action_get_vport(attr
, push
, pop
);
273 if (!push
&& !pop
&& fwd
) {
274 /* tracks VF --> wire rules without vlan push action */
275 if (attr
->out_rep
->vport
== FDB_UPLINK_VPORT
)
276 vport
->vlan_refcount
--;
282 vport
->vlan_refcount
--;
283 if (vport
->vlan_refcount
)
284 goto skip_unset_push
;
287 err
= __mlx5_eswitch_set_vport_vlan(esw
, vport
->vport
,
288 0, 0, SET_VLAN_STRIP
);
294 offloads
->vlan_push_pop_refcount
--;
295 if (offloads
->vlan_push_pop_refcount
)
298 /* no more vlan rules, stop global vlan pop policy */
299 err
= esw_set_global_vlan_pop(esw
, 0);
305 static struct mlx5_flow_handle
*
306 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch
*esw
, int vport
, u32 sqn
)
308 struct mlx5_flow_act flow_act
= {0};
309 struct mlx5_flow_destination dest
;
310 struct mlx5_flow_handle
*flow_rule
;
311 struct mlx5_flow_spec
*spec
;
314 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
316 flow_rule
= ERR_PTR(-ENOMEM
);
320 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
321 MLX5_SET(fte_match_set_misc
, misc
, source_sqn
, sqn
);
322 MLX5_SET(fte_match_set_misc
, misc
, source_port
, 0x0); /* source vport is 0 */
324 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
325 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_sqn
);
326 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
328 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
329 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
330 dest
.vport_num
= vport
;
331 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
333 flow_rule
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.fdb
, spec
,
334 &flow_act
, &dest
, 1);
335 if (IS_ERR(flow_rule
))
336 esw_warn(esw
->dev
, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule
));
342 void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch
*esw
,
343 struct mlx5_eswitch_rep
*rep
)
345 struct mlx5_esw_sq
*esw_sq
, *tmp
;
347 if (esw
->mode
!= SRIOV_OFFLOADS
)
350 list_for_each_entry_safe(esw_sq
, tmp
, &rep
->vport_sqs_list
, list
) {
351 mlx5_del_flow_rules(esw_sq
->send_to_vport_rule
);
352 list_del(&esw_sq
->list
);
357 int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch
*esw
,
358 struct mlx5_eswitch_rep
*rep
,
359 u16
*sqns_array
, int sqns_num
)
361 struct mlx5_flow_handle
*flow_rule
;
362 struct mlx5_esw_sq
*esw_sq
;
366 if (esw
->mode
!= SRIOV_OFFLOADS
)
369 for (i
= 0; i
< sqns_num
; i
++) {
370 esw_sq
= kzalloc(sizeof(*esw_sq
), GFP_KERNEL
);
376 /* Add re-inject rule to the PF/representor sqs */
377 flow_rule
= mlx5_eswitch_add_send_to_vport_rule(esw
,
380 if (IS_ERR(flow_rule
)) {
381 err
= PTR_ERR(flow_rule
);
385 esw_sq
->send_to_vport_rule
= flow_rule
;
386 list_add(&esw_sq
->list
, &rep
->vport_sqs_list
);
391 mlx5_eswitch_sqs2vport_stop(esw
, rep
);
395 static int esw_add_fdb_miss_rule(struct mlx5_eswitch
*esw
)
397 struct mlx5_flow_act flow_act
= {0};
398 struct mlx5_flow_destination dest
;
399 struct mlx5_flow_handle
*flow_rule
= NULL
;
400 struct mlx5_flow_spec
*spec
;
403 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
409 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_VPORT
;
411 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
413 flow_rule
= mlx5_add_flow_rules(esw
->fdb_table
.offloads
.fdb
, spec
,
414 &flow_act
, &dest
, 1);
415 if (IS_ERR(flow_rule
)) {
416 err
= PTR_ERR(flow_rule
);
417 esw_warn(esw
->dev
, "FDB: Failed to add miss flow rule err %d\n", err
);
421 esw
->fdb_table
.offloads
.miss_rule
= flow_rule
;
427 #define ESW_OFFLOADS_NUM_GROUPS 4
429 static int esw_create_offloads_fast_fdb_table(struct mlx5_eswitch
*esw
)
431 struct mlx5_core_dev
*dev
= esw
->dev
;
432 struct mlx5_flow_namespace
*root_ns
;
433 struct mlx5_flow_table
*fdb
= NULL
;
434 int esw_size
, err
= 0;
437 root_ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_FDB
);
439 esw_warn(dev
, "Failed to get FDB flow namespace\n");
444 esw_debug(dev
, "Create offloads FDB table, min (max esw size(2^%d), max counters(%d)*groups(%d))\n",
445 MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, log_max_ft_size
),
446 MLX5_CAP_GEN(dev
, max_flow_counter
), ESW_OFFLOADS_NUM_GROUPS
);
448 esw_size
= min_t(int, MLX5_CAP_GEN(dev
, max_flow_counter
) * ESW_OFFLOADS_NUM_GROUPS
,
449 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, log_max_ft_size
));
451 if (esw
->offloads
.encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
)
452 flags
|= MLX5_FLOW_TABLE_TUNNEL_EN
;
454 fdb
= mlx5_create_auto_grouped_flow_table(root_ns
, FDB_FAST_PATH
,
456 ESW_OFFLOADS_NUM_GROUPS
, 0,
460 esw_warn(dev
, "Failed to create Fast path FDB Table err %d\n", err
);
463 esw
->fdb_table
.fdb
= fdb
;
469 static void esw_destroy_offloads_fast_fdb_table(struct mlx5_eswitch
*esw
)
471 mlx5_destroy_flow_table(esw
->fdb_table
.fdb
);
474 #define MAX_PF_SQ 256
476 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch
*esw
, int nvports
)
478 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
479 struct mlx5_flow_table_attr ft_attr
= {};
480 struct mlx5_core_dev
*dev
= esw
->dev
;
481 struct mlx5_flow_namespace
*root_ns
;
482 struct mlx5_flow_table
*fdb
= NULL
;
483 int table_size
, ix
, err
= 0;
484 struct mlx5_flow_group
*g
;
485 void *match_criteria
;
488 esw_debug(esw
->dev
, "Create offloads FDB Tables\n");
489 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
493 root_ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_FDB
);
495 esw_warn(dev
, "Failed to get FDB flow namespace\n");
500 err
= esw_create_offloads_fast_fdb_table(esw
);
504 table_size
= nvports
+ MAX_PF_SQ
+ 1;
506 ft_attr
.max_fte
= table_size
;
507 ft_attr
.prio
= FDB_SLOW_PATH
;
509 fdb
= mlx5_create_flow_table(root_ns
, &ft_attr
);
512 esw_warn(dev
, "Failed to create slow path FDB Table err %d\n", err
);
515 esw
->fdb_table
.offloads
.fdb
= fdb
;
517 /* create send-to-vport group */
518 memset(flow_group_in
, 0, inlen
);
519 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
520 MLX5_MATCH_MISC_PARAMETERS
);
522 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
524 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_sqn
);
525 MLX5_SET_TO_ONES(fte_match_param
, match_criteria
, misc_parameters
.source_port
);
527 ix
= nvports
+ MAX_PF_SQ
;
528 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
529 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, ix
- 1);
531 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
534 esw_warn(dev
, "Failed to create send-to-vport flow group err(%d)\n", err
);
537 esw
->fdb_table
.offloads
.send_to_vport_grp
= g
;
539 /* create miss group */
540 memset(flow_group_in
, 0, inlen
);
541 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
, 0);
543 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, ix
);
544 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, ix
+ 1);
546 g
= mlx5_create_flow_group(fdb
, flow_group_in
);
549 esw_warn(dev
, "Failed to create miss flow group err(%d)\n", err
);
552 esw
->fdb_table
.offloads
.miss_grp
= g
;
554 err
= esw_add_fdb_miss_rule(esw
);
561 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
563 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
565 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.fdb
);
567 mlx5_destroy_flow_table(esw
->fdb_table
.fdb
);
570 kvfree(flow_group_in
);
574 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch
*esw
)
576 if (!esw
->fdb_table
.fdb
)
579 esw_debug(esw
->dev
, "Destroy offloads FDB Tables\n");
580 mlx5_del_flow_rules(esw
->fdb_table
.offloads
.miss_rule
);
581 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.send_to_vport_grp
);
582 mlx5_destroy_flow_group(esw
->fdb_table
.offloads
.miss_grp
);
584 mlx5_destroy_flow_table(esw
->fdb_table
.offloads
.fdb
);
585 esw_destroy_offloads_fast_fdb_table(esw
);
588 static int esw_create_offloads_table(struct mlx5_eswitch
*esw
)
590 struct mlx5_flow_table_attr ft_attr
= {};
591 struct mlx5_core_dev
*dev
= esw
->dev
;
592 struct mlx5_flow_table
*ft_offloads
;
593 struct mlx5_flow_namespace
*ns
;
596 ns
= mlx5_get_flow_namespace(dev
, MLX5_FLOW_NAMESPACE_OFFLOADS
);
598 esw_warn(esw
->dev
, "Failed to get offloads flow namespace\n");
602 ft_attr
.max_fte
= dev
->priv
.sriov
.num_vfs
+ 2;
604 ft_offloads
= mlx5_create_flow_table(ns
, &ft_attr
);
605 if (IS_ERR(ft_offloads
)) {
606 err
= PTR_ERR(ft_offloads
);
607 esw_warn(esw
->dev
, "Failed to create offloads table, err %d\n", err
);
611 esw
->offloads
.ft_offloads
= ft_offloads
;
615 static void esw_destroy_offloads_table(struct mlx5_eswitch
*esw
)
617 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
619 mlx5_destroy_flow_table(offloads
->ft_offloads
);
622 static int esw_create_vport_rx_group(struct mlx5_eswitch
*esw
)
624 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
625 struct mlx5_flow_group
*g
;
626 struct mlx5_priv
*priv
= &esw
->dev
->priv
;
628 void *match_criteria
, *misc
;
630 int nvports
= priv
->sriov
.num_vfs
+ 2;
632 flow_group_in
= kvzalloc(inlen
, GFP_KERNEL
);
636 /* create vport rx group */
637 memset(flow_group_in
, 0, inlen
);
638 MLX5_SET(create_flow_group_in
, flow_group_in
, match_criteria_enable
,
639 MLX5_MATCH_MISC_PARAMETERS
);
641 match_criteria
= MLX5_ADDR_OF(create_flow_group_in
, flow_group_in
, match_criteria
);
642 misc
= MLX5_ADDR_OF(fte_match_param
, match_criteria
, misc_parameters
);
643 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
645 MLX5_SET(create_flow_group_in
, flow_group_in
, start_flow_index
, 0);
646 MLX5_SET(create_flow_group_in
, flow_group_in
, end_flow_index
, nvports
- 1);
648 g
= mlx5_create_flow_group(esw
->offloads
.ft_offloads
, flow_group_in
);
652 mlx5_core_warn(esw
->dev
, "Failed to create vport rx group err %d\n", err
);
656 esw
->offloads
.vport_rx_group
= g
;
658 kfree(flow_group_in
);
662 static void esw_destroy_vport_rx_group(struct mlx5_eswitch
*esw
)
664 mlx5_destroy_flow_group(esw
->offloads
.vport_rx_group
);
667 struct mlx5_flow_handle
*
668 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch
*esw
, int vport
, u32 tirn
)
670 struct mlx5_flow_act flow_act
= {0};
671 struct mlx5_flow_destination dest
;
672 struct mlx5_flow_handle
*flow_rule
;
673 struct mlx5_flow_spec
*spec
;
676 spec
= kvzalloc(sizeof(*spec
), GFP_KERNEL
);
678 flow_rule
= ERR_PTR(-ENOMEM
);
682 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_value
, misc_parameters
);
683 MLX5_SET(fte_match_set_misc
, misc
, source_port
, vport
);
685 misc
= MLX5_ADDR_OF(fte_match_param
, spec
->match_criteria
, misc_parameters
);
686 MLX5_SET_TO_ONES(fte_match_set_misc
, misc
, source_port
);
688 spec
->match_criteria_enable
= MLX5_MATCH_MISC_PARAMETERS
;
689 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_TIR
;
692 flow_act
.action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
693 flow_rule
= mlx5_add_flow_rules(esw
->offloads
.ft_offloads
, spec
,
694 &flow_act
, &dest
, 1);
695 if (IS_ERR(flow_rule
)) {
696 esw_warn(esw
->dev
, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule
));
705 static int esw_offloads_start(struct mlx5_eswitch
*esw
)
707 int err
, err1
, num_vfs
= esw
->dev
->priv
.sriov
.num_vfs
;
709 if (esw
->mode
!= SRIOV_LEGACY
) {
710 esw_warn(esw
->dev
, "Can't set offloads mode, SRIOV legacy not enabled\n");
714 mlx5_eswitch_disable_sriov(esw
);
715 err
= mlx5_eswitch_enable_sriov(esw
, num_vfs
, SRIOV_OFFLOADS
);
717 esw_warn(esw
->dev
, "Failed setting eswitch to offloads, err %d\n", err
);
718 err1
= mlx5_eswitch_enable_sriov(esw
, num_vfs
, SRIOV_LEGACY
);
720 esw_warn(esw
->dev
, "Failed setting eswitch back to legacy, err %d\n", err1
);
722 if (esw
->offloads
.inline_mode
== MLX5_INLINE_MODE_NONE
) {
723 if (mlx5_eswitch_inline_mode_get(esw
,
725 &esw
->offloads
.inline_mode
)) {
726 esw
->offloads
.inline_mode
= MLX5_INLINE_MODE_L2
;
727 esw_warn(esw
->dev
, "Inline mode is different between vports\n");
733 int esw_offloads_init(struct mlx5_eswitch
*esw
, int nvports
)
735 struct mlx5_eswitch_rep
*rep
;
739 /* disable PF RoCE so missed packets don't go through RoCE steering */
740 mlx5_dev_list_lock();
741 mlx5_remove_dev_by_protocol(esw
->dev
, MLX5_INTERFACE_PROTOCOL_IB
);
742 mlx5_dev_list_unlock();
744 err
= esw_create_offloads_fdb_tables(esw
, nvports
);
748 err
= esw_create_offloads_table(esw
);
752 err
= esw_create_vport_rx_group(esw
);
756 for (vport
= 0; vport
< nvports
; vport
++) {
757 rep
= &esw
->offloads
.vport_reps
[vport
];
761 err
= rep
->load(esw
, rep
);
769 for (vport
--; vport
>= 0; vport
--) {
770 rep
= &esw
->offloads
.vport_reps
[vport
];
773 rep
->unload(esw
, rep
);
775 esw_destroy_vport_rx_group(esw
);
778 esw_destroy_offloads_table(esw
);
781 esw_destroy_offloads_fdb_tables(esw
);
784 /* enable back PF RoCE */
785 mlx5_dev_list_lock();
786 mlx5_add_dev_by_protocol(esw
->dev
, MLX5_INTERFACE_PROTOCOL_IB
);
787 mlx5_dev_list_unlock();
792 static int esw_offloads_stop(struct mlx5_eswitch
*esw
)
794 int err
, err1
, num_vfs
= esw
->dev
->priv
.sriov
.num_vfs
;
796 mlx5_eswitch_disable_sriov(esw
);
797 err
= mlx5_eswitch_enable_sriov(esw
, num_vfs
, SRIOV_LEGACY
);
799 esw_warn(esw
->dev
, "Failed setting eswitch to legacy, err %d\n", err
);
800 err1
= mlx5_eswitch_enable_sriov(esw
, num_vfs
, SRIOV_OFFLOADS
);
802 esw_warn(esw
->dev
, "Failed setting eswitch back to offloads, err %d\n", err
);
805 /* enable back PF RoCE */
806 mlx5_dev_list_lock();
807 mlx5_add_dev_by_protocol(esw
->dev
, MLX5_INTERFACE_PROTOCOL_IB
);
808 mlx5_dev_list_unlock();
813 void esw_offloads_cleanup(struct mlx5_eswitch
*esw
, int nvports
)
815 struct mlx5_eswitch_rep
*rep
;
818 for (vport
= 0; vport
< nvports
; vport
++) {
819 rep
= &esw
->offloads
.vport_reps
[vport
];
822 rep
->unload(esw
, rep
);
825 esw_destroy_vport_rx_group(esw
);
826 esw_destroy_offloads_table(esw
);
827 esw_destroy_offloads_fdb_tables(esw
);
830 static int esw_mode_from_devlink(u16 mode
, u16
*mlx5_mode
)
833 case DEVLINK_ESWITCH_MODE_LEGACY
:
834 *mlx5_mode
= SRIOV_LEGACY
;
836 case DEVLINK_ESWITCH_MODE_SWITCHDEV
:
837 *mlx5_mode
= SRIOV_OFFLOADS
;
846 static int esw_mode_to_devlink(u16 mlx5_mode
, u16
*mode
)
850 *mode
= DEVLINK_ESWITCH_MODE_LEGACY
;
853 *mode
= DEVLINK_ESWITCH_MODE_SWITCHDEV
;
862 static int esw_inline_mode_from_devlink(u8 mode
, u8
*mlx5_mode
)
865 case DEVLINK_ESWITCH_INLINE_MODE_NONE
:
866 *mlx5_mode
= MLX5_INLINE_MODE_NONE
;
868 case DEVLINK_ESWITCH_INLINE_MODE_LINK
:
869 *mlx5_mode
= MLX5_INLINE_MODE_L2
;
871 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK
:
872 *mlx5_mode
= MLX5_INLINE_MODE_IP
;
874 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT
:
875 *mlx5_mode
= MLX5_INLINE_MODE_TCP_UDP
;
884 static int esw_inline_mode_to_devlink(u8 mlx5_mode
, u8
*mode
)
887 case MLX5_INLINE_MODE_NONE
:
888 *mode
= DEVLINK_ESWITCH_INLINE_MODE_NONE
;
890 case MLX5_INLINE_MODE_L2
:
891 *mode
= DEVLINK_ESWITCH_INLINE_MODE_LINK
;
893 case MLX5_INLINE_MODE_IP
:
894 *mode
= DEVLINK_ESWITCH_INLINE_MODE_NETWORK
;
896 case MLX5_INLINE_MODE_TCP_UDP
:
897 *mode
= DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT
;
906 static int mlx5_devlink_eswitch_check(struct devlink
*devlink
)
908 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
910 if (MLX5_CAP_GEN(dev
, port_type
) != MLX5_CAP_PORT_TYPE_ETH
)
913 if (!MLX5_CAP_GEN(dev
, vport_group_manager
))
916 if (dev
->priv
.eswitch
->mode
== SRIOV_NONE
)
922 int mlx5_devlink_eswitch_mode_set(struct devlink
*devlink
, u16 mode
)
924 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
925 u16 cur_mlx5_mode
, mlx5_mode
= 0;
928 err
= mlx5_devlink_eswitch_check(devlink
);
932 cur_mlx5_mode
= dev
->priv
.eswitch
->mode
;
934 if (esw_mode_from_devlink(mode
, &mlx5_mode
))
937 if (cur_mlx5_mode
== mlx5_mode
)
940 if (mode
== DEVLINK_ESWITCH_MODE_SWITCHDEV
)
941 return esw_offloads_start(dev
->priv
.eswitch
);
942 else if (mode
== DEVLINK_ESWITCH_MODE_LEGACY
)
943 return esw_offloads_stop(dev
->priv
.eswitch
);
948 int mlx5_devlink_eswitch_mode_get(struct devlink
*devlink
, u16
*mode
)
950 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
953 err
= mlx5_devlink_eswitch_check(devlink
);
957 return esw_mode_to_devlink(dev
->priv
.eswitch
->mode
, mode
);
960 int mlx5_devlink_eswitch_inline_mode_set(struct devlink
*devlink
, u8 mode
)
962 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
963 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
967 err
= mlx5_devlink_eswitch_check(devlink
);
971 switch (MLX5_CAP_ETH(dev
, wqe_inline_mode
)) {
972 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED
:
973 if (mode
== DEVLINK_ESWITCH_INLINE_MODE_NONE
)
976 case MLX5_CAP_INLINE_MODE_L2
:
977 esw_warn(dev
, "Inline mode can't be set\n");
979 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
:
983 if (esw
->offloads
.num_flows
> 0) {
984 esw_warn(dev
, "Can't set inline mode when flows are configured\n");
988 err
= esw_inline_mode_from_devlink(mode
, &mlx5_mode
);
992 for (vport
= 1; vport
< esw
->enabled_vports
; vport
++) {
993 err
= mlx5_modify_nic_vport_min_inline(dev
, vport
, mlx5_mode
);
995 esw_warn(dev
, "Failed to set min inline on vport %d\n",
997 goto revert_inline_mode
;
1001 esw
->offloads
.inline_mode
= mlx5_mode
;
1006 mlx5_modify_nic_vport_min_inline(dev
,
1008 esw
->offloads
.inline_mode
);
1013 int mlx5_devlink_eswitch_inline_mode_get(struct devlink
*devlink
, u8
*mode
)
1015 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
1016 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
1019 err
= mlx5_devlink_eswitch_check(devlink
);
1023 return esw_inline_mode_to_devlink(esw
->offloads
.inline_mode
, mode
);
1026 int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch
*esw
, int nvfs
, u8
*mode
)
1028 u8 prev_mlx5_mode
, mlx5_mode
= MLX5_INLINE_MODE_L2
;
1029 struct mlx5_core_dev
*dev
= esw
->dev
;
1032 if (!MLX5_CAP_GEN(dev
, vport_group_manager
))
1035 if (esw
->mode
== SRIOV_NONE
)
1038 switch (MLX5_CAP_ETH(dev
, wqe_inline_mode
)) {
1039 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED
:
1040 mlx5_mode
= MLX5_INLINE_MODE_NONE
;
1042 case MLX5_CAP_INLINE_MODE_L2
:
1043 mlx5_mode
= MLX5_INLINE_MODE_L2
;
1045 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT
:
1050 for (vport
= 1; vport
<= nvfs
; vport
++) {
1051 mlx5_query_nic_vport_min_inline(dev
, vport
, &mlx5_mode
);
1052 if (vport
> 1 && prev_mlx5_mode
!= mlx5_mode
)
1054 prev_mlx5_mode
= mlx5_mode
;
1062 int mlx5_devlink_eswitch_encap_mode_set(struct devlink
*devlink
, u8 encap
)
1064 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
1065 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
1068 err
= mlx5_devlink_eswitch_check(devlink
);
1072 if (encap
!= DEVLINK_ESWITCH_ENCAP_MODE_NONE
&&
1073 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, encap
) ||
1074 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, decap
)))
1077 if (encap
&& encap
!= DEVLINK_ESWITCH_ENCAP_MODE_BASIC
)
1080 if (esw
->mode
== SRIOV_LEGACY
) {
1081 esw
->offloads
.encap
= encap
;
1085 if (esw
->offloads
.encap
== encap
)
1088 if (esw
->offloads
.num_flows
> 0) {
1089 esw_warn(dev
, "Can't set encapsulation when flows are configured\n");
1093 esw_destroy_offloads_fast_fdb_table(esw
);
1095 esw
->offloads
.encap
= encap
;
1096 err
= esw_create_offloads_fast_fdb_table(esw
);
1098 esw_warn(esw
->dev
, "Failed re-creating fast FDB table, err %d\n", err
);
1099 esw
->offloads
.encap
= !encap
;
1100 (void)esw_create_offloads_fast_fdb_table(esw
);
1105 int mlx5_devlink_eswitch_encap_mode_get(struct devlink
*devlink
, u8
*encap
)
1107 struct mlx5_core_dev
*dev
= devlink_priv(devlink
);
1108 struct mlx5_eswitch
*esw
= dev
->priv
.eswitch
;
1111 err
= mlx5_devlink_eswitch_check(devlink
);
1115 *encap
= esw
->offloads
.encap
;
1119 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch
*esw
,
1121 struct mlx5_eswitch_rep
*__rep
)
1123 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
1124 struct mlx5_eswitch_rep
*rep
;
1126 rep
= &offloads
->vport_reps
[vport_index
];
1128 memset(rep
, 0, sizeof(*rep
));
1130 rep
->load
= __rep
->load
;
1131 rep
->unload
= __rep
->unload
;
1132 rep
->vport
= __rep
->vport
;
1133 rep
->netdev
= __rep
->netdev
;
1134 ether_addr_copy(rep
->hw_id
, __rep
->hw_id
);
1136 INIT_LIST_HEAD(&rep
->vport_sqs_list
);
1140 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch
*esw
,
1143 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
1144 struct mlx5_eswitch_rep
*rep
;
1146 rep
= &offloads
->vport_reps
[vport_index
];
1148 if (esw
->mode
== SRIOV_OFFLOADS
&& esw
->vports
[vport_index
].enabled
)
1149 rep
->unload(esw
, rep
);
1154 struct net_device
*mlx5_eswitch_get_uplink_netdev(struct mlx5_eswitch
*esw
)
1156 #define UPLINK_REP_INDEX 0
1157 struct mlx5_esw_offload
*offloads
= &esw
->offloads
;
1158 struct mlx5_eswitch_rep
*rep
;
1160 rep
= &offloads
->vport_reps
[UPLINK_REP_INDEX
];