2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/mutex.h>
34 #include <linux/mlx5/driver.h>
35 #include <linux/mlx5/vport.h>
36 #include <linux/mlx5/eswitch.h>
38 #include "mlx5_core.h"
41 #include "diag/fs_tracepoint.h"
42 #include "accel/ipsec.h"
43 #include "fpga/ipsec.h"
46 #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
47 sizeof(struct init_tree_node))
49 #define ADD_PRIO(num_prios_val, min_level_val, num_levels_val, caps_val,\
50 ...) {.type = FS_TYPE_PRIO,\
51 .min_ft_level = min_level_val,\
52 .num_levels = num_levels_val,\
53 .num_leaf_prios = num_prios_val,\
55 .children = (struct init_tree_node[]) {__VA_ARGS__},\
56 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
59 #define ADD_MULTIPLE_PRIO(num_prios_val, num_levels_val, ...)\
60 ADD_PRIO(num_prios_val, 0, num_levels_val, {},\
63 #define ADD_NS(def_miss_act, ...) {.type = FS_TYPE_NAMESPACE, \
64 .def_miss_action = def_miss_act,\
65 .children = (struct init_tree_node[]) {__VA_ARGS__},\
66 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
69 #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
72 #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
74 #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
75 .caps = (long[]) {__VA_ARGS__} }
77 #define FS_CHAINING_CAPS FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en), \
78 FS_CAP(flow_table_properties_nic_receive.modify_root), \
79 FS_CAP(flow_table_properties_nic_receive.identified_miss_table_mode), \
80 FS_CAP(flow_table_properties_nic_receive.flow_table_modify))
82 #define FS_CHAINING_CAPS_EGRESS \
84 FS_CAP(flow_table_properties_nic_transmit.flow_modify_en), \
85 FS_CAP(flow_table_properties_nic_transmit.modify_root), \
86 FS_CAP(flow_table_properties_nic_transmit \
87 .identified_miss_table_mode), \
88 FS_CAP(flow_table_properties_nic_transmit.flow_table_modify))
90 #define LEFTOVERS_NUM_LEVELS 1
91 #define LEFTOVERS_NUM_PRIOS 1
93 #define BY_PASS_PRIO_NUM_LEVELS 1
94 #define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
97 #define ETHTOOL_PRIO_NUM_LEVELS 1
98 #define ETHTOOL_NUM_PRIOS 11
99 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
100 /* Vlan, mac, ttc, inner ttc, aRFS */
101 #define KERNEL_NIC_PRIO_NUM_LEVELS 5
102 #define KERNEL_NIC_NUM_PRIOS 1
103 /* One more level for tc */
104 #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
106 #define KERNEL_NIC_TC_NUM_PRIOS 1
107 #define KERNEL_NIC_TC_NUM_LEVELS 2
109 #define ANCHOR_NUM_LEVELS 1
110 #define ANCHOR_NUM_PRIOS 1
111 #define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
113 #define OFFLOADS_MAX_FT 1
114 #define OFFLOADS_NUM_PRIOS 1
115 #define OFFLOADS_MIN_LEVEL (ANCHOR_MIN_LEVEL + 1)
117 #define LAG_PRIO_NUM_LEVELS 1
118 #define LAG_NUM_PRIOS 1
119 #define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
126 static struct init_tree_node
{
127 enum fs_node_type type
;
128 struct init_tree_node
*children
;
130 struct node_caps caps
;
135 enum mlx5_flow_table_miss_action def_miss_action
;
137 .type
= FS_TYPE_NAMESPACE
,
139 .children
= (struct init_tree_node
[]){
140 ADD_PRIO(0, BY_PASS_MIN_LEVEL
, 0, FS_CHAINING_CAPS
,
141 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
142 ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS
,
143 BY_PASS_PRIO_NUM_LEVELS
))),
144 ADD_PRIO(0, LAG_MIN_LEVEL
, 0, FS_CHAINING_CAPS
,
145 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
146 ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS
,
147 LAG_PRIO_NUM_LEVELS
))),
148 ADD_PRIO(0, OFFLOADS_MIN_LEVEL
, 0, {},
149 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
150 ADD_MULTIPLE_PRIO(OFFLOADS_NUM_PRIOS
,
152 ADD_PRIO(0, ETHTOOL_MIN_LEVEL
, 0, FS_CHAINING_CAPS
,
153 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
154 ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS
,
155 ETHTOOL_PRIO_NUM_LEVELS
))),
156 ADD_PRIO(0, KERNEL_MIN_LEVEL
, 0, {},
157 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
158 ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS
,
159 KERNEL_NIC_TC_NUM_LEVELS
),
160 ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS
,
161 KERNEL_NIC_PRIO_NUM_LEVELS
))),
162 ADD_PRIO(0, BY_PASS_MIN_LEVEL
, 0, FS_CHAINING_CAPS
,
163 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
164 ADD_MULTIPLE_PRIO(LEFTOVERS_NUM_PRIOS
,
165 LEFTOVERS_NUM_LEVELS
))),
166 ADD_PRIO(0, ANCHOR_MIN_LEVEL
, 0, {},
167 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
168 ADD_MULTIPLE_PRIO(ANCHOR_NUM_PRIOS
,
169 ANCHOR_NUM_LEVELS
))),
173 static struct init_tree_node egress_root_fs
= {
174 .type
= FS_TYPE_NAMESPACE
,
176 .children
= (struct init_tree_node
[]) {
177 ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS
, 0,
178 FS_CHAINING_CAPS_EGRESS
,
179 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
180 ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS
,
181 BY_PASS_PRIO_NUM_LEVELS
))),
185 #define RDMA_RX_BYPASS_PRIO 0
186 #define RDMA_RX_KERNEL_PRIO 1
187 static struct init_tree_node rdma_rx_root_fs
= {
188 .type
= FS_TYPE_NAMESPACE
,
190 .children
= (struct init_tree_node
[]) {
191 [RDMA_RX_BYPASS_PRIO
] =
192 ADD_PRIO(0, MLX5_BY_PASS_NUM_REGULAR_PRIOS
, 0,
194 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF
,
195 ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_REGULAR_PRIOS
,
196 BY_PASS_PRIO_NUM_LEVELS
))),
197 [RDMA_RX_KERNEL_PRIO
] =
198 ADD_PRIO(0, MLX5_BY_PASS_NUM_REGULAR_PRIOS
+ 1, 0,
200 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_SWITCH_DOMAIN
,
201 ADD_MULTIPLE_PRIO(1, 1))),
205 enum fs_i_lock_class
{
211 static const struct rhashtable_params rhash_fte
= {
212 .key_len
= FIELD_SIZEOF(struct fs_fte
, val
),
213 .key_offset
= offsetof(struct fs_fte
, val
),
214 .head_offset
= offsetof(struct fs_fte
, hash
),
215 .automatic_shrinking
= true,
219 static const struct rhashtable_params rhash_fg
= {
220 .key_len
= FIELD_SIZEOF(struct mlx5_flow_group
, mask
),
221 .key_offset
= offsetof(struct mlx5_flow_group
, mask
),
222 .head_offset
= offsetof(struct mlx5_flow_group
, hash
),
223 .automatic_shrinking
= true,
228 static void del_hw_flow_table(struct fs_node
*node
);
229 static void del_hw_flow_group(struct fs_node
*node
);
230 static void del_hw_fte(struct fs_node
*node
);
231 static void del_sw_flow_table(struct fs_node
*node
);
232 static void del_sw_flow_group(struct fs_node
*node
);
233 static void del_sw_fte(struct fs_node
*node
);
234 static void del_sw_prio(struct fs_node
*node
);
235 static void del_sw_ns(struct fs_node
*node
);
236 /* Delete rule (destination) is special case that
237 * requires to lock the FTE for all the deletion process.
239 static void del_sw_hw_rule(struct fs_node
*node
);
240 static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination
*d1
,
241 struct mlx5_flow_destination
*d2
);
242 static void cleanup_root_ns(struct mlx5_flow_root_namespace
*root_ns
);
243 static struct mlx5_flow_rule
*
244 find_flow_rule(struct fs_fte
*fte
,
245 struct mlx5_flow_destination
*dest
);
247 static void tree_init_node(struct fs_node
*node
,
248 void (*del_hw_func
)(struct fs_node
*),
249 void (*del_sw_func
)(struct fs_node
*))
251 refcount_set(&node
->refcount
, 1);
252 INIT_LIST_HEAD(&node
->list
);
253 INIT_LIST_HEAD(&node
->children
);
254 init_rwsem(&node
->lock
);
255 node
->del_hw_func
= del_hw_func
;
256 node
->del_sw_func
= del_sw_func
;
257 node
->active
= false;
260 static void tree_add_node(struct fs_node
*node
, struct fs_node
*parent
)
263 refcount_inc(&parent
->refcount
);
264 node
->parent
= parent
;
266 /* Parent is the root */
270 node
->root
= parent
->root
;
273 static int tree_get_node(struct fs_node
*node
)
275 return refcount_inc_not_zero(&node
->refcount
);
278 static void nested_down_read_ref_node(struct fs_node
*node
,
279 enum fs_i_lock_class
class)
282 down_read_nested(&node
->lock
, class);
283 refcount_inc(&node
->refcount
);
287 static void nested_down_write_ref_node(struct fs_node
*node
,
288 enum fs_i_lock_class
class)
291 down_write_nested(&node
->lock
, class);
292 refcount_inc(&node
->refcount
);
296 static void down_write_ref_node(struct fs_node
*node
, bool locked
)
300 down_write(&node
->lock
);
301 refcount_inc(&node
->refcount
);
305 static void up_read_ref_node(struct fs_node
*node
)
307 refcount_dec(&node
->refcount
);
308 up_read(&node
->lock
);
311 static void up_write_ref_node(struct fs_node
*node
, bool locked
)
313 refcount_dec(&node
->refcount
);
315 up_write(&node
->lock
);
318 static void tree_put_node(struct fs_node
*node
, bool locked
)
320 struct fs_node
*parent_node
= node
->parent
;
322 if (refcount_dec_and_test(&node
->refcount
)) {
323 if (node
->del_hw_func
)
324 node
->del_hw_func(node
);
326 /* Only root namespace doesn't have parent and we just
327 * need to free its node.
329 down_write_ref_node(parent_node
, locked
);
330 list_del_init(&node
->list
);
331 if (node
->del_sw_func
)
332 node
->del_sw_func(node
);
333 up_write_ref_node(parent_node
, locked
);
339 if (!node
&& parent_node
)
340 tree_put_node(parent_node
, locked
);
343 static int tree_remove_node(struct fs_node
*node
, bool locked
)
345 if (refcount_read(&node
->refcount
) > 1) {
346 refcount_dec(&node
->refcount
);
349 tree_put_node(node
, locked
);
353 static struct fs_prio
*find_prio(struct mlx5_flow_namespace
*ns
,
356 struct fs_prio
*iter_prio
;
358 fs_for_each_prio(iter_prio
, ns
) {
359 if (iter_prio
->prio
== prio
)
366 static bool check_valid_spec(const struct mlx5_flow_spec
*spec
)
370 for (i
= 0; i
< MLX5_ST_SZ_DW_MATCH_PARAM
; i
++)
371 if (spec
->match_value
[i
] & ~spec
->match_criteria
[i
]) {
372 pr_warn("mlx5_core: match_value differs from match_criteria\n");
379 static struct mlx5_flow_root_namespace
*find_root(struct fs_node
*node
)
381 struct fs_node
*root
;
382 struct mlx5_flow_namespace
*ns
;
386 if (WARN_ON(root
->type
!= FS_TYPE_NAMESPACE
)) {
387 pr_warn("mlx5: flow steering node is not in tree or garbaged\n");
391 ns
= container_of(root
, struct mlx5_flow_namespace
, node
);
392 return container_of(ns
, struct mlx5_flow_root_namespace
, ns
);
395 static inline struct mlx5_flow_steering
*get_steering(struct fs_node
*node
)
397 struct mlx5_flow_root_namespace
*root
= find_root(node
);
400 return root
->dev
->priv
.steering
;
404 static inline struct mlx5_core_dev
*get_dev(struct fs_node
*node
)
406 struct mlx5_flow_root_namespace
*root
= find_root(node
);
413 static void del_sw_ns(struct fs_node
*node
)
418 static void del_sw_prio(struct fs_node
*node
)
423 static void del_hw_flow_table(struct fs_node
*node
)
425 struct mlx5_flow_root_namespace
*root
;
426 struct mlx5_flow_table
*ft
;
427 struct mlx5_core_dev
*dev
;
430 fs_get_obj(ft
, node
);
431 dev
= get_dev(&ft
->node
);
432 root
= find_root(&ft
->node
);
433 trace_mlx5_fs_del_ft(ft
);
436 err
= root
->cmds
->destroy_flow_table(root
, ft
);
438 mlx5_core_warn(dev
, "flow steering can't destroy ft\n");
442 static void del_sw_flow_table(struct fs_node
*node
)
444 struct mlx5_flow_table
*ft
;
445 struct fs_prio
*prio
;
447 fs_get_obj(ft
, node
);
449 rhltable_destroy(&ft
->fgs_hash
);
450 fs_get_obj(prio
, ft
->node
.parent
);
455 static void modify_fte(struct fs_fte
*fte
)
457 struct mlx5_flow_root_namespace
*root
;
458 struct mlx5_flow_table
*ft
;
459 struct mlx5_flow_group
*fg
;
460 struct mlx5_core_dev
*dev
;
463 fs_get_obj(fg
, fte
->node
.parent
);
464 fs_get_obj(ft
, fg
->node
.parent
);
465 dev
= get_dev(&fte
->node
);
467 root
= find_root(&ft
->node
);
468 err
= root
->cmds
->update_fte(root
, ft
, fg
, fte
->modify_mask
, fte
);
471 "%s can't del rule fg id=%d fte_index=%d\n",
472 __func__
, fg
->id
, fte
->index
);
473 fte
->modify_mask
= 0;
476 static void del_sw_hw_rule(struct fs_node
*node
)
478 struct mlx5_flow_rule
*rule
;
481 fs_get_obj(rule
, node
);
482 fs_get_obj(fte
, rule
->node
.parent
);
483 trace_mlx5_fs_del_rule(rule
);
484 if (rule
->sw_action
== MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
) {
485 mutex_lock(&rule
->dest_attr
.ft
->lock
);
486 list_del(&rule
->next_ft
);
487 mutex_unlock(&rule
->dest_attr
.ft
->lock
);
490 if (rule
->dest_attr
.type
== MLX5_FLOW_DESTINATION_TYPE_COUNTER
&&
493 BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION
) |
494 BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS
);
495 fte
->action
.action
&= ~MLX5_FLOW_CONTEXT_ACTION_COUNT
;
499 if ((fte
->action
.action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
) &&
502 BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST
);
508 static void del_hw_fte(struct fs_node
*node
)
510 struct mlx5_flow_root_namespace
*root
;
511 struct mlx5_flow_table
*ft
;
512 struct mlx5_flow_group
*fg
;
513 struct mlx5_core_dev
*dev
;
517 fs_get_obj(fte
, node
);
518 fs_get_obj(fg
, fte
->node
.parent
);
519 fs_get_obj(ft
, fg
->node
.parent
);
521 trace_mlx5_fs_del_fte(fte
);
522 dev
= get_dev(&ft
->node
);
523 root
= find_root(&ft
->node
);
525 err
= root
->cmds
->delete_fte(root
, ft
, fte
);
528 "flow steering can't delete fte in index %d of flow group id %d\n",
534 static void del_sw_fte_rcu(struct rcu_head
*head
)
536 struct fs_fte
*fte
= container_of(head
, struct fs_fte
, rcu
);
537 struct mlx5_flow_steering
*steering
= get_steering(&fte
->node
);
539 kmem_cache_free(steering
->ftes_cache
, fte
);
542 static void del_sw_fte(struct fs_node
*node
)
544 struct mlx5_flow_group
*fg
;
548 fs_get_obj(fte
, node
);
549 fs_get_obj(fg
, fte
->node
.parent
);
551 err
= rhashtable_remove_fast(&fg
->ftes_hash
,
555 ida_simple_remove(&fg
->fte_allocator
, fte
->index
- fg
->start_index
);
557 call_rcu(&fte
->rcu
, del_sw_fte_rcu
);
560 static void del_hw_flow_group(struct fs_node
*node
)
562 struct mlx5_flow_root_namespace
*root
;
563 struct mlx5_flow_group
*fg
;
564 struct mlx5_flow_table
*ft
;
565 struct mlx5_core_dev
*dev
;
567 fs_get_obj(fg
, node
);
568 fs_get_obj(ft
, fg
->node
.parent
);
569 dev
= get_dev(&ft
->node
);
570 trace_mlx5_fs_del_fg(fg
);
572 root
= find_root(&ft
->node
);
573 if (fg
->node
.active
&& root
->cmds
->destroy_flow_group(root
, ft
, fg
))
574 mlx5_core_warn(dev
, "flow steering can't destroy fg %d of ft %d\n",
578 static void del_sw_flow_group(struct fs_node
*node
)
580 struct mlx5_flow_steering
*steering
= get_steering(node
);
581 struct mlx5_flow_group
*fg
;
582 struct mlx5_flow_table
*ft
;
585 fs_get_obj(fg
, node
);
586 fs_get_obj(ft
, fg
->node
.parent
);
588 rhashtable_destroy(&fg
->ftes_hash
);
589 ida_destroy(&fg
->fte_allocator
);
590 if (ft
->autogroup
.active
&& fg
->max_ftes
== ft
->autogroup
.group_size
)
591 ft
->autogroup
.num_groups
--;
592 err
= rhltable_remove(&ft
->fgs_hash
,
596 kmem_cache_free(steering
->fgs_cache
, fg
);
599 static int insert_fte(struct mlx5_flow_group
*fg
, struct fs_fte
*fte
)
604 index
= ida_simple_get(&fg
->fte_allocator
, 0, fg
->max_ftes
, GFP_KERNEL
);
608 fte
->index
= index
+ fg
->start_index
;
609 ret
= rhashtable_insert_fast(&fg
->ftes_hash
,
615 tree_add_node(&fte
->node
, &fg
->node
);
616 list_add_tail(&fte
->node
.list
, &fg
->node
.children
);
620 ida_simple_remove(&fg
->fte_allocator
, index
);
624 static struct fs_fte
*alloc_fte(struct mlx5_flow_table
*ft
,
625 const struct mlx5_flow_spec
*spec
,
626 struct mlx5_flow_act
*flow_act
)
628 struct mlx5_flow_steering
*steering
= get_steering(&ft
->node
);
631 fte
= kmem_cache_zalloc(steering
->ftes_cache
, GFP_KERNEL
);
633 return ERR_PTR(-ENOMEM
);
635 memcpy(fte
->val
, &spec
->match_value
, sizeof(fte
->val
));
636 fte
->node
.type
= FS_TYPE_FLOW_ENTRY
;
637 fte
->action
= *flow_act
;
638 fte
->flow_context
= spec
->flow_context
;
640 tree_init_node(&fte
->node
, NULL
, del_sw_fte
);
645 static void dealloc_flow_group(struct mlx5_flow_steering
*steering
,
646 struct mlx5_flow_group
*fg
)
648 rhashtable_destroy(&fg
->ftes_hash
);
649 kmem_cache_free(steering
->fgs_cache
, fg
);
652 static struct mlx5_flow_group
*alloc_flow_group(struct mlx5_flow_steering
*steering
,
653 u8 match_criteria_enable
,
654 const void *match_criteria
,
658 struct mlx5_flow_group
*fg
;
661 fg
= kmem_cache_zalloc(steering
->fgs_cache
, GFP_KERNEL
);
663 return ERR_PTR(-ENOMEM
);
665 ret
= rhashtable_init(&fg
->ftes_hash
, &rhash_fte
);
667 kmem_cache_free(steering
->fgs_cache
, fg
);
671 ida_init(&fg
->fte_allocator
);
672 fg
->mask
.match_criteria_enable
= match_criteria_enable
;
673 memcpy(&fg
->mask
.match_criteria
, match_criteria
,
674 sizeof(fg
->mask
.match_criteria
));
675 fg
->node
.type
= FS_TYPE_FLOW_GROUP
;
676 fg
->start_index
= start_index
;
677 fg
->max_ftes
= end_index
- start_index
+ 1;
682 static struct mlx5_flow_group
*alloc_insert_flow_group(struct mlx5_flow_table
*ft
,
683 u8 match_criteria_enable
,
684 const void *match_criteria
,
687 struct list_head
*prev
)
689 struct mlx5_flow_steering
*steering
= get_steering(&ft
->node
);
690 struct mlx5_flow_group
*fg
;
693 fg
= alloc_flow_group(steering
, match_criteria_enable
, match_criteria
,
694 start_index
, end_index
);
698 /* initialize refcnt, add to parent list */
699 ret
= rhltable_insert(&ft
->fgs_hash
,
703 dealloc_flow_group(steering
, fg
);
707 tree_init_node(&fg
->node
, del_hw_flow_group
, del_sw_flow_group
);
708 tree_add_node(&fg
->node
, &ft
->node
);
709 /* Add node to group list */
710 list_add(&fg
->node
.list
, prev
);
711 atomic_inc(&ft
->node
.version
);
716 static struct mlx5_flow_table
*alloc_flow_table(int level
, u16 vport
, int max_fte
,
717 enum fs_flow_table_type table_type
,
718 enum fs_flow_table_op_mod op_mod
,
721 struct mlx5_flow_table
*ft
;
724 ft
= kzalloc(sizeof(*ft
), GFP_KERNEL
);
726 return ERR_PTR(-ENOMEM
);
728 ret
= rhltable_init(&ft
->fgs_hash
, &rhash_fg
);
735 ft
->node
.type
= FS_TYPE_FLOW_TABLE
;
737 ft
->type
= table_type
;
739 ft
->max_fte
= max_fte
;
741 INIT_LIST_HEAD(&ft
->fwd_rules
);
742 mutex_init(&ft
->lock
);
747 /* If reverse is false, then we search for the first flow table in the
748 * root sub-tree from start(closest from right), else we search for the
749 * last flow table in the root sub-tree till start(closest from left).
751 static struct mlx5_flow_table
*find_closest_ft_recursive(struct fs_node
*root
,
752 struct list_head
*start
,
755 #define list_advance_entry(pos, reverse) \
756 ((reverse) ? list_prev_entry(pos, list) : list_next_entry(pos, list))
758 #define list_for_each_advance_continue(pos, head, reverse) \
759 for (pos = list_advance_entry(pos, reverse); \
760 &pos->list != (head); \
761 pos = list_advance_entry(pos, reverse))
763 struct fs_node
*iter
= list_entry(start
, struct fs_node
, list
);
764 struct mlx5_flow_table
*ft
= NULL
;
766 if (!root
|| root
->type
== FS_TYPE_PRIO_CHAINS
)
769 list_for_each_advance_continue(iter
, &root
->children
, reverse
) {
770 if (iter
->type
== FS_TYPE_FLOW_TABLE
) {
771 fs_get_obj(ft
, iter
);
774 ft
= find_closest_ft_recursive(iter
, &iter
->children
, reverse
);
782 /* If reverse if false then return the first flow table in next priority of
783 * prio in the tree, else return the last flow table in the previous priority
784 * of prio in the tree.
786 static struct mlx5_flow_table
*find_closest_ft(struct fs_prio
*prio
, bool reverse
)
788 struct mlx5_flow_table
*ft
= NULL
;
789 struct fs_node
*curr_node
;
790 struct fs_node
*parent
;
792 parent
= prio
->node
.parent
;
793 curr_node
= &prio
->node
;
794 while (!ft
&& parent
) {
795 ft
= find_closest_ft_recursive(parent
, &curr_node
->list
, reverse
);
797 parent
= curr_node
->parent
;
802 /* Assuming all the tree is locked by mutex chain lock */
803 static struct mlx5_flow_table
*find_next_chained_ft(struct fs_prio
*prio
)
805 return find_closest_ft(prio
, false);
808 /* Assuming all the tree is locked by mutex chain lock */
809 static struct mlx5_flow_table
*find_prev_chained_ft(struct fs_prio
*prio
)
811 return find_closest_ft(prio
, true);
814 static int connect_fts_in_prio(struct mlx5_core_dev
*dev
,
815 struct fs_prio
*prio
,
816 struct mlx5_flow_table
*ft
)
818 struct mlx5_flow_root_namespace
*root
= find_root(&prio
->node
);
819 struct mlx5_flow_table
*iter
;
823 fs_for_each_ft(iter
, prio
) {
825 err
= root
->cmds
->modify_flow_table(root
, iter
, ft
);
827 mlx5_core_warn(dev
, "Failed to modify flow table %d\n",
829 /* The driver is out of sync with the FW */
838 /* Connect flow tables from previous priority of prio to ft */
839 static int connect_prev_fts(struct mlx5_core_dev
*dev
,
840 struct mlx5_flow_table
*ft
,
841 struct fs_prio
*prio
)
843 struct mlx5_flow_table
*prev_ft
;
845 prev_ft
= find_prev_chained_ft(prio
);
847 struct fs_prio
*prev_prio
;
849 fs_get_obj(prev_prio
, prev_ft
->node
.parent
);
850 return connect_fts_in_prio(dev
, prev_prio
, ft
);
855 static int update_root_ft_create(struct mlx5_flow_table
*ft
, struct fs_prio
858 struct mlx5_flow_root_namespace
*root
= find_root(&prio
->node
);
859 struct mlx5_ft_underlay_qp
*uqp
;
860 int min_level
= INT_MAX
;
865 min_level
= root
->root_ft
->level
;
867 if (ft
->level
>= min_level
)
870 if (list_empty(&root
->underlay_qpns
)) {
871 /* Don't set any QPN (zero) in case QPN list is empty */
873 err
= root
->cmds
->update_root_ft(root
, ft
, qpn
, false);
875 list_for_each_entry(uqp
, &root
->underlay_qpns
, list
) {
877 err
= root
->cmds
->update_root_ft(root
, ft
,
885 mlx5_core_warn(root
->dev
,
886 "Update root flow table of id(%u) qpn(%d) failed\n",
894 static int _mlx5_modify_rule_destination(struct mlx5_flow_rule
*rule
,
895 struct mlx5_flow_destination
*dest
)
897 struct mlx5_flow_root_namespace
*root
;
898 struct mlx5_flow_table
*ft
;
899 struct mlx5_flow_group
*fg
;
901 int modify_mask
= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST
);
904 fs_get_obj(fte
, rule
->node
.parent
);
905 if (!(fte
->action
.action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
))
907 down_write_ref_node(&fte
->node
, false);
908 fs_get_obj(fg
, fte
->node
.parent
);
909 fs_get_obj(ft
, fg
->node
.parent
);
911 memcpy(&rule
->dest_attr
, dest
, sizeof(*dest
));
912 root
= find_root(&ft
->node
);
913 err
= root
->cmds
->update_fte(root
, ft
, fg
,
915 up_write_ref_node(&fte
->node
, false);
920 int mlx5_modify_rule_destination(struct mlx5_flow_handle
*handle
,
921 struct mlx5_flow_destination
*new_dest
,
922 struct mlx5_flow_destination
*old_dest
)
927 if (handle
->num_rules
!= 1)
929 return _mlx5_modify_rule_destination(handle
->rule
[0],
933 for (i
= 0; i
< handle
->num_rules
; i
++) {
934 if (mlx5_flow_dests_cmp(new_dest
, &handle
->rule
[i
]->dest_attr
))
935 return _mlx5_modify_rule_destination(handle
->rule
[i
],
942 /* Modify/set FWD rules that point on old_next_ft to point on new_next_ft */
943 static int connect_fwd_rules(struct mlx5_core_dev
*dev
,
944 struct mlx5_flow_table
*new_next_ft
,
945 struct mlx5_flow_table
*old_next_ft
)
947 struct mlx5_flow_destination dest
= {};
948 struct mlx5_flow_rule
*iter
;
951 /* new_next_ft and old_next_ft could be NULL only
952 * when we create/destroy the anchor flow table.
954 if (!new_next_ft
|| !old_next_ft
)
957 dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
958 dest
.ft
= new_next_ft
;
960 mutex_lock(&old_next_ft
->lock
);
961 list_splice_init(&old_next_ft
->fwd_rules
, &new_next_ft
->fwd_rules
);
962 mutex_unlock(&old_next_ft
->lock
);
963 list_for_each_entry(iter
, &new_next_ft
->fwd_rules
, next_ft
) {
964 err
= _mlx5_modify_rule_destination(iter
, &dest
);
966 pr_err("mlx5_core: failed to modify rule to point on flow table %d\n",
972 static int connect_flow_table(struct mlx5_core_dev
*dev
, struct mlx5_flow_table
*ft
,
973 struct fs_prio
*prio
)
975 struct mlx5_flow_table
*next_ft
;
978 /* Connect_prev_fts and update_root_ft_create are mutually exclusive */
980 if (list_empty(&prio
->node
.children
)) {
981 err
= connect_prev_fts(dev
, ft
, prio
);
985 next_ft
= find_next_chained_ft(prio
);
986 err
= connect_fwd_rules(dev
, ft
, next_ft
);
991 if (MLX5_CAP_FLOWTABLE(dev
,
992 flow_table_properties_nic_receive
.modify_root
))
993 err
= update_root_ft_create(ft
, prio
);
997 static void list_add_flow_table(struct mlx5_flow_table
*ft
,
998 struct fs_prio
*prio
)
1000 struct list_head
*prev
= &prio
->node
.children
;
1001 struct mlx5_flow_table
*iter
;
1003 fs_for_each_ft(iter
, prio
) {
1004 if (iter
->level
> ft
->level
)
1006 prev
= &iter
->node
.list
;
1008 list_add(&ft
->node
.list
, prev
);
1011 static struct mlx5_flow_table
*__mlx5_create_flow_table(struct mlx5_flow_namespace
*ns
,
1012 struct mlx5_flow_table_attr
*ft_attr
,
1013 enum fs_flow_table_op_mod op_mod
,
1016 struct mlx5_flow_root_namespace
*root
= find_root(&ns
->node
);
1017 struct mlx5_flow_table
*next_ft
= NULL
;
1018 struct fs_prio
*fs_prio
= NULL
;
1019 struct mlx5_flow_table
*ft
;
1024 pr_err("mlx5: flow steering failed to find root of namespace\n");
1025 return ERR_PTR(-ENODEV
);
1028 mutex_lock(&root
->chain_lock
);
1029 fs_prio
= find_prio(ns
, ft_attr
->prio
);
1034 if (ft_attr
->level
>= fs_prio
->num_levels
) {
1038 /* The level is related to the
1039 * priority level range.
1041 ft_attr
->level
+= fs_prio
->start_level
;
1042 ft
= alloc_flow_table(ft_attr
->level
,
1044 ft_attr
->max_fte
? roundup_pow_of_two(ft_attr
->max_fte
) : 0,
1046 op_mod
, ft_attr
->flags
);
1052 tree_init_node(&ft
->node
, del_hw_flow_table
, del_sw_flow_table
);
1053 log_table_sz
= ft
->max_fte
? ilog2(ft
->max_fte
) : 0;
1054 next_ft
= find_next_chained_ft(fs_prio
);
1055 ft
->def_miss_action
= ns
->def_miss_action
;
1056 err
= root
->cmds
->create_flow_table(root
, ft
, log_table_sz
, next_ft
);
1060 err
= connect_flow_table(root
->dev
, ft
, fs_prio
);
1063 ft
->node
.active
= true;
1064 down_write_ref_node(&fs_prio
->node
, false);
1065 tree_add_node(&ft
->node
, &fs_prio
->node
);
1066 list_add_flow_table(ft
, fs_prio
);
1068 up_write_ref_node(&fs_prio
->node
, false);
1069 mutex_unlock(&root
->chain_lock
);
1070 trace_mlx5_fs_add_ft(ft
);
1073 root
->cmds
->destroy_flow_table(root
, ft
);
1077 mutex_unlock(&root
->chain_lock
);
1078 return ERR_PTR(err
);
1081 struct mlx5_flow_table
*mlx5_create_flow_table(struct mlx5_flow_namespace
*ns
,
1082 struct mlx5_flow_table_attr
*ft_attr
)
1084 return __mlx5_create_flow_table(ns
, ft_attr
, FS_FT_OP_MOD_NORMAL
, 0);
1087 struct mlx5_flow_table
*mlx5_create_vport_flow_table(struct mlx5_flow_namespace
*ns
,
1088 int prio
, int max_fte
,
1089 u32 level
, u16 vport
)
1091 struct mlx5_flow_table_attr ft_attr
= {};
1093 ft_attr
.max_fte
= max_fte
;
1094 ft_attr
.level
= level
;
1095 ft_attr
.prio
= prio
;
1097 return __mlx5_create_flow_table(ns
, &ft_attr
, FS_FT_OP_MOD_NORMAL
, vport
);
1100 struct mlx5_flow_table
*
1101 mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace
*ns
,
1102 int prio
, u32 level
)
1104 struct mlx5_flow_table_attr ft_attr
= {};
1106 ft_attr
.level
= level
;
1107 ft_attr
.prio
= prio
;
1108 return __mlx5_create_flow_table(ns
, &ft_attr
, FS_FT_OP_MOD_LAG_DEMUX
, 0);
1110 EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table
);
1112 struct mlx5_flow_table
*
1113 mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace
*ns
,
1115 int num_flow_table_entries
,
1120 struct mlx5_flow_table_attr ft_attr
= {};
1121 struct mlx5_flow_table
*ft
;
1123 if (max_num_groups
> num_flow_table_entries
)
1124 return ERR_PTR(-EINVAL
);
1126 ft_attr
.max_fte
= num_flow_table_entries
;
1127 ft_attr
.prio
= prio
;
1128 ft_attr
.level
= level
;
1129 ft_attr
.flags
= flags
;
1131 ft
= mlx5_create_flow_table(ns
, &ft_attr
);
1135 ft
->autogroup
.active
= true;
1136 ft
->autogroup
.required_groups
= max_num_groups
;
1137 /* We save place for flow groups in addition to max types */
1138 ft
->autogroup
.group_size
= ft
->max_fte
/ (max_num_groups
+ 1);
1142 EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table
);
1144 struct mlx5_flow_group
*mlx5_create_flow_group(struct mlx5_flow_table
*ft
,
1147 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1148 void *match_criteria
= MLX5_ADDR_OF(create_flow_group_in
,
1149 fg_in
, match_criteria
);
1150 u8 match_criteria_enable
= MLX5_GET(create_flow_group_in
,
1152 match_criteria_enable
);
1153 int start_index
= MLX5_GET(create_flow_group_in
, fg_in
,
1155 int end_index
= MLX5_GET(create_flow_group_in
, fg_in
,
1157 struct mlx5_flow_group
*fg
;
1160 if (ft
->autogroup
.active
)
1161 return ERR_PTR(-EPERM
);
1163 down_write_ref_node(&ft
->node
, false);
1164 fg
= alloc_insert_flow_group(ft
, match_criteria_enable
, match_criteria
,
1165 start_index
, end_index
,
1166 ft
->node
.children
.prev
);
1167 up_write_ref_node(&ft
->node
, false);
1171 err
= root
->cmds
->create_flow_group(root
, ft
, fg_in
, fg
);
1173 tree_put_node(&fg
->node
, false);
1174 return ERR_PTR(err
);
1176 trace_mlx5_fs_add_fg(fg
);
1177 fg
->node
.active
= true;
1182 static struct mlx5_flow_rule
*alloc_rule(struct mlx5_flow_destination
*dest
)
1184 struct mlx5_flow_rule
*rule
;
1186 rule
= kzalloc(sizeof(*rule
), GFP_KERNEL
);
1190 INIT_LIST_HEAD(&rule
->next_ft
);
1191 rule
->node
.type
= FS_TYPE_FLOW_DEST
;
1193 memcpy(&rule
->dest_attr
, dest
, sizeof(*dest
));
1198 static struct mlx5_flow_handle
*alloc_handle(int num_rules
)
1200 struct mlx5_flow_handle
*handle
;
1202 handle
= kzalloc(struct_size(handle
, rule
, num_rules
), GFP_KERNEL
);
1206 handle
->num_rules
= num_rules
;
1211 static void destroy_flow_handle(struct fs_fte
*fte
,
1212 struct mlx5_flow_handle
*handle
,
1213 struct mlx5_flow_destination
*dest
,
1217 if (refcount_dec_and_test(&handle
->rule
[i
]->node
.refcount
)) {
1219 list_del(&handle
->rule
[i
]->node
.list
);
1220 kfree(handle
->rule
[i
]);
1226 static struct mlx5_flow_handle
*
1227 create_flow_handle(struct fs_fte
*fte
,
1228 struct mlx5_flow_destination
*dest
,
1233 struct mlx5_flow_handle
*handle
;
1234 struct mlx5_flow_rule
*rule
= NULL
;
1235 static int count
= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_FLOW_COUNTERS
);
1236 static int dst
= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_DESTINATION_LIST
);
1240 handle
= alloc_handle((dest_num
) ? dest_num
: 1);
1242 return ERR_PTR(-ENOMEM
);
1246 rule
= find_flow_rule(fte
, dest
+ i
);
1248 refcount_inc(&rule
->node
.refcount
);
1254 rule
= alloc_rule(dest
+ i
);
1258 /* Add dest to dests list- we need flow tables to be in the
1259 * end of the list for forward to next prio rules.
1261 tree_init_node(&rule
->node
, NULL
, del_sw_hw_rule
);
1263 dest
[i
].type
!= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
)
1264 list_add(&rule
->node
.list
, &fte
->node
.children
);
1266 list_add_tail(&rule
->node
.list
, &fte
->node
.children
);
1270 type
= dest
[i
].type
==
1271 MLX5_FLOW_DESTINATION_TYPE_COUNTER
;
1272 *modify_mask
|= type
? count
: dst
;
1275 handle
->rule
[i
] = rule
;
1276 } while (++i
< dest_num
);
1281 destroy_flow_handle(fte
, handle
, dest
, i
);
1282 return ERR_PTR(-ENOMEM
);
1285 /* fte should not be deleted while calling this function */
1286 static struct mlx5_flow_handle
*
1287 add_rule_fte(struct fs_fte
*fte
,
1288 struct mlx5_flow_group
*fg
,
1289 struct mlx5_flow_destination
*dest
,
1293 struct mlx5_flow_root_namespace
*root
;
1294 struct mlx5_flow_handle
*handle
;
1295 struct mlx5_flow_table
*ft
;
1296 int modify_mask
= 0;
1298 bool new_rule
= false;
1300 handle
= create_flow_handle(fte
, dest
, dest_num
, &modify_mask
,
1302 if (IS_ERR(handle
) || !new_rule
)
1306 modify_mask
|= BIT(MLX5_SET_FTE_MODIFY_ENABLE_MASK_ACTION
);
1308 fs_get_obj(ft
, fg
->node
.parent
);
1309 root
= find_root(&fg
->node
);
1310 if (!(fte
->status
& FS_FTE_STATUS_EXISTING
))
1311 err
= root
->cmds
->create_fte(root
, ft
, fg
, fte
);
1313 err
= root
->cmds
->update_fte(root
, ft
, fg
, modify_mask
, fte
);
1317 fte
->node
.active
= true;
1318 fte
->status
|= FS_FTE_STATUS_EXISTING
;
1319 atomic_inc(&fte
->node
.version
);
1325 destroy_flow_handle(fte
, handle
, dest
, handle
->num_rules
);
1326 return ERR_PTR(err
);
1329 static struct mlx5_flow_group
*alloc_auto_flow_group(struct mlx5_flow_table
*ft
,
1330 const struct mlx5_flow_spec
*spec
)
1332 struct list_head
*prev
= &ft
->node
.children
;
1333 struct mlx5_flow_group
*fg
;
1334 unsigned int candidate_index
= 0;
1335 unsigned int group_size
= 0;
1337 if (!ft
->autogroup
.active
)
1338 return ERR_PTR(-ENOENT
);
1340 if (ft
->autogroup
.num_groups
< ft
->autogroup
.required_groups
)
1341 group_size
= ft
->autogroup
.group_size
;
1343 /* ft->max_fte == ft->autogroup.max_types */
1344 if (group_size
== 0)
1347 /* sorted by start_index */
1348 fs_for_each_fg(fg
, ft
) {
1349 if (candidate_index
+ group_size
> fg
->start_index
)
1350 candidate_index
= fg
->start_index
+ fg
->max_ftes
;
1353 prev
= &fg
->node
.list
;
1356 if (candidate_index
+ group_size
> ft
->max_fte
)
1357 return ERR_PTR(-ENOSPC
);
1359 fg
= alloc_insert_flow_group(ft
,
1360 spec
->match_criteria_enable
,
1361 spec
->match_criteria
,
1363 candidate_index
+ group_size
- 1,
1368 if (group_size
== ft
->autogroup
.group_size
)
1369 ft
->autogroup
.num_groups
++;
1375 static int create_auto_flow_group(struct mlx5_flow_table
*ft
,
1376 struct mlx5_flow_group
*fg
)
1378 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1379 int inlen
= MLX5_ST_SZ_BYTES(create_flow_group_in
);
1380 void *match_criteria_addr
;
1381 u8 src_esw_owner_mask_on
;
1386 in
= kvzalloc(inlen
, GFP_KERNEL
);
1390 MLX5_SET(create_flow_group_in
, in
, match_criteria_enable
,
1391 fg
->mask
.match_criteria_enable
);
1392 MLX5_SET(create_flow_group_in
, in
, start_flow_index
, fg
->start_index
);
1393 MLX5_SET(create_flow_group_in
, in
, end_flow_index
, fg
->start_index
+
1396 misc
= MLX5_ADDR_OF(fte_match_param
, fg
->mask
.match_criteria
,
1398 src_esw_owner_mask_on
= !!MLX5_GET(fte_match_set_misc
, misc
,
1399 source_eswitch_owner_vhca_id
);
1400 MLX5_SET(create_flow_group_in
, in
,
1401 source_eswitch_owner_vhca_id_valid
, src_esw_owner_mask_on
);
1403 match_criteria_addr
= MLX5_ADDR_OF(create_flow_group_in
,
1404 in
, match_criteria
);
1405 memcpy(match_criteria_addr
, fg
->mask
.match_criteria
,
1406 sizeof(fg
->mask
.match_criteria
));
1408 err
= root
->cmds
->create_flow_group(root
, ft
, in
, fg
);
1410 fg
->node
.active
= true;
1411 trace_mlx5_fs_add_fg(fg
);
1418 static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination
*d1
,
1419 struct mlx5_flow_destination
*d2
)
1421 if (d1
->type
== d2
->type
) {
1422 if ((d1
->type
== MLX5_FLOW_DESTINATION_TYPE_VPORT
&&
1423 d1
->vport
.num
== d2
->vport
.num
&&
1424 d1
->vport
.flags
== d2
->vport
.flags
&&
1425 ((d1
->vport
.flags
& MLX5_FLOW_DEST_VPORT_VHCA_ID
) ?
1426 (d1
->vport
.vhca_id
== d2
->vport
.vhca_id
) : true) &&
1427 ((d1
->vport
.flags
& MLX5_FLOW_DEST_VPORT_REFORMAT_ID
) ?
1428 (d1
->vport
.pkt_reformat
->id
==
1429 d2
->vport
.pkt_reformat
->id
) : true)) ||
1430 (d1
->type
== MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
&&
1431 d1
->ft
== d2
->ft
) ||
1432 (d1
->type
== MLX5_FLOW_DESTINATION_TYPE_TIR
&&
1433 d1
->tir_num
== d2
->tir_num
) ||
1434 (d1
->type
== MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM
&&
1435 d1
->ft_num
== d2
->ft_num
))
1442 static struct mlx5_flow_rule
*find_flow_rule(struct fs_fte
*fte
,
1443 struct mlx5_flow_destination
*dest
)
1445 struct mlx5_flow_rule
*rule
;
1447 list_for_each_entry(rule
, &fte
->node
.children
, node
.list
) {
1448 if (mlx5_flow_dests_cmp(&rule
->dest_attr
, dest
))
1454 static bool check_conflicting_actions(u32 action1
, u32 action2
)
1456 u32 xored_actions
= action1
^ action2
;
1458 /* if one rule only wants to count, it's ok */
1459 if (action1
== MLX5_FLOW_CONTEXT_ACTION_COUNT
||
1460 action2
== MLX5_FLOW_CONTEXT_ACTION_COUNT
)
1463 if (xored_actions
& (MLX5_FLOW_CONTEXT_ACTION_DROP
|
1464 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT
|
1465 MLX5_FLOW_CONTEXT_ACTION_DECAP
|
1466 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR
|
1467 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP
|
1468 MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH
|
1469 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2
|
1470 MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2
))
1476 static int check_conflicting_ftes(struct fs_fte
*fte
,
1477 const struct mlx5_flow_context
*flow_context
,
1478 const struct mlx5_flow_act
*flow_act
)
1480 if (check_conflicting_actions(flow_act
->action
, fte
->action
.action
)) {
1481 mlx5_core_warn(get_dev(&fte
->node
),
1482 "Found two FTEs with conflicting actions\n");
1486 if ((flow_context
->flags
& FLOW_CONTEXT_HAS_TAG
) &&
1487 fte
->flow_context
.flow_tag
!= flow_context
->flow_tag
) {
1488 mlx5_core_warn(get_dev(&fte
->node
),
1489 "FTE flow tag %u already exists with different flow tag %u\n",
1490 fte
->flow_context
.flow_tag
,
1491 flow_context
->flow_tag
);
1498 static struct mlx5_flow_handle
*add_rule_fg(struct mlx5_flow_group
*fg
,
1499 const struct mlx5_flow_spec
*spec
,
1500 struct mlx5_flow_act
*flow_act
,
1501 struct mlx5_flow_destination
*dest
,
1505 struct mlx5_flow_handle
*handle
;
1510 ret
= check_conflicting_ftes(fte
, &spec
->flow_context
, flow_act
);
1512 return ERR_PTR(ret
);
1514 old_action
= fte
->action
.action
;
1515 fte
->action
.action
|= flow_act
->action
;
1516 handle
= add_rule_fte(fte
, fg
, dest
, dest_num
,
1517 old_action
!= flow_act
->action
);
1518 if (IS_ERR(handle
)) {
1519 fte
->action
.action
= old_action
;
1522 trace_mlx5_fs_set_fte(fte
, false);
1524 for (i
= 0; i
< handle
->num_rules
; i
++) {
1525 if (refcount_read(&handle
->rule
[i
]->node
.refcount
) == 1) {
1526 tree_add_node(&handle
->rule
[i
]->node
, &fte
->node
);
1527 trace_mlx5_fs_add_rule(handle
->rule
[i
]);
1533 static bool counter_is_valid(u32 action
)
1535 return (action
& (MLX5_FLOW_CONTEXT_ACTION_DROP
|
1536 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
));
1539 static bool dest_is_valid(struct mlx5_flow_destination
*dest
,
1541 struct mlx5_flow_table
*ft
)
1543 if (dest
&& (dest
->type
== MLX5_FLOW_DESTINATION_TYPE_COUNTER
))
1544 return counter_is_valid(action
);
1546 if (!(action
& MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
))
1549 if (!dest
|| ((dest
->type
==
1550 MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
) &&
1551 (dest
->ft
->level
<= ft
->level
)))
1557 struct list_head list
;
1558 struct mlx5_flow_group
*g
;
1561 struct match_list_head
{
1562 struct list_head list
;
1563 struct match_list first
;
1566 static void free_match_list(struct match_list_head
*head
)
1568 if (!list_empty(&head
->list
)) {
1569 struct match_list
*iter
, *match_tmp
;
1571 list_del(&head
->first
.list
);
1572 tree_put_node(&head
->first
.g
->node
, false);
1573 list_for_each_entry_safe(iter
, match_tmp
, &head
->list
,
1575 tree_put_node(&iter
->g
->node
, false);
1576 list_del(&iter
->list
);
1582 static int build_match_list(struct match_list_head
*match_head
,
1583 struct mlx5_flow_table
*ft
,
1584 const struct mlx5_flow_spec
*spec
)
1586 struct rhlist_head
*tmp
, *list
;
1587 struct mlx5_flow_group
*g
;
1591 INIT_LIST_HEAD(&match_head
->list
);
1592 /* Collect all fgs which has a matching match_criteria */
1593 list
= rhltable_lookup(&ft
->fgs_hash
, spec
, rhash_fg
);
1594 /* RCU is atomic, we can't execute FW commands here */
1595 rhl_for_each_entry_rcu(g
, tmp
, list
, hash
) {
1596 struct match_list
*curr_match
;
1598 if (likely(list_empty(&match_head
->list
))) {
1599 if (!tree_get_node(&g
->node
))
1601 match_head
->first
.g
= g
;
1602 list_add_tail(&match_head
->first
.list
,
1607 curr_match
= kmalloc(sizeof(*curr_match
), GFP_ATOMIC
);
1609 free_match_list(match_head
);
1613 if (!tree_get_node(&g
->node
)) {
1618 list_add_tail(&curr_match
->list
, &match_head
->list
);
1625 static u64
matched_fgs_get_version(struct list_head
*match_head
)
1627 struct match_list
*iter
;
1630 list_for_each_entry(iter
, match_head
, list
)
1631 version
+= (u64
)atomic_read(&iter
->g
->node
.version
);
1635 static struct fs_fte
*
1636 lookup_fte_for_write_locked(struct mlx5_flow_group
*g
, const u32
*match_value
)
1638 struct fs_fte
*fte_tmp
;
1640 nested_down_write_ref_node(&g
->node
, FS_LOCK_PARENT
);
1642 fte_tmp
= rhashtable_lookup_fast(&g
->ftes_hash
, match_value
, rhash_fte
);
1643 if (!fte_tmp
|| !tree_get_node(&fte_tmp
->node
)) {
1648 if (!fte_tmp
->node
.active
) {
1649 tree_put_node(&fte_tmp
->node
, false);
1653 nested_down_write_ref_node(&fte_tmp
->node
, FS_LOCK_CHILD
);
1656 up_write_ref_node(&g
->node
, false);
1660 static struct fs_fte
*
1661 lookup_fte_for_read_locked(struct mlx5_flow_group
*g
, const u32
*match_value
)
1663 struct fs_fte
*fte_tmp
;
1665 if (!tree_get_node(&g
->node
))
1669 fte_tmp
= rhashtable_lookup(&g
->ftes_hash
, match_value
, rhash_fte
);
1670 if (!fte_tmp
|| !tree_get_node(&fte_tmp
->node
)) {
1677 if (!fte_tmp
->node
.active
) {
1678 tree_put_node(&fte_tmp
->node
, false);
1683 nested_down_write_ref_node(&fte_tmp
->node
, FS_LOCK_CHILD
);
1686 tree_put_node(&g
->node
, false);
1690 static struct fs_fte
*
1691 lookup_fte_locked(struct mlx5_flow_group
*g
, const u32
*match_value
, bool write
)
1694 return lookup_fte_for_write_locked(g
, match_value
);
1696 return lookup_fte_for_read_locked(g
, match_value
);
1699 static struct mlx5_flow_handle
*
1700 try_add_to_existing_fg(struct mlx5_flow_table
*ft
,
1701 struct list_head
*match_head
,
1702 const struct mlx5_flow_spec
*spec
,
1703 struct mlx5_flow_act
*flow_act
,
1704 struct mlx5_flow_destination
*dest
,
1708 struct mlx5_flow_steering
*steering
= get_steering(&ft
->node
);
1709 struct mlx5_flow_group
*g
;
1710 struct mlx5_flow_handle
*rule
;
1711 struct match_list
*iter
;
1712 bool take_write
= false;
1717 fte
= alloc_fte(ft
, spec
, flow_act
);
1719 return ERR_PTR(-ENOMEM
);
1721 search_again_locked
:
1722 version
= matched_fgs_get_version(match_head
);
1723 if (flow_act
->flags
& FLOW_ACT_NO_APPEND
)
1725 /* Try to find a fg that already contains a matching fte */
1726 list_for_each_entry(iter
, match_head
, list
) {
1727 struct fs_fte
*fte_tmp
;
1730 fte_tmp
= lookup_fte_locked(g
, spec
->match_value
, take_write
);
1733 rule
= add_rule_fg(g
, spec
, flow_act
, dest
, dest_num
, fte_tmp
);
1734 up_write_ref_node(&fte_tmp
->node
, false);
1735 tree_put_node(&fte_tmp
->node
, false);
1736 kmem_cache_free(steering
->ftes_cache
, fte
);
1741 /* No group with matching fte found, or we skipped the search.
1742 * Try to add a new fte to any matching fg.
1745 /* Check the ft version, for case that new flow group
1746 * was added while the fgs weren't locked
1748 if (atomic_read(&ft
->node
.version
) != ft_version
) {
1749 rule
= ERR_PTR(-EAGAIN
);
1753 /* Check the fgs version, for case the new FTE with the
1754 * same values was added while the fgs weren't locked
1756 if (version
!= matched_fgs_get_version(match_head
)) {
1758 goto search_again_locked
;
1761 list_for_each_entry(iter
, match_head
, list
) {
1764 if (!g
->node
.active
)
1767 nested_down_write_ref_node(&g
->node
, FS_LOCK_PARENT
);
1769 err
= insert_fte(g
, fte
);
1771 up_write_ref_node(&g
->node
, false);
1774 kmem_cache_free(steering
->ftes_cache
, fte
);
1775 return ERR_PTR(err
);
1778 nested_down_write_ref_node(&fte
->node
, FS_LOCK_CHILD
);
1779 up_write_ref_node(&g
->node
, false);
1780 rule
= add_rule_fg(g
, spec
, flow_act
, dest
, dest_num
, fte
);
1781 up_write_ref_node(&fte
->node
, false);
1782 tree_put_node(&fte
->node
, false);
1785 rule
= ERR_PTR(-ENOENT
);
1787 kmem_cache_free(steering
->ftes_cache
, fte
);
1791 static struct mlx5_flow_handle
*
1792 _mlx5_add_flow_rules(struct mlx5_flow_table
*ft
,
1793 const struct mlx5_flow_spec
*spec
,
1794 struct mlx5_flow_act
*flow_act
,
1795 struct mlx5_flow_destination
*dest
,
1799 struct mlx5_flow_steering
*steering
= get_steering(&ft
->node
);
1800 struct mlx5_flow_group
*g
;
1801 struct mlx5_flow_handle
*rule
;
1802 struct match_list_head match_head
;
1803 bool take_write
= false;
1809 if (!check_valid_spec(spec
))
1810 return ERR_PTR(-EINVAL
);
1812 for (i
= 0; i
< dest_num
; i
++) {
1813 if (!dest_is_valid(&dest
[i
], flow_act
->action
, ft
))
1814 return ERR_PTR(-EINVAL
);
1816 nested_down_read_ref_node(&ft
->node
, FS_LOCK_GRANDPARENT
);
1817 search_again_locked
:
1818 version
= atomic_read(&ft
->node
.version
);
1820 /* Collect all fgs which has a matching match_criteria */
1821 err
= build_match_list(&match_head
, ft
, spec
);
1824 up_write_ref_node(&ft
->node
, false);
1826 up_read_ref_node(&ft
->node
);
1827 return ERR_PTR(err
);
1831 up_read_ref_node(&ft
->node
);
1833 rule
= try_add_to_existing_fg(ft
, &match_head
.list
, spec
, flow_act
, dest
,
1835 free_match_list(&match_head
);
1836 if (!IS_ERR(rule
) ||
1837 (PTR_ERR(rule
) != -ENOENT
&& PTR_ERR(rule
) != -EAGAIN
)) {
1839 up_write_ref_node(&ft
->node
, false);
1844 nested_down_write_ref_node(&ft
->node
, FS_LOCK_GRANDPARENT
);
1848 if (PTR_ERR(rule
) == -EAGAIN
||
1849 version
!= atomic_read(&ft
->node
.version
))
1850 goto search_again_locked
;
1852 g
= alloc_auto_flow_group(ft
, spec
);
1855 up_write_ref_node(&ft
->node
, false);
1859 fte
= alloc_fte(ft
, spec
, flow_act
);
1861 up_write_ref_node(&ft
->node
, false);
1866 nested_down_write_ref_node(&g
->node
, FS_LOCK_PARENT
);
1867 up_write_ref_node(&ft
->node
, false);
1869 err
= create_auto_flow_group(ft
, g
);
1871 goto err_release_fg
;
1873 err
= insert_fte(g
, fte
);
1875 goto err_release_fg
;
1877 nested_down_write_ref_node(&fte
->node
, FS_LOCK_CHILD
);
1878 up_write_ref_node(&g
->node
, false);
1879 rule
= add_rule_fg(g
, spec
, flow_act
, dest
, dest_num
, fte
);
1880 up_write_ref_node(&fte
->node
, false);
1881 tree_put_node(&fte
->node
, false);
1882 tree_put_node(&g
->node
, false);
1886 up_write_ref_node(&g
->node
, false);
1887 kmem_cache_free(steering
->ftes_cache
, fte
);
1889 tree_put_node(&g
->node
, false);
1890 return ERR_PTR(err
);
1893 static bool fwd_next_prio_supported(struct mlx5_flow_table
*ft
)
1895 return ((ft
->type
== FS_FT_NIC_RX
) &&
1896 (MLX5_CAP_FLOWTABLE(get_dev(&ft
->node
), nic_rx_multi_path_tirs
)));
1899 struct mlx5_flow_handle
*
1900 mlx5_add_flow_rules(struct mlx5_flow_table
*ft
,
1901 const struct mlx5_flow_spec
*spec
,
1902 struct mlx5_flow_act
*flow_act
,
1903 struct mlx5_flow_destination
*dest
,
1906 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1907 struct mlx5_flow_destination gen_dest
= {};
1908 struct mlx5_flow_table
*next_ft
= NULL
;
1909 struct mlx5_flow_handle
*handle
= NULL
;
1910 u32 sw_action
= flow_act
->action
;
1911 struct fs_prio
*prio
;
1913 fs_get_obj(prio
, ft
->node
.parent
);
1914 if (flow_act
->action
== MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
) {
1915 if (!fwd_next_prio_supported(ft
))
1916 return ERR_PTR(-EOPNOTSUPP
);
1918 return ERR_PTR(-EINVAL
);
1919 mutex_lock(&root
->chain_lock
);
1920 next_ft
= find_next_chained_ft(prio
);
1922 gen_dest
.type
= MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE
;
1923 gen_dest
.ft
= next_ft
;
1926 flow_act
->action
= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST
;
1928 mutex_unlock(&root
->chain_lock
);
1929 return ERR_PTR(-EOPNOTSUPP
);
1933 handle
= _mlx5_add_flow_rules(ft
, spec
, flow_act
, dest
, num_dest
);
1935 if (sw_action
== MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
) {
1936 if (!IS_ERR_OR_NULL(handle
) &&
1937 (list_empty(&handle
->rule
[0]->next_ft
))) {
1938 mutex_lock(&next_ft
->lock
);
1939 list_add(&handle
->rule
[0]->next_ft
,
1940 &next_ft
->fwd_rules
);
1941 mutex_unlock(&next_ft
->lock
);
1942 handle
->rule
[0]->sw_action
= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO
;
1944 mutex_unlock(&root
->chain_lock
);
1948 EXPORT_SYMBOL(mlx5_add_flow_rules
);
1950 void mlx5_del_flow_rules(struct mlx5_flow_handle
*handle
)
1955 /* In order to consolidate the HW changes we lock the FTE for other
1956 * changes, and increase its refcount, in order not to perform the
1957 * "del" functions of the FTE. Will handle them here.
1958 * The removal of the rules is done under locked FTE.
1959 * After removing all the handle's rules, if there are remaining
1960 * rules, it means we just need to modify the FTE in FW, and
1961 * unlock/decrease the refcount we increased before.
1962 * Otherwise, it means the FTE should be deleted. First delete the
1963 * FTE in FW. Then, unlock the FTE, and proceed the tree_put_node of
1964 * the FTE, which will handle the last decrease of the refcount, as
1965 * well as required handling of its parent.
1967 fs_get_obj(fte
, handle
->rule
[0]->node
.parent
);
1968 down_write_ref_node(&fte
->node
, false);
1969 for (i
= handle
->num_rules
- 1; i
>= 0; i
--)
1970 tree_remove_node(&handle
->rule
[i
]->node
, true);
1971 if (fte
->modify_mask
&& fte
->dests_size
) {
1973 up_write_ref_node(&fte
->node
, false);
1975 del_hw_fte(&fte
->node
);
1976 up_write(&fte
->node
.lock
);
1977 tree_put_node(&fte
->node
, false);
1981 EXPORT_SYMBOL(mlx5_del_flow_rules
);
1983 /* Assuming prio->node.children(flow tables) is sorted by level */
1984 static struct mlx5_flow_table
*find_next_ft(struct mlx5_flow_table
*ft
)
1986 struct fs_prio
*prio
;
1988 fs_get_obj(prio
, ft
->node
.parent
);
1990 if (!list_is_last(&ft
->node
.list
, &prio
->node
.children
))
1991 return list_next_entry(ft
, node
.list
);
1992 return find_next_chained_ft(prio
);
1995 static int update_root_ft_destroy(struct mlx5_flow_table
*ft
)
1997 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
1998 struct mlx5_ft_underlay_qp
*uqp
;
1999 struct mlx5_flow_table
*new_root_ft
= NULL
;
2003 if (root
->root_ft
!= ft
)
2006 new_root_ft
= find_next_ft(ft
);
2008 root
->root_ft
= NULL
;
2012 if (list_empty(&root
->underlay_qpns
)) {
2013 /* Don't set any QPN (zero) in case QPN list is empty */
2015 err
= root
->cmds
->update_root_ft(root
, new_root_ft
,
2018 list_for_each_entry(uqp
, &root
->underlay_qpns
, list
) {
2020 err
= root
->cmds
->update_root_ft(root
,
2029 mlx5_core_warn(root
->dev
,
2030 "Update root flow table of id(%u) qpn(%d) failed\n",
2033 root
->root_ft
= new_root_ft
;
2038 /* Connect flow table from previous priority to
2039 * the next flow table.
2041 static int disconnect_flow_table(struct mlx5_flow_table
*ft
)
2043 struct mlx5_core_dev
*dev
= get_dev(&ft
->node
);
2044 struct mlx5_flow_table
*next_ft
;
2045 struct fs_prio
*prio
;
2048 err
= update_root_ft_destroy(ft
);
2052 fs_get_obj(prio
, ft
->node
.parent
);
2053 if (!(list_first_entry(&prio
->node
.children
,
2054 struct mlx5_flow_table
,
2058 next_ft
= find_next_chained_ft(prio
);
2059 err
= connect_fwd_rules(dev
, next_ft
, ft
);
2063 err
= connect_prev_fts(dev
, next_ft
, prio
);
2065 mlx5_core_warn(dev
, "Failed to disconnect flow table %d\n",
2070 int mlx5_destroy_flow_table(struct mlx5_flow_table
*ft
)
2072 struct mlx5_flow_root_namespace
*root
= find_root(&ft
->node
);
2075 mutex_lock(&root
->chain_lock
);
2076 err
= disconnect_flow_table(ft
);
2078 mutex_unlock(&root
->chain_lock
);
2081 if (tree_remove_node(&ft
->node
, false))
2082 mlx5_core_warn(get_dev(&ft
->node
), "Flow table %d wasn't destroyed, refcount > 1\n",
2084 mutex_unlock(&root
->chain_lock
);
2088 EXPORT_SYMBOL(mlx5_destroy_flow_table
);
2090 void mlx5_destroy_flow_group(struct mlx5_flow_group
*fg
)
2092 if (tree_remove_node(&fg
->node
, false))
2093 mlx5_core_warn(get_dev(&fg
->node
), "Flow group %d wasn't destroyed, refcount > 1\n",
2097 struct mlx5_flow_namespace
*mlx5_get_fdb_sub_ns(struct mlx5_core_dev
*dev
,
2100 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2102 if (!steering
|| !steering
->fdb_sub_ns
)
2105 return steering
->fdb_sub_ns
[n
];
2107 EXPORT_SYMBOL(mlx5_get_fdb_sub_ns
);
2109 struct mlx5_flow_namespace
*mlx5_get_flow_namespace(struct mlx5_core_dev
*dev
,
2110 enum mlx5_flow_namespace_type type
)
2112 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2113 struct mlx5_flow_root_namespace
*root_ns
;
2115 struct fs_prio
*fs_prio
;
2116 struct mlx5_flow_namespace
*ns
;
2122 case MLX5_FLOW_NAMESPACE_FDB
:
2123 if (steering
->fdb_root_ns
)
2124 return &steering
->fdb_root_ns
->ns
;
2126 case MLX5_FLOW_NAMESPACE_SNIFFER_RX
:
2127 if (steering
->sniffer_rx_root_ns
)
2128 return &steering
->sniffer_rx_root_ns
->ns
;
2130 case MLX5_FLOW_NAMESPACE_SNIFFER_TX
:
2131 if (steering
->sniffer_tx_root_ns
)
2132 return &steering
->sniffer_tx_root_ns
->ns
;
2138 if (type
== MLX5_FLOW_NAMESPACE_EGRESS
) {
2139 root_ns
= steering
->egress_root_ns
;
2140 } else if (type
== MLX5_FLOW_NAMESPACE_RDMA_RX
) {
2141 root_ns
= steering
->rdma_rx_root_ns
;
2142 prio
= RDMA_RX_BYPASS_PRIO
;
2143 } else if (type
== MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL
) {
2144 root_ns
= steering
->rdma_rx_root_ns
;
2145 prio
= RDMA_RX_KERNEL_PRIO
;
2146 } else { /* Must be NIC RX */
2147 root_ns
= steering
->root_ns
;
2154 fs_prio
= find_prio(&root_ns
->ns
, prio
);
2158 ns
= list_first_entry(&fs_prio
->node
.children
,
2164 EXPORT_SYMBOL(mlx5_get_flow_namespace
);
2166 struct mlx5_flow_namespace
*mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev
*dev
,
2167 enum mlx5_flow_namespace_type type
,
2170 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2172 if (!steering
|| vport
>= mlx5_eswitch_get_total_vports(dev
))
2176 case MLX5_FLOW_NAMESPACE_ESW_EGRESS
:
2177 if (steering
->esw_egress_root_ns
&&
2178 steering
->esw_egress_root_ns
[vport
])
2179 return &steering
->esw_egress_root_ns
[vport
]->ns
;
2182 case MLX5_FLOW_NAMESPACE_ESW_INGRESS
:
2183 if (steering
->esw_ingress_root_ns
&&
2184 steering
->esw_ingress_root_ns
[vport
])
2185 return &steering
->esw_ingress_root_ns
[vport
]->ns
;
2193 static struct fs_prio
*_fs_create_prio(struct mlx5_flow_namespace
*ns
,
2196 enum fs_node_type type
)
2198 struct fs_prio
*fs_prio
;
2200 fs_prio
= kzalloc(sizeof(*fs_prio
), GFP_KERNEL
);
2202 return ERR_PTR(-ENOMEM
);
2204 fs_prio
->node
.type
= type
;
2205 tree_init_node(&fs_prio
->node
, NULL
, del_sw_prio
);
2206 tree_add_node(&fs_prio
->node
, &ns
->node
);
2207 fs_prio
->num_levels
= num_levels
;
2208 fs_prio
->prio
= prio
;
2209 list_add_tail(&fs_prio
->node
.list
, &ns
->node
.children
);
2214 static struct fs_prio
*fs_create_prio_chained(struct mlx5_flow_namespace
*ns
,
2218 return _fs_create_prio(ns
, prio
, num_levels
, FS_TYPE_PRIO_CHAINS
);
2221 static struct fs_prio
*fs_create_prio(struct mlx5_flow_namespace
*ns
,
2222 unsigned int prio
, int num_levels
)
2224 return _fs_create_prio(ns
, prio
, num_levels
, FS_TYPE_PRIO
);
2227 static struct mlx5_flow_namespace
*fs_init_namespace(struct mlx5_flow_namespace
2230 ns
->node
.type
= FS_TYPE_NAMESPACE
;
2235 static struct mlx5_flow_namespace
*fs_create_namespace(struct fs_prio
*prio
,
2238 struct mlx5_flow_namespace
*ns
;
2240 ns
= kzalloc(sizeof(*ns
), GFP_KERNEL
);
2242 return ERR_PTR(-ENOMEM
);
2244 fs_init_namespace(ns
);
2245 ns
->def_miss_action
= def_miss_act
;
2246 tree_init_node(&ns
->node
, NULL
, del_sw_ns
);
2247 tree_add_node(&ns
->node
, &prio
->node
);
2248 list_add_tail(&ns
->node
.list
, &prio
->node
.children
);
2253 static int create_leaf_prios(struct mlx5_flow_namespace
*ns
, int prio
,
2254 struct init_tree_node
*prio_metadata
)
2256 struct fs_prio
*fs_prio
;
2259 for (i
= 0; i
< prio_metadata
->num_leaf_prios
; i
++) {
2260 fs_prio
= fs_create_prio(ns
, prio
++, prio_metadata
->num_levels
);
2261 if (IS_ERR(fs_prio
))
2262 return PTR_ERR(fs_prio
);
2267 #define FLOW_TABLE_BIT_SZ 1
2268 #define GET_FLOW_TABLE_CAP(dev, offset) \
2269 ((be32_to_cpu(*((__be32 *)(dev->caps.hca_cur[MLX5_CAP_FLOW_TABLE]) + \
2271 (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
2272 static bool has_required_caps(struct mlx5_core_dev
*dev
, struct node_caps
*caps
)
2276 for (i
= 0; i
< caps
->arr_sz
; i
++) {
2277 if (!GET_FLOW_TABLE_CAP(dev
, caps
->caps
[i
]))
2283 static int init_root_tree_recursive(struct mlx5_flow_steering
*steering
,
2284 struct init_tree_node
*init_node
,
2285 struct fs_node
*fs_parent_node
,
2286 struct init_tree_node
*init_parent_node
,
2289 int max_ft_level
= MLX5_CAP_FLOWTABLE(steering
->dev
,
2290 flow_table_properties_nic_receive
.
2292 struct mlx5_flow_namespace
*fs_ns
;
2293 struct fs_prio
*fs_prio
;
2294 struct fs_node
*base
;
2298 if (init_node
->type
== FS_TYPE_PRIO
) {
2299 if ((init_node
->min_ft_level
> max_ft_level
) ||
2300 !has_required_caps(steering
->dev
, &init_node
->caps
))
2303 fs_get_obj(fs_ns
, fs_parent_node
);
2304 if (init_node
->num_leaf_prios
)
2305 return create_leaf_prios(fs_ns
, prio
, init_node
);
2306 fs_prio
= fs_create_prio(fs_ns
, prio
, init_node
->num_levels
);
2307 if (IS_ERR(fs_prio
))
2308 return PTR_ERR(fs_prio
);
2309 base
= &fs_prio
->node
;
2310 } else if (init_node
->type
== FS_TYPE_NAMESPACE
) {
2311 fs_get_obj(fs_prio
, fs_parent_node
);
2312 fs_ns
= fs_create_namespace(fs_prio
, init_node
->def_miss_action
);
2314 return PTR_ERR(fs_ns
);
2315 base
= &fs_ns
->node
;
2320 for (i
= 0; i
< init_node
->ar_size
; i
++) {
2321 err
= init_root_tree_recursive(steering
, &init_node
->children
[i
],
2322 base
, init_node
, prio
);
2325 if (init_node
->children
[i
].type
== FS_TYPE_PRIO
&&
2326 init_node
->children
[i
].num_leaf_prios
) {
2327 prio
+= init_node
->children
[i
].num_leaf_prios
;
2334 static int init_root_tree(struct mlx5_flow_steering
*steering
,
2335 struct init_tree_node
*init_node
,
2336 struct fs_node
*fs_parent_node
)
2339 struct mlx5_flow_namespace
*fs_ns
;
2342 fs_get_obj(fs_ns
, fs_parent_node
);
2343 for (i
= 0; i
< init_node
->ar_size
; i
++) {
2344 err
= init_root_tree_recursive(steering
, &init_node
->children
[i
],
2353 static struct mlx5_flow_root_namespace
2354 *create_root_ns(struct mlx5_flow_steering
*steering
,
2355 enum fs_flow_table_type table_type
)
2357 const struct mlx5_flow_cmds
*cmds
= mlx5_fs_cmd_get_default(table_type
);
2358 struct mlx5_flow_root_namespace
*root_ns
;
2359 struct mlx5_flow_namespace
*ns
;
2361 if (mlx5_accel_ipsec_device_caps(steering
->dev
) & MLX5_ACCEL_IPSEC_CAP_DEVICE
&&
2362 (table_type
== FS_FT_NIC_RX
|| table_type
== FS_FT_NIC_TX
))
2363 cmds
= mlx5_fs_cmd_get_default_ipsec_fpga_cmds(table_type
);
2365 /* Create the root namespace */
2366 root_ns
= kzalloc(sizeof(*root_ns
), GFP_KERNEL
);
2370 root_ns
->dev
= steering
->dev
;
2371 root_ns
->table_type
= table_type
;
2372 root_ns
->cmds
= cmds
;
2374 INIT_LIST_HEAD(&root_ns
->underlay_qpns
);
2377 fs_init_namespace(ns
);
2378 mutex_init(&root_ns
->chain_lock
);
2379 tree_init_node(&ns
->node
, NULL
, NULL
);
2380 tree_add_node(&ns
->node
, NULL
);
2385 static void set_prio_attrs_in_prio(struct fs_prio
*prio
, int acc_level
);
2387 static int set_prio_attrs_in_ns(struct mlx5_flow_namespace
*ns
, int acc_level
)
2389 struct fs_prio
*prio
;
2391 fs_for_each_prio(prio
, ns
) {
2392 /* This updates prio start_level and num_levels */
2393 set_prio_attrs_in_prio(prio
, acc_level
);
2394 acc_level
+= prio
->num_levels
;
2399 static void set_prio_attrs_in_prio(struct fs_prio
*prio
, int acc_level
)
2401 struct mlx5_flow_namespace
*ns
;
2402 int acc_level_ns
= acc_level
;
2404 prio
->start_level
= acc_level
;
2405 fs_for_each_ns(ns
, prio
) {
2406 /* This updates start_level and num_levels of ns's priority descendants */
2407 acc_level_ns
= set_prio_attrs_in_ns(ns
, acc_level
);
2409 /* If this a prio with chains, and we can jump from one chain
2410 * (namepsace) to another, so we accumulate the levels
2412 if (prio
->node
.type
== FS_TYPE_PRIO_CHAINS
)
2413 acc_level
= acc_level_ns
;
2416 if (!prio
->num_levels
)
2417 prio
->num_levels
= acc_level_ns
- prio
->start_level
;
2418 WARN_ON(prio
->num_levels
< acc_level_ns
- prio
->start_level
);
2421 static void set_prio_attrs(struct mlx5_flow_root_namespace
*root_ns
)
2423 struct mlx5_flow_namespace
*ns
= &root_ns
->ns
;
2424 struct fs_prio
*prio
;
2425 int start_level
= 0;
2427 fs_for_each_prio(prio
, ns
) {
2428 set_prio_attrs_in_prio(prio
, start_level
);
2429 start_level
+= prio
->num_levels
;
2433 #define ANCHOR_PRIO 0
2434 #define ANCHOR_SIZE 1
2435 #define ANCHOR_LEVEL 0
2436 static int create_anchor_flow_table(struct mlx5_flow_steering
*steering
)
2438 struct mlx5_flow_namespace
*ns
= NULL
;
2439 struct mlx5_flow_table_attr ft_attr
= {};
2440 struct mlx5_flow_table
*ft
;
2442 ns
= mlx5_get_flow_namespace(steering
->dev
, MLX5_FLOW_NAMESPACE_ANCHOR
);
2446 ft_attr
.max_fte
= ANCHOR_SIZE
;
2447 ft_attr
.level
= ANCHOR_LEVEL
;
2448 ft_attr
.prio
= ANCHOR_PRIO
;
2450 ft
= mlx5_create_flow_table(ns
, &ft_attr
);
2452 mlx5_core_err(steering
->dev
, "Failed to create last anchor flow table");
2458 static int init_root_ns(struct mlx5_flow_steering
*steering
)
2462 steering
->root_ns
= create_root_ns(steering
, FS_FT_NIC_RX
);
2463 if (!steering
->root_ns
)
2466 err
= init_root_tree(steering
, &root_fs
, &steering
->root_ns
->ns
.node
);
2470 set_prio_attrs(steering
->root_ns
);
2471 err
= create_anchor_flow_table(steering
);
2478 cleanup_root_ns(steering
->root_ns
);
2479 steering
->root_ns
= NULL
;
2483 static void clean_tree(struct fs_node
*node
)
2486 struct fs_node
*iter
;
2487 struct fs_node
*temp
;
2489 tree_get_node(node
);
2490 list_for_each_entry_safe(iter
, temp
, &node
->children
, list
)
2492 tree_put_node(node
, false);
2493 tree_remove_node(node
, false);
2497 static void cleanup_root_ns(struct mlx5_flow_root_namespace
*root_ns
)
2502 clean_tree(&root_ns
->ns
.node
);
2505 static void cleanup_egress_acls_root_ns(struct mlx5_core_dev
*dev
)
2507 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2510 if (!steering
->esw_egress_root_ns
)
2513 for (i
= 0; i
< mlx5_eswitch_get_total_vports(dev
); i
++)
2514 cleanup_root_ns(steering
->esw_egress_root_ns
[i
]);
2516 kfree(steering
->esw_egress_root_ns
);
2517 steering
->esw_egress_root_ns
= NULL
;
2520 static void cleanup_ingress_acls_root_ns(struct mlx5_core_dev
*dev
)
2522 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2525 if (!steering
->esw_ingress_root_ns
)
2528 for (i
= 0; i
< mlx5_eswitch_get_total_vports(dev
); i
++)
2529 cleanup_root_ns(steering
->esw_ingress_root_ns
[i
]);
2531 kfree(steering
->esw_ingress_root_ns
);
2532 steering
->esw_ingress_root_ns
= NULL
;
2535 void mlx5_cleanup_fs(struct mlx5_core_dev
*dev
)
2537 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2539 cleanup_root_ns(steering
->root_ns
);
2540 cleanup_egress_acls_root_ns(dev
);
2541 cleanup_ingress_acls_root_ns(dev
);
2542 cleanup_root_ns(steering
->fdb_root_ns
);
2543 steering
->fdb_root_ns
= NULL
;
2544 kfree(steering
->fdb_sub_ns
);
2545 steering
->fdb_sub_ns
= NULL
;
2546 cleanup_root_ns(steering
->sniffer_rx_root_ns
);
2547 cleanup_root_ns(steering
->sniffer_tx_root_ns
);
2548 cleanup_root_ns(steering
->rdma_rx_root_ns
);
2549 cleanup_root_ns(steering
->egress_root_ns
);
2550 mlx5_cleanup_fc_stats(dev
);
2551 kmem_cache_destroy(steering
->ftes_cache
);
2552 kmem_cache_destroy(steering
->fgs_cache
);
2556 static int init_sniffer_tx_root_ns(struct mlx5_flow_steering
*steering
)
2558 struct fs_prio
*prio
;
2560 steering
->sniffer_tx_root_ns
= create_root_ns(steering
, FS_FT_SNIFFER_TX
);
2561 if (!steering
->sniffer_tx_root_ns
)
2564 /* Create single prio */
2565 prio
= fs_create_prio(&steering
->sniffer_tx_root_ns
->ns
, 0, 1);
2566 return PTR_ERR_OR_ZERO(prio
);
2569 static int init_sniffer_rx_root_ns(struct mlx5_flow_steering
*steering
)
2571 struct fs_prio
*prio
;
2573 steering
->sniffer_rx_root_ns
= create_root_ns(steering
, FS_FT_SNIFFER_RX
);
2574 if (!steering
->sniffer_rx_root_ns
)
2577 /* Create single prio */
2578 prio
= fs_create_prio(&steering
->sniffer_rx_root_ns
->ns
, 0, 1);
2579 return PTR_ERR_OR_ZERO(prio
);
2582 static int init_rdma_rx_root_ns(struct mlx5_flow_steering
*steering
)
2586 steering
->rdma_rx_root_ns
= create_root_ns(steering
, FS_FT_RDMA_RX
);
2587 if (!steering
->rdma_rx_root_ns
)
2590 err
= init_root_tree(steering
, &rdma_rx_root_fs
,
2591 &steering
->rdma_rx_root_ns
->ns
.node
);
2595 set_prio_attrs(steering
->rdma_rx_root_ns
);
2600 cleanup_root_ns(steering
->rdma_rx_root_ns
);
2601 steering
->rdma_rx_root_ns
= NULL
;
2605 /* FT and tc chains are stored in the same array so we can re-use the
2606 * mlx5_get_fdb_sub_ns() and tc api for FT chains.
2607 * When creating a new ns for each chain store it in the first available slot.
2608 * Assume tc chains are created and stored first and only then the FT chain.
2610 static void store_fdb_sub_ns_prio_chain(struct mlx5_flow_steering
*steering
,
2611 struct mlx5_flow_namespace
*ns
)
2615 while (steering
->fdb_sub_ns
[chain
])
2618 steering
->fdb_sub_ns
[chain
] = ns
;
2621 static int create_fdb_sub_ns_prio_chain(struct mlx5_flow_steering
*steering
,
2622 struct fs_prio
*maj_prio
)
2624 struct mlx5_flow_namespace
*ns
;
2625 struct fs_prio
*min_prio
;
2628 ns
= fs_create_namespace(maj_prio
, MLX5_FLOW_TABLE_MISS_ACTION_DEF
);
2632 for (prio
= 0; prio
< FDB_TC_MAX_PRIO
; prio
++) {
2633 min_prio
= fs_create_prio(ns
, prio
, FDB_TC_LEVELS_PER_PRIO
);
2634 if (IS_ERR(min_prio
))
2635 return PTR_ERR(min_prio
);
2638 store_fdb_sub_ns_prio_chain(steering
, ns
);
2643 static int create_fdb_chains(struct mlx5_flow_steering
*steering
,
2647 struct fs_prio
*maj_prio
;
2652 levels
= FDB_TC_LEVELS_PER_PRIO
* FDB_TC_MAX_PRIO
* chains
;
2653 maj_prio
= fs_create_prio_chained(&steering
->fdb_root_ns
->ns
,
2656 if (IS_ERR(maj_prio
))
2657 return PTR_ERR(maj_prio
);
2659 for (chain
= 0; chain
< chains
; chain
++) {
2660 err
= create_fdb_sub_ns_prio_chain(steering
, maj_prio
);
2668 static int create_fdb_fast_path(struct mlx5_flow_steering
*steering
)
2672 steering
->fdb_sub_ns
= kcalloc(FDB_NUM_CHAINS
,
2673 sizeof(*steering
->fdb_sub_ns
),
2675 if (!steering
->fdb_sub_ns
)
2678 err
= create_fdb_chains(steering
, FDB_TC_OFFLOAD
, FDB_TC_MAX_CHAIN
+ 1);
2682 err
= create_fdb_chains(steering
, FDB_FT_OFFLOAD
, 1);
2689 static int init_fdb_root_ns(struct mlx5_flow_steering
*steering
)
2691 struct fs_prio
*maj_prio
;
2694 steering
->fdb_root_ns
= create_root_ns(steering
, FS_FT_FDB
);
2695 if (!steering
->fdb_root_ns
)
2698 maj_prio
= fs_create_prio(&steering
->fdb_root_ns
->ns
, FDB_BYPASS_PATH
,
2700 if (IS_ERR(maj_prio
)) {
2701 err
= PTR_ERR(maj_prio
);
2704 err
= create_fdb_fast_path(steering
);
2708 maj_prio
= fs_create_prio(&steering
->fdb_root_ns
->ns
, FDB_SLOW_PATH
, 1);
2709 if (IS_ERR(maj_prio
)) {
2710 err
= PTR_ERR(maj_prio
);
2714 set_prio_attrs(steering
->fdb_root_ns
);
2718 cleanup_root_ns(steering
->fdb_root_ns
);
2719 kfree(steering
->fdb_sub_ns
);
2720 steering
->fdb_sub_ns
= NULL
;
2721 steering
->fdb_root_ns
= NULL
;
2725 static int init_egress_acl_root_ns(struct mlx5_flow_steering
*steering
, int vport
)
2727 struct fs_prio
*prio
;
2729 steering
->esw_egress_root_ns
[vport
] = create_root_ns(steering
, FS_FT_ESW_EGRESS_ACL
);
2730 if (!steering
->esw_egress_root_ns
[vport
])
2734 prio
= fs_create_prio(&steering
->esw_egress_root_ns
[vport
]->ns
, 0, 1);
2735 return PTR_ERR_OR_ZERO(prio
);
2738 static int init_ingress_acl_root_ns(struct mlx5_flow_steering
*steering
, int vport
)
2740 struct fs_prio
*prio
;
2742 steering
->esw_ingress_root_ns
[vport
] = create_root_ns(steering
, FS_FT_ESW_INGRESS_ACL
);
2743 if (!steering
->esw_ingress_root_ns
[vport
])
2747 prio
= fs_create_prio(&steering
->esw_ingress_root_ns
[vport
]->ns
, 0, 1);
2748 return PTR_ERR_OR_ZERO(prio
);
2751 static int init_egress_acls_root_ns(struct mlx5_core_dev
*dev
)
2753 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2754 int total_vports
= mlx5_eswitch_get_total_vports(dev
);
2758 steering
->esw_egress_root_ns
=
2759 kcalloc(total_vports
,
2760 sizeof(*steering
->esw_egress_root_ns
),
2762 if (!steering
->esw_egress_root_ns
)
2765 for (i
= 0; i
< total_vports
; i
++) {
2766 err
= init_egress_acl_root_ns(steering
, i
);
2768 goto cleanup_root_ns
;
2774 for (i
--; i
>= 0; i
--)
2775 cleanup_root_ns(steering
->esw_egress_root_ns
[i
]);
2776 kfree(steering
->esw_egress_root_ns
);
2777 steering
->esw_egress_root_ns
= NULL
;
2781 static int init_ingress_acls_root_ns(struct mlx5_core_dev
*dev
)
2783 struct mlx5_flow_steering
*steering
= dev
->priv
.steering
;
2784 int total_vports
= mlx5_eswitch_get_total_vports(dev
);
2788 steering
->esw_ingress_root_ns
=
2789 kcalloc(total_vports
,
2790 sizeof(*steering
->esw_ingress_root_ns
),
2792 if (!steering
->esw_ingress_root_ns
)
2795 for (i
= 0; i
< total_vports
; i
++) {
2796 err
= init_ingress_acl_root_ns(steering
, i
);
2798 goto cleanup_root_ns
;
2804 for (i
--; i
>= 0; i
--)
2805 cleanup_root_ns(steering
->esw_ingress_root_ns
[i
]);
2806 kfree(steering
->esw_ingress_root_ns
);
2807 steering
->esw_ingress_root_ns
= NULL
;
2811 static int init_egress_root_ns(struct mlx5_flow_steering
*steering
)
2815 steering
->egress_root_ns
= create_root_ns(steering
,
2817 if (!steering
->egress_root_ns
)
2820 err
= init_root_tree(steering
, &egress_root_fs
,
2821 &steering
->egress_root_ns
->ns
.node
);
2824 set_prio_attrs(steering
->egress_root_ns
);
2827 cleanup_root_ns(steering
->egress_root_ns
);
2828 steering
->egress_root_ns
= NULL
;
2832 int mlx5_init_fs(struct mlx5_core_dev
*dev
)
2834 struct mlx5_flow_steering
*steering
;
2837 err
= mlx5_init_fc_stats(dev
);
2841 steering
= kzalloc(sizeof(*steering
), GFP_KERNEL
);
2844 steering
->dev
= dev
;
2845 dev
->priv
.steering
= steering
;
2847 steering
->fgs_cache
= kmem_cache_create("mlx5_fs_fgs",
2848 sizeof(struct mlx5_flow_group
), 0,
2850 steering
->ftes_cache
= kmem_cache_create("mlx5_fs_ftes", sizeof(struct fs_fte
), 0,
2852 if (!steering
->ftes_cache
|| !steering
->fgs_cache
) {
2857 if ((((MLX5_CAP_GEN(dev
, port_type
) == MLX5_CAP_PORT_TYPE_ETH
) &&
2858 (MLX5_CAP_GEN(dev
, nic_flow_table
))) ||
2859 ((MLX5_CAP_GEN(dev
, port_type
) == MLX5_CAP_PORT_TYPE_IB
) &&
2860 MLX5_CAP_GEN(dev
, ipoib_enhanced_offloads
))) &&
2861 MLX5_CAP_FLOWTABLE_NIC_RX(dev
, ft_support
)) {
2862 err
= init_root_ns(steering
);
2867 if (MLX5_ESWITCH_MANAGER(dev
)) {
2868 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev
, ft_support
)) {
2869 err
= init_fdb_root_ns(steering
);
2873 if (MLX5_CAP_ESW_EGRESS_ACL(dev
, ft_support
)) {
2874 err
= init_egress_acls_root_ns(dev
);
2878 if (MLX5_CAP_ESW_INGRESS_ACL(dev
, ft_support
)) {
2879 err
= init_ingress_acls_root_ns(dev
);
2885 if (MLX5_CAP_FLOWTABLE_SNIFFER_RX(dev
, ft_support
)) {
2886 err
= init_sniffer_rx_root_ns(steering
);
2891 if (MLX5_CAP_FLOWTABLE_SNIFFER_TX(dev
, ft_support
)) {
2892 err
= init_sniffer_tx_root_ns(steering
);
2897 if (MLX5_CAP_FLOWTABLE_RDMA_RX(dev
, ft_support
) &&
2898 MLX5_CAP_FLOWTABLE_RDMA_RX(dev
, table_miss_action_domain
)) {
2899 err
= init_rdma_rx_root_ns(steering
);
2904 if (MLX5_IPSEC_DEV(dev
) || MLX5_CAP_FLOWTABLE_NIC_TX(dev
, ft_support
)) {
2905 err
= init_egress_root_ns(steering
);
2912 mlx5_cleanup_fs(dev
);
2916 int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev
*dev
, u32 underlay_qpn
)
2918 struct mlx5_flow_root_namespace
*root
= dev
->priv
.steering
->root_ns
;
2919 struct mlx5_ft_underlay_qp
*new_uqp
;
2922 new_uqp
= kzalloc(sizeof(*new_uqp
), GFP_KERNEL
);
2926 mutex_lock(&root
->chain_lock
);
2928 if (!root
->root_ft
) {
2930 goto update_ft_fail
;
2933 err
= root
->cmds
->update_root_ft(root
, root
->root_ft
, underlay_qpn
,
2936 mlx5_core_warn(dev
, "Failed adding underlay QPN (%u) to root FT err(%d)\n",
2938 goto update_ft_fail
;
2941 new_uqp
->qpn
= underlay_qpn
;
2942 list_add_tail(&new_uqp
->list
, &root
->underlay_qpns
);
2944 mutex_unlock(&root
->chain_lock
);
2949 mutex_unlock(&root
->chain_lock
);
2953 EXPORT_SYMBOL(mlx5_fs_add_rx_underlay_qpn
);
2955 int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev
*dev
, u32 underlay_qpn
)
2957 struct mlx5_flow_root_namespace
*root
= dev
->priv
.steering
->root_ns
;
2958 struct mlx5_ft_underlay_qp
*uqp
;
2962 mutex_lock(&root
->chain_lock
);
2963 list_for_each_entry(uqp
, &root
->underlay_qpns
, list
) {
2964 if (uqp
->qpn
== underlay_qpn
) {
2971 mlx5_core_warn(dev
, "Failed finding underlay qp (%u) in qpn list\n",
2977 err
= root
->cmds
->update_root_ft(root
, root
->root_ft
, underlay_qpn
,
2980 mlx5_core_warn(dev
, "Failed removing underlay QPN (%u) from root FT err(%d)\n",
2983 list_del(&uqp
->list
);
2984 mutex_unlock(&root
->chain_lock
);
2990 mutex_unlock(&root
->chain_lock
);
2993 EXPORT_SYMBOL(mlx5_fs_remove_rx_underlay_qpn
);
2995 static struct mlx5_flow_root_namespace
2996 *get_root_namespace(struct mlx5_core_dev
*dev
, enum mlx5_flow_namespace_type ns_type
)
2998 struct mlx5_flow_namespace
*ns
;
3000 if (ns_type
== MLX5_FLOW_NAMESPACE_ESW_EGRESS
||
3001 ns_type
== MLX5_FLOW_NAMESPACE_ESW_INGRESS
)
3002 ns
= mlx5_get_flow_vport_acl_namespace(dev
, ns_type
, 0);
3004 ns
= mlx5_get_flow_namespace(dev
, ns_type
);
3008 return find_root(&ns
->node
);
3011 struct mlx5_modify_hdr
*mlx5_modify_header_alloc(struct mlx5_core_dev
*dev
,
3012 u8 ns_type
, u8 num_actions
,
3013 void *modify_actions
)
3015 struct mlx5_flow_root_namespace
*root
;
3016 struct mlx5_modify_hdr
*modify_hdr
;
3019 root
= get_root_namespace(dev
, ns_type
);
3021 return ERR_PTR(-EOPNOTSUPP
);
3023 modify_hdr
= kzalloc(sizeof(*modify_hdr
), GFP_KERNEL
);
3025 return ERR_PTR(-ENOMEM
);
3027 modify_hdr
->ns_type
= ns_type
;
3028 err
= root
->cmds
->modify_header_alloc(root
, ns_type
, num_actions
,
3029 modify_actions
, modify_hdr
);
3032 return ERR_PTR(err
);
3037 EXPORT_SYMBOL(mlx5_modify_header_alloc
);
3039 void mlx5_modify_header_dealloc(struct mlx5_core_dev
*dev
,
3040 struct mlx5_modify_hdr
*modify_hdr
)
3042 struct mlx5_flow_root_namespace
*root
;
3044 root
= get_root_namespace(dev
, modify_hdr
->ns_type
);
3047 root
->cmds
->modify_header_dealloc(root
, modify_hdr
);
3050 EXPORT_SYMBOL(mlx5_modify_header_dealloc
);
3052 struct mlx5_pkt_reformat
*mlx5_packet_reformat_alloc(struct mlx5_core_dev
*dev
,
3055 void *reformat_data
,
3056 enum mlx5_flow_namespace_type ns_type
)
3058 struct mlx5_pkt_reformat
*pkt_reformat
;
3059 struct mlx5_flow_root_namespace
*root
;
3062 root
= get_root_namespace(dev
, ns_type
);
3064 return ERR_PTR(-EOPNOTSUPP
);
3066 pkt_reformat
= kzalloc(sizeof(*pkt_reformat
), GFP_KERNEL
);
3068 return ERR_PTR(-ENOMEM
);
3070 pkt_reformat
->ns_type
= ns_type
;
3071 pkt_reformat
->reformat_type
= reformat_type
;
3072 err
= root
->cmds
->packet_reformat_alloc(root
, reformat_type
, size
,
3073 reformat_data
, ns_type
,
3076 kfree(pkt_reformat
);
3077 return ERR_PTR(err
);
3080 return pkt_reformat
;
3082 EXPORT_SYMBOL(mlx5_packet_reformat_alloc
);
3084 void mlx5_packet_reformat_dealloc(struct mlx5_core_dev
*dev
,
3085 struct mlx5_pkt_reformat
*pkt_reformat
)
3087 struct mlx5_flow_root_namespace
*root
;
3089 root
= get_root_namespace(dev
, pkt_reformat
->ns_type
);
3092 root
->cmds
->packet_reformat_dealloc(root
, pkt_reformat
);
3093 kfree(pkt_reformat
);
3095 EXPORT_SYMBOL(mlx5_packet_reformat_dealloc
);
3097 int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace
*ns
,
3098 struct mlx5_flow_root_namespace
*peer_ns
)
3100 if (peer_ns
&& ns
->mode
!= peer_ns
->mode
) {
3101 mlx5_core_err(ns
->dev
,
3102 "Can't peer namespace of different steering mode\n");
3106 return ns
->cmds
->set_peer(ns
, peer_ns
);
3109 /* This function should be called only at init stage of the namespace.
3110 * It is not safe to call this function while steering operations
3111 * are executed in the namespace.
3113 int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace
*ns
,
3114 enum mlx5_flow_steering_mode mode
)
3116 struct mlx5_flow_root_namespace
*root
;
3117 const struct mlx5_flow_cmds
*cmds
;
3120 root
= find_root(&ns
->node
);
3121 if (&root
->ns
!= ns
)
3122 /* Can't set cmds to non root namespace */
3125 if (root
->table_type
!= FS_FT_FDB
)
3128 if (root
->mode
== mode
)
3131 if (mode
== MLX5_FLOW_STEERING_MODE_SMFS
)
3132 cmds
= mlx5_fs_cmd_get_dr_cmds();
3134 cmds
= mlx5_fs_cmd_get_fw_cmds();
3138 err
= cmds
->create_ns(root
);
3140 mlx5_core_err(root
->dev
, "Failed to create flow namespace (%d)\n",
3145 root
->cmds
->destroy_ns(root
);