1 /* Broadcom NetXtreme-C/E network driver.
3 * Copyright (c) 2017 Broadcom Limited
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
10 #include <linux/netdevice.h>
11 #include <linux/inetdevice.h>
12 #include <linux/if_vlan.h>
13 #include <net/flow_dissector.h>
14 #include <net/pkt_cls.h>
15 #include <net/tc_act/tc_gact.h>
16 #include <net/tc_act/tc_skbedit.h>
17 #include <net/tc_act/tc_mirred.h>
18 #include <net/tc_act/tc_vlan.h>
22 #include "bnxt_sriov.h"
26 #ifdef CONFIG_BNXT_FLOWER_OFFLOAD
28 #define BNXT_FID_INVALID 0xffff
29 #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT))
31 /* Return the dst fid of the func for flow forwarding
32 * For PFs: src_fid is the fid of the PF
33 * For VF-reps: src_fid the fid of the VF
35 static u16
bnxt_flow_get_dst_fid(struct bnxt
*pf_bp
, struct net_device
*dev
)
39 /* check if dev belongs to the same switch */
40 if (!switchdev_port_same_parent_id(pf_bp
->dev
, dev
)) {
41 netdev_info(pf_bp
->dev
, "dev(ifindex=%d) not on same switch",
43 return BNXT_FID_INVALID
;
46 /* Is dev a VF-rep? */
47 if (dev
!= pf_bp
->dev
)
48 return bnxt_vf_rep_get_fid(dev
);
50 bp
= netdev_priv(dev
);
54 static int bnxt_tc_parse_redir(struct bnxt
*bp
,
55 struct bnxt_tc_actions
*actions
,
56 const struct tc_action
*tc_act
)
58 int ifindex
= tcf_mirred_ifindex(tc_act
);
59 struct net_device
*dev
;
62 dev
= __dev_get_by_index(dev_net(bp
->dev
), ifindex
);
64 netdev_info(bp
->dev
, "no dev for ifindex=%d", ifindex
);
68 /* find the FID from dev */
69 dst_fid
= bnxt_flow_get_dst_fid(bp
, dev
);
70 if (dst_fid
== BNXT_FID_INVALID
) {
71 netdev_info(bp
->dev
, "can't get fid for ifindex=%d", ifindex
);
75 actions
->flags
|= BNXT_TC_ACTION_FLAG_FWD
;
76 actions
->dst_fid
= dst_fid
;
77 actions
->dst_dev
= dev
;
81 static void bnxt_tc_parse_vlan(struct bnxt
*bp
,
82 struct bnxt_tc_actions
*actions
,
83 const struct tc_action
*tc_act
)
85 if (tcf_vlan_action(tc_act
) == TCA_VLAN_ACT_POP
) {
86 actions
->flags
|= BNXT_TC_ACTION_FLAG_POP_VLAN
;
87 } else if (tcf_vlan_action(tc_act
) == TCA_VLAN_ACT_PUSH
) {
88 actions
->flags
|= BNXT_TC_ACTION_FLAG_PUSH_VLAN
;
89 actions
->push_vlan_tci
= htons(tcf_vlan_push_vid(tc_act
));
90 actions
->push_vlan_tpid
= tcf_vlan_push_proto(tc_act
);
94 static int bnxt_tc_parse_actions(struct bnxt
*bp
,
95 struct bnxt_tc_actions
*actions
,
96 struct tcf_exts
*tc_exts
)
98 const struct tc_action
*tc_act
;
99 LIST_HEAD(tc_actions
);
102 if (!tcf_exts_has_actions(tc_exts
)) {
103 netdev_info(bp
->dev
, "no actions");
107 tcf_exts_to_list(tc_exts
, &tc_actions
);
108 list_for_each_entry(tc_act
, &tc_actions
, list
) {
110 if (is_tcf_gact_shot(tc_act
)) {
111 actions
->flags
|= BNXT_TC_ACTION_FLAG_DROP
;
112 return 0; /* don't bother with other actions */
115 /* Redirect action */
116 if (is_tcf_mirred_egress_redirect(tc_act
)) {
117 rc
= bnxt_tc_parse_redir(bp
, actions
, tc_act
);
124 if (is_tcf_vlan(tc_act
)) {
125 bnxt_tc_parse_vlan(bp
, actions
, tc_act
);
133 #define GET_KEY(flow_cmd, key_type) \
134 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
136 #define GET_MASK(flow_cmd, key_type) \
137 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
140 static int bnxt_tc_parse_flow(struct bnxt
*bp
,
141 struct tc_cls_flower_offload
*tc_flow_cmd
,
142 struct bnxt_tc_flow
*flow
)
144 struct flow_dissector
*dissector
= tc_flow_cmd
->dissector
;
147 /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */
148 if ((dissector
->used_keys
& BIT(FLOW_DISSECTOR_KEY_CONTROL
)) == 0 ||
149 (dissector
->used_keys
& BIT(FLOW_DISSECTOR_KEY_BASIC
)) == 0) {
150 netdev_info(bp
->dev
, "cannot form TC key: used_keys = 0x%x",
151 dissector
->used_keys
);
155 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_CONTROL
)) {
156 struct flow_dissector_key_control
*key
=
157 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_CONTROL
);
159 addr_type
= key
->addr_type
;
162 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_BASIC
)) {
163 struct flow_dissector_key_basic
*key
=
164 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_BASIC
);
165 struct flow_dissector_key_basic
*mask
=
166 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_BASIC
);
168 flow
->l2_key
.ether_type
= key
->n_proto
;
169 flow
->l2_mask
.ether_type
= mask
->n_proto
;
171 if (key
->n_proto
== htons(ETH_P_IP
) ||
172 key
->n_proto
== htons(ETH_P_IPV6
)) {
173 flow
->l4_key
.ip_proto
= key
->ip_proto
;
174 flow
->l4_mask
.ip_proto
= mask
->ip_proto
;
178 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ETH_ADDRS
)) {
179 struct flow_dissector_key_eth_addrs
*key
=
180 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ETH_ADDRS
);
181 struct flow_dissector_key_eth_addrs
*mask
=
182 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ETH_ADDRS
);
184 flow
->flags
|= BNXT_TC_FLOW_FLAGS_ETH_ADDRS
;
185 ether_addr_copy(flow
->l2_key
.dmac
, key
->dst
);
186 ether_addr_copy(flow
->l2_mask
.dmac
, mask
->dst
);
187 ether_addr_copy(flow
->l2_key
.smac
, key
->src
);
188 ether_addr_copy(flow
->l2_mask
.smac
, mask
->src
);
191 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_VLAN
)) {
192 struct flow_dissector_key_vlan
*key
=
193 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_VLAN
);
194 struct flow_dissector_key_vlan
*mask
=
195 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_VLAN
);
197 flow
->l2_key
.inner_vlan_tci
=
198 cpu_to_be16(VLAN_TCI(key
->vlan_id
, key
->vlan_priority
));
199 flow
->l2_mask
.inner_vlan_tci
=
200 cpu_to_be16((VLAN_TCI(mask
->vlan_id
, mask
->vlan_priority
)));
201 flow
->l2_key
.inner_vlan_tpid
= htons(ETH_P_8021Q
);
202 flow
->l2_mask
.inner_vlan_tpid
= htons(0xffff);
203 flow
->l2_key
.num_vlans
= 1;
206 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
)) {
207 struct flow_dissector_key_ipv4_addrs
*key
=
208 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
);
209 struct flow_dissector_key_ipv4_addrs
*mask
=
210 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
);
212 flow
->flags
|= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS
;
213 flow
->l3_key
.ipv4
.daddr
.s_addr
= key
->dst
;
214 flow
->l3_mask
.ipv4
.daddr
.s_addr
= mask
->dst
;
215 flow
->l3_key
.ipv4
.saddr
.s_addr
= key
->src
;
216 flow
->l3_mask
.ipv4
.saddr
.s_addr
= mask
->src
;
217 } else if (dissector_uses_key(dissector
,
218 FLOW_DISSECTOR_KEY_IPV6_ADDRS
)) {
219 struct flow_dissector_key_ipv6_addrs
*key
=
220 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
);
221 struct flow_dissector_key_ipv6_addrs
*mask
=
222 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
);
224 flow
->flags
|= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS
;
225 flow
->l3_key
.ipv6
.daddr
= key
->dst
;
226 flow
->l3_mask
.ipv6
.daddr
= mask
->dst
;
227 flow
->l3_key
.ipv6
.saddr
= key
->src
;
228 flow
->l3_mask
.ipv6
.saddr
= mask
->src
;
231 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_PORTS
)) {
232 struct flow_dissector_key_ports
*key
=
233 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_PORTS
);
234 struct flow_dissector_key_ports
*mask
=
235 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_PORTS
);
237 flow
->flags
|= BNXT_TC_FLOW_FLAGS_PORTS
;
238 flow
->l4_key
.ports
.dport
= key
->dst
;
239 flow
->l4_mask
.ports
.dport
= mask
->dst
;
240 flow
->l4_key
.ports
.sport
= key
->src
;
241 flow
->l4_mask
.ports
.sport
= mask
->src
;
244 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ICMP
)) {
245 struct flow_dissector_key_icmp
*key
=
246 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ICMP
);
247 struct flow_dissector_key_icmp
*mask
=
248 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ICMP
);
250 flow
->flags
|= BNXT_TC_FLOW_FLAGS_ICMP
;
251 flow
->l4_key
.icmp
.type
= key
->type
;
252 flow
->l4_key
.icmp
.code
= key
->code
;
253 flow
->l4_mask
.icmp
.type
= mask
->type
;
254 flow
->l4_mask
.icmp
.code
= mask
->code
;
257 return bnxt_tc_parse_actions(bp
, &flow
->actions
, tc_flow_cmd
->exts
);
260 static int bnxt_hwrm_cfa_flow_free(struct bnxt
*bp
, __le16 flow_handle
)
262 struct hwrm_cfa_flow_free_input req
= { 0 };
265 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_FREE
, -1, -1);
266 req
.flow_handle
= flow_handle
;
268 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
270 netdev_info(bp
->dev
, "Error: %s: flow_handle=0x%x rc=%d",
271 __func__
, flow_handle
, rc
);
275 static int ipv6_mask_len(struct in6_addr
*mask
)
279 for (i
= 0; i
< 4; i
++)
280 mask_len
+= inet_mask_len(mask
->s6_addr32
[i
]);
285 static bool is_wildcard(void *mask
, int len
)
290 for (i
= 0; i
< len
; i
++) {
297 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
298 __le16 ref_flow_handle
, __le16
*flow_handle
)
300 struct hwrm_cfa_flow_alloc_output
*resp
= bp
->hwrm_cmd_resp_addr
;
301 struct bnxt_tc_actions
*actions
= &flow
->actions
;
302 struct bnxt_tc_l3_key
*l3_mask
= &flow
->l3_mask
;
303 struct bnxt_tc_l3_key
*l3_key
= &flow
->l3_key
;
304 struct hwrm_cfa_flow_alloc_input req
= { 0 };
305 u16 flow_flags
= 0, action_flags
= 0;
308 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_ALLOC
, -1, -1);
310 req
.src_fid
= cpu_to_le16(flow
->src_fid
);
311 req
.ref_flow_handle
= ref_flow_handle
;
312 req
.ethertype
= flow
->l2_key
.ether_type
;
313 req
.ip_proto
= flow
->l4_key
.ip_proto
;
315 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_ETH_ADDRS
) {
316 memcpy(req
.dmac
, flow
->l2_key
.dmac
, ETH_ALEN
);
317 memcpy(req
.smac
, flow
->l2_key
.smac
, ETH_ALEN
);
320 if (flow
->l2_key
.num_vlans
> 0) {
321 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE
;
322 /* FW expects the inner_vlan_tci value to be set
323 * in outer_vlan_tci when num_vlans is 1 (which is
324 * always the case in TC.)
326 req
.outer_vlan_tci
= flow
->l2_key
.inner_vlan_tci
;
329 /* If all IP and L4 fields are wildcarded then this is an L2 flow */
330 if (is_wildcard(&l3_mask
, sizeof(l3_mask
)) &&
331 is_wildcard(&flow
->l4_mask
, sizeof(flow
->l4_mask
))) {
332 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2
;
334 flow_flags
|= flow
->l2_key
.ether_type
== htons(ETH_P_IP
) ?
335 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4
:
336 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
;
338 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_IPV4_ADDRS
) {
339 req
.ip_dst
[0] = l3_key
->ipv4
.daddr
.s_addr
;
340 req
.ip_dst_mask_len
=
341 inet_mask_len(l3_mask
->ipv4
.daddr
.s_addr
);
342 req
.ip_src
[0] = l3_key
->ipv4
.saddr
.s_addr
;
343 req
.ip_src_mask_len
=
344 inet_mask_len(l3_mask
->ipv4
.saddr
.s_addr
);
345 } else if (flow
->flags
& BNXT_TC_FLOW_FLAGS_IPV6_ADDRS
) {
346 memcpy(req
.ip_dst
, l3_key
->ipv6
.daddr
.s6_addr32
,
348 req
.ip_dst_mask_len
=
349 ipv6_mask_len(&l3_mask
->ipv6
.daddr
);
350 memcpy(req
.ip_src
, l3_key
->ipv6
.saddr
.s6_addr32
,
352 req
.ip_src_mask_len
=
353 ipv6_mask_len(&l3_mask
->ipv6
.saddr
);
357 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_PORTS
) {
358 req
.l4_src_port
= flow
->l4_key
.ports
.sport
;
359 req
.l4_src_port_mask
= flow
->l4_mask
.ports
.sport
;
360 req
.l4_dst_port
= flow
->l4_key
.ports
.dport
;
361 req
.l4_dst_port_mask
= flow
->l4_mask
.ports
.dport
;
362 } else if (flow
->flags
& BNXT_TC_FLOW_FLAGS_ICMP
) {
363 /* l4 ports serve as type/code when ip_proto is ICMP */
364 req
.l4_src_port
= htons(flow
->l4_key
.icmp
.type
);
365 req
.l4_src_port_mask
= htons(flow
->l4_mask
.icmp
.type
);
366 req
.l4_dst_port
= htons(flow
->l4_key
.icmp
.code
);
367 req
.l4_dst_port_mask
= htons(flow
->l4_mask
.icmp
.code
);
369 req
.flags
= cpu_to_le16(flow_flags
);
371 if (actions
->flags
& BNXT_TC_ACTION_FLAG_DROP
) {
372 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP
;
374 if (actions
->flags
& BNXT_TC_ACTION_FLAG_FWD
) {
375 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD
;
376 req
.dst_fid
= cpu_to_le16(actions
->dst_fid
);
378 if (actions
->flags
& BNXT_TC_ACTION_FLAG_PUSH_VLAN
) {
380 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE
;
381 req
.l2_rewrite_vlan_tpid
= actions
->push_vlan_tpid
;
382 req
.l2_rewrite_vlan_tci
= actions
->push_vlan_tci
;
383 memcpy(&req
.l2_rewrite_dmac
, &req
.dmac
, ETH_ALEN
);
384 memcpy(&req
.l2_rewrite_smac
, &req
.smac
, ETH_ALEN
);
386 if (actions
->flags
& BNXT_TC_ACTION_FLAG_POP_VLAN
) {
388 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE
;
389 /* Rewrite config with tpid = 0 implies vlan pop */
390 req
.l2_rewrite_vlan_tpid
= 0;
391 memcpy(&req
.l2_rewrite_dmac
, &req
.dmac
, ETH_ALEN
);
392 memcpy(&req
.l2_rewrite_smac
, &req
.smac
, ETH_ALEN
);
395 req
.action_flags
= cpu_to_le16(action_flags
);
397 mutex_lock(&bp
->hwrm_cmd_lock
);
399 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
401 *flow_handle
= resp
->flow_handle
;
403 mutex_unlock(&bp
->hwrm_cmd_lock
);
408 /* Add val to accum while handling a possible wraparound
409 * of val. Eventhough val is of type u64, its actual width
410 * is denoted by mask and will wrap-around beyond that width.
412 static void accumulate_val(u64
*accum
, u64 val
, u64 mask
)
414 #define low_bits(x, mask) ((x) & (mask))
415 #define high_bits(x, mask) ((x) & ~(mask))
416 bool wrapped
= val
< low_bits(*accum
, mask
);
418 *accum
= high_bits(*accum
, mask
) + val
;
420 *accum
+= (mask
+ 1);
423 /* The HW counters' width is much less than 64bits.
424 * Handle possible wrap-around while updating the stat counters
426 static void bnxt_flow_stats_fix_wraparound(struct bnxt_tc_info
*tc_info
,
427 struct bnxt_tc_flow_stats
*stats
,
428 struct bnxt_tc_flow_stats
*hw_stats
)
430 accumulate_val(&stats
->bytes
, hw_stats
->bytes
, tc_info
->bytes_mask
);
431 accumulate_val(&stats
->packets
, hw_stats
->packets
,
432 tc_info
->packets_mask
);
435 /* Fix possible wraparound of the stats queried from HW, calculate
436 * the delta from prev_stats, and also update the prev_stats.
437 * The HW flow stats are fetched under the hwrm_cmd_lock mutex.
438 * This routine is best called while under the mutex so that the
439 * stats processing happens atomically.
441 static void bnxt_flow_stats_calc(struct bnxt_tc_info
*tc_info
,
442 struct bnxt_tc_flow
*flow
,
443 struct bnxt_tc_flow_stats
*stats
)
445 struct bnxt_tc_flow_stats
*acc_stats
, *prev_stats
;
447 acc_stats
= &flow
->stats
;
448 bnxt_flow_stats_fix_wraparound(tc_info
, acc_stats
, stats
);
450 prev_stats
= &flow
->prev_stats
;
451 stats
->bytes
= acc_stats
->bytes
- prev_stats
->bytes
;
452 stats
->packets
= acc_stats
->packets
- prev_stats
->packets
;
453 *prev_stats
= *acc_stats
;
456 static int bnxt_hwrm_cfa_flow_stats_get(struct bnxt
*bp
,
458 struct bnxt_tc_flow
*flow
,
459 struct bnxt_tc_flow_stats
*stats
)
461 struct hwrm_cfa_flow_stats_output
*resp
= bp
->hwrm_cmd_resp_addr
;
462 struct hwrm_cfa_flow_stats_input req
= { 0 };
465 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_STATS
, -1, -1);
466 req
.num_flows
= cpu_to_le16(1);
467 req
.flow_handle_0
= flow_handle
;
469 mutex_lock(&bp
->hwrm_cmd_lock
);
470 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
472 stats
->packets
= le64_to_cpu(resp
->packet_0
);
473 stats
->bytes
= le64_to_cpu(resp
->byte_0
);
474 bnxt_flow_stats_calc(&bp
->tc_info
, flow
, stats
);
476 netdev_info(bp
->dev
, "error rc=%d", rc
);
479 mutex_unlock(&bp
->hwrm_cmd_lock
);
483 static int bnxt_tc_put_l2_node(struct bnxt
*bp
,
484 struct bnxt_tc_flow_node
*flow_node
)
486 struct bnxt_tc_l2_node
*l2_node
= flow_node
->l2_node
;
487 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
490 /* remove flow_node from the L2 shared flow list */
491 list_del(&flow_node
->l2_list_node
);
492 if (--l2_node
->refcount
== 0) {
493 rc
= rhashtable_remove_fast(&tc_info
->l2_table
, &l2_node
->node
,
494 tc_info
->l2_ht_params
);
497 "Error: %s: rhashtable_remove_fast: %d",
499 kfree_rcu(l2_node
, rcu
);
504 static struct bnxt_tc_l2_node
*
505 bnxt_tc_get_l2_node(struct bnxt
*bp
, struct rhashtable
*l2_table
,
506 struct rhashtable_params ht_params
,
507 struct bnxt_tc_l2_key
*l2_key
)
509 struct bnxt_tc_l2_node
*l2_node
;
512 l2_node
= rhashtable_lookup_fast(l2_table
, l2_key
, ht_params
);
514 l2_node
= kzalloc(sizeof(*l2_node
), GFP_KERNEL
);
520 l2_node
->key
= *l2_key
;
521 rc
= rhashtable_insert_fast(l2_table
, &l2_node
->node
,
526 "Error: %s: rhashtable_insert_fast: %d",
530 INIT_LIST_HEAD(&l2_node
->common_l2_flows
);
535 /* Get the ref_flow_handle for a flow by checking if there are any other
536 * flows that share the same L2 key as this flow.
539 bnxt_tc_get_ref_flow_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
540 struct bnxt_tc_flow_node
*flow_node
,
541 __le16
*ref_flow_handle
)
543 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
544 struct bnxt_tc_flow_node
*ref_flow_node
;
545 struct bnxt_tc_l2_node
*l2_node
;
547 l2_node
= bnxt_tc_get_l2_node(bp
, &tc_info
->l2_table
,
548 tc_info
->l2_ht_params
,
553 /* If any other flow is using this l2_node, use it's flow_handle
554 * as the ref_flow_handle
556 if (l2_node
->refcount
> 0) {
557 ref_flow_node
= list_first_entry(&l2_node
->common_l2_flows
,
558 struct bnxt_tc_flow_node
,
560 *ref_flow_handle
= ref_flow_node
->flow_handle
;
562 *ref_flow_handle
= cpu_to_le16(0xffff);
565 /* Insert the l2_node into the flow_node so that subsequent flows
566 * with a matching l2 key can use the flow_handle of this flow
567 * as their ref_flow_handle
569 flow_node
->l2_node
= l2_node
;
570 list_add(&flow_node
->l2_list_node
, &l2_node
->common_l2_flows
);
575 /* After the flow parsing is done, this routine is used for checking
576 * if there are any aspects of the flow that prevent it from being
579 static bool bnxt_tc_can_offload(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
)
581 /* If L4 ports are specified then ip_proto must be TCP or UDP */
582 if ((flow
->flags
& BNXT_TC_FLOW_FLAGS_PORTS
) &&
583 (flow
->l4_key
.ip_proto
!= IPPROTO_TCP
&&
584 flow
->l4_key
.ip_proto
!= IPPROTO_UDP
)) {
585 netdev_info(bp
->dev
, "Cannot offload non-TCP/UDP (%d) ports",
586 flow
->l4_key
.ip_proto
);
593 static int __bnxt_tc_del_flow(struct bnxt
*bp
,
594 struct bnxt_tc_flow_node
*flow_node
)
596 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
599 /* send HWRM cmd to free the flow-id */
600 bnxt_hwrm_cfa_flow_free(bp
, flow_node
->flow_handle
);
602 mutex_lock(&tc_info
->lock
);
604 /* release reference to l2 node */
605 bnxt_tc_put_l2_node(bp
, flow_node
);
607 mutex_unlock(&tc_info
->lock
);
609 rc
= rhashtable_remove_fast(&tc_info
->flow_table
, &flow_node
->node
,
610 tc_info
->flow_ht_params
);
612 netdev_err(bp
->dev
, "Error: %s: rhashtable_remove_fast rc=%d",
615 kfree_rcu(flow_node
, rcu
);
619 /* Add a new flow or replace an existing flow.
621 * There are essentially two critical sections here.
622 * 1. while adding a new flow
624 * b) issue HWRM cmd and get flow_handle
625 * c) link l2-key with flow
626 * 2. while deleting a flow
627 * a) unlinking l2-key from flow
628 * A lock is needed to protect these two critical sections.
630 * The hash-tables are already protected by the rhashtable API.
632 static int bnxt_tc_add_flow(struct bnxt
*bp
, u16 src_fid
,
633 struct tc_cls_flower_offload
*tc_flow_cmd
)
635 struct bnxt_tc_flow_node
*new_node
, *old_node
;
636 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
637 struct bnxt_tc_flow
*flow
;
638 __le16 ref_flow_handle
;
641 /* allocate memory for the new flow and it's node */
642 new_node
= kzalloc(sizeof(*new_node
), GFP_KERNEL
);
647 new_node
->cookie
= tc_flow_cmd
->cookie
;
648 flow
= &new_node
->flow
;
650 rc
= bnxt_tc_parse_flow(bp
, tc_flow_cmd
, flow
);
653 flow
->src_fid
= src_fid
;
655 if (!bnxt_tc_can_offload(bp
, flow
)) {
660 /* If a flow exists with the same cookie, delete it */
661 old_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
662 &tc_flow_cmd
->cookie
,
663 tc_info
->flow_ht_params
);
665 __bnxt_tc_del_flow(bp
, old_node
);
667 /* Check if the L2 part of the flow has been offloaded already.
668 * If so, bump up it's refcnt and get it's reference handle.
670 mutex_lock(&tc_info
->lock
);
671 rc
= bnxt_tc_get_ref_flow_handle(bp
, flow
, new_node
, &ref_flow_handle
);
675 /* send HWRM cmd to alloc the flow */
676 rc
= bnxt_hwrm_cfa_flow_alloc(bp
, flow
, ref_flow_handle
,
677 &new_node
->flow_handle
);
681 /* add new flow to flow-table */
682 rc
= rhashtable_insert_fast(&tc_info
->flow_table
, &new_node
->node
,
683 tc_info
->flow_ht_params
);
687 mutex_unlock(&tc_info
->lock
);
691 bnxt_hwrm_cfa_flow_free(bp
, new_node
->flow_handle
);
693 bnxt_tc_put_l2_node(bp
, new_node
);
695 mutex_unlock(&tc_info
->lock
);
699 netdev_err(bp
->dev
, "Error: %s: cookie=0x%lx error=%d",
700 __func__
, tc_flow_cmd
->cookie
, rc
);
704 static int bnxt_tc_del_flow(struct bnxt
*bp
,
705 struct tc_cls_flower_offload
*tc_flow_cmd
)
707 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
708 struct bnxt_tc_flow_node
*flow_node
;
710 flow_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
711 &tc_flow_cmd
->cookie
,
712 tc_info
->flow_ht_params
);
714 netdev_info(bp
->dev
, "ERROR: no flow_node for cookie %lx",
715 tc_flow_cmd
->cookie
);
719 return __bnxt_tc_del_flow(bp
, flow_node
);
722 static int bnxt_tc_get_flow_stats(struct bnxt
*bp
,
723 struct tc_cls_flower_offload
*tc_flow_cmd
)
725 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
726 struct bnxt_tc_flow_node
*flow_node
;
727 struct bnxt_tc_flow_stats stats
;
730 flow_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
731 &tc_flow_cmd
->cookie
,
732 tc_info
->flow_ht_params
);
734 netdev_info(bp
->dev
, "Error: no flow_node for cookie %lx",
735 tc_flow_cmd
->cookie
);
739 rc
= bnxt_hwrm_cfa_flow_stats_get(bp
, flow_node
->flow_handle
,
740 &flow_node
->flow
, &stats
);
744 tcf_exts_stats_update(tc_flow_cmd
->exts
, stats
.bytes
, stats
.packets
, 0);
748 int bnxt_tc_setup_flower(struct bnxt
*bp
, u16 src_fid
,
749 struct tc_cls_flower_offload
*cls_flower
)
753 switch (cls_flower
->command
) {
754 case TC_CLSFLOWER_REPLACE
:
755 rc
= bnxt_tc_add_flow(bp
, src_fid
, cls_flower
);
758 case TC_CLSFLOWER_DESTROY
:
759 rc
= bnxt_tc_del_flow(bp
, cls_flower
);
762 case TC_CLSFLOWER_STATS
:
763 rc
= bnxt_tc_get_flow_stats(bp
, cls_flower
);
769 static const struct rhashtable_params bnxt_tc_flow_ht_params
= {
770 .head_offset
= offsetof(struct bnxt_tc_flow_node
, node
),
771 .key_offset
= offsetof(struct bnxt_tc_flow_node
, cookie
),
772 .key_len
= sizeof(((struct bnxt_tc_flow_node
*)0)->cookie
),
773 .automatic_shrinking
= true
776 static const struct rhashtable_params bnxt_tc_l2_ht_params
= {
777 .head_offset
= offsetof(struct bnxt_tc_l2_node
, node
),
778 .key_offset
= offsetof(struct bnxt_tc_l2_node
, key
),
779 .key_len
= BNXT_TC_L2_KEY_LEN
,
780 .automatic_shrinking
= true
783 /* convert counter width in bits to a mask */
784 #define mask(width) ((u64)~0 >> (64 - (width)))
786 int bnxt_init_tc(struct bnxt
*bp
)
788 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
791 if (bp
->hwrm_spec_code
< 0x10800) {
793 "Firmware does not support TC flower offload.\n");
796 mutex_init(&tc_info
->lock
);
798 /* Counter widths are programmed by FW */
799 tc_info
->bytes_mask
= mask(36);
800 tc_info
->packets_mask
= mask(28);
802 tc_info
->flow_ht_params
= bnxt_tc_flow_ht_params
;
803 rc
= rhashtable_init(&tc_info
->flow_table
, &tc_info
->flow_ht_params
);
807 tc_info
->l2_ht_params
= bnxt_tc_l2_ht_params
;
808 rc
= rhashtable_init(&tc_info
->l2_table
, &tc_info
->l2_ht_params
);
810 goto destroy_flow_table
;
812 tc_info
->enabled
= true;
813 bp
->dev
->hw_features
|= NETIF_F_HW_TC
;
814 bp
->dev
->features
|= NETIF_F_HW_TC
;
818 rhashtable_destroy(&tc_info
->flow_table
);
822 void bnxt_shutdown_tc(struct bnxt
*bp
)
824 struct bnxt_tc_info
*tc_info
= &bp
->tc_info
;
826 if (!tc_info
->enabled
)
829 rhashtable_destroy(&tc_info
->flow_table
);
830 rhashtable_destroy(&tc_info
->l2_table
);