1 /* Broadcom NetXtreme-C/E network driver.
3 * Copyright (c) 2017 Broadcom Limited
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
10 #include <linux/netdevice.h>
11 #include <linux/inetdevice.h>
12 #include <linux/if_vlan.h>
13 #include <net/flow_dissector.h>
14 #include <net/pkt_cls.h>
15 #include <net/tc_act/tc_gact.h>
16 #include <net/tc_act/tc_skbedit.h>
17 #include <net/tc_act/tc_mirred.h>
18 #include <net/tc_act/tc_vlan.h>
19 #include <net/tc_act/tc_tunnel_key.h>
23 #include "bnxt_sriov.h"
27 #define BNXT_FID_INVALID 0xffff
28 #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT))
30 /* Return the dst fid of the func for flow forwarding
31 * For PFs: src_fid is the fid of the PF
32 * For VF-reps: src_fid the fid of the VF
34 static u16
bnxt_flow_get_dst_fid(struct bnxt
*pf_bp
, struct net_device
*dev
)
38 /* check if dev belongs to the same switch */
39 if (!switchdev_port_same_parent_id(pf_bp
->dev
, dev
)) {
40 netdev_info(pf_bp
->dev
, "dev(ifindex=%d) not on same switch",
42 return BNXT_FID_INVALID
;
45 /* Is dev a VF-rep? */
46 if (dev
!= pf_bp
->dev
)
47 return bnxt_vf_rep_get_fid(dev
);
49 bp
= netdev_priv(dev
);
53 static int bnxt_tc_parse_redir(struct bnxt
*bp
,
54 struct bnxt_tc_actions
*actions
,
55 const struct tc_action
*tc_act
)
57 int ifindex
= tcf_mirred_ifindex(tc_act
);
58 struct net_device
*dev
;
61 dev
= __dev_get_by_index(dev_net(bp
->dev
), ifindex
);
63 netdev_info(bp
->dev
, "no dev for ifindex=%d", ifindex
);
67 /* find the FID from dev */
68 dst_fid
= bnxt_flow_get_dst_fid(bp
, dev
);
69 if (dst_fid
== BNXT_FID_INVALID
) {
70 netdev_info(bp
->dev
, "can't get fid for ifindex=%d", ifindex
);
74 actions
->flags
|= BNXT_TC_ACTION_FLAG_FWD
;
75 actions
->dst_fid
= dst_fid
;
76 actions
->dst_dev
= dev
;
80 static void bnxt_tc_parse_vlan(struct bnxt
*bp
,
81 struct bnxt_tc_actions
*actions
,
82 const struct tc_action
*tc_act
)
84 if (tcf_vlan_action(tc_act
) == TCA_VLAN_ACT_POP
) {
85 actions
->flags
|= BNXT_TC_ACTION_FLAG_POP_VLAN
;
86 } else if (tcf_vlan_action(tc_act
) == TCA_VLAN_ACT_PUSH
) {
87 actions
->flags
|= BNXT_TC_ACTION_FLAG_PUSH_VLAN
;
88 actions
->push_vlan_tci
= htons(tcf_vlan_push_vid(tc_act
));
89 actions
->push_vlan_tpid
= tcf_vlan_push_proto(tc_act
);
93 static int bnxt_tc_parse_tunnel_set(struct bnxt
*bp
,
94 struct bnxt_tc_actions
*actions
,
95 const struct tc_action
*tc_act
)
97 struct ip_tunnel_info
*tun_info
= tcf_tunnel_info(tc_act
);
98 struct ip_tunnel_key
*tun_key
= &tun_info
->key
;
100 if (ip_tunnel_info_af(tun_info
) != AF_INET
) {
101 netdev_info(bp
->dev
, "only IPv4 tunnel-encap is supported");
105 actions
->tun_encap_key
= *tun_key
;
106 actions
->flags
|= BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
;
110 static int bnxt_tc_parse_actions(struct bnxt
*bp
,
111 struct bnxt_tc_actions
*actions
,
112 struct tcf_exts
*tc_exts
)
114 const struct tc_action
*tc_act
;
115 LIST_HEAD(tc_actions
);
118 if (!tcf_exts_has_actions(tc_exts
)) {
119 netdev_info(bp
->dev
, "no actions");
123 tcf_exts_to_list(tc_exts
, &tc_actions
);
124 list_for_each_entry(tc_act
, &tc_actions
, list
) {
126 if (is_tcf_gact_shot(tc_act
)) {
127 actions
->flags
|= BNXT_TC_ACTION_FLAG_DROP
;
128 return 0; /* don't bother with other actions */
131 /* Redirect action */
132 if (is_tcf_mirred_egress_redirect(tc_act
)) {
133 rc
= bnxt_tc_parse_redir(bp
, actions
, tc_act
);
140 if (is_tcf_vlan(tc_act
)) {
141 bnxt_tc_parse_vlan(bp
, actions
, tc_act
);
146 if (is_tcf_tunnel_set(tc_act
)) {
147 rc
= bnxt_tc_parse_tunnel_set(bp
, actions
, tc_act
);
154 if (is_tcf_tunnel_release(tc_act
)) {
155 actions
->flags
|= BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
;
163 /* Tunnel encap/decap action must be accompanied by a redirect action */
164 if ((actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
||
165 actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
) &&
166 !(actions
->flags
& BNXT_TC_ACTION_FLAG_FWD
)) {
168 "error: no redir action along with encap/decap");
175 #define GET_KEY(flow_cmd, key_type) \
176 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
178 #define GET_MASK(flow_cmd, key_type) \
179 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
182 static int bnxt_tc_parse_flow(struct bnxt
*bp
,
183 struct tc_cls_flower_offload
*tc_flow_cmd
,
184 struct bnxt_tc_flow
*flow
)
186 struct flow_dissector
*dissector
= tc_flow_cmd
->dissector
;
189 /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */
190 if ((dissector
->used_keys
& BIT(FLOW_DISSECTOR_KEY_CONTROL
)) == 0 ||
191 (dissector
->used_keys
& BIT(FLOW_DISSECTOR_KEY_BASIC
)) == 0) {
192 netdev_info(bp
->dev
, "cannot form TC key: used_keys = 0x%x",
193 dissector
->used_keys
);
197 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_CONTROL
)) {
198 struct flow_dissector_key_control
*key
=
199 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_CONTROL
);
201 addr_type
= key
->addr_type
;
204 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_BASIC
)) {
205 struct flow_dissector_key_basic
*key
=
206 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_BASIC
);
207 struct flow_dissector_key_basic
*mask
=
208 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_BASIC
);
210 flow
->l2_key
.ether_type
= key
->n_proto
;
211 flow
->l2_mask
.ether_type
= mask
->n_proto
;
213 if (key
->n_proto
== htons(ETH_P_IP
) ||
214 key
->n_proto
== htons(ETH_P_IPV6
)) {
215 flow
->l4_key
.ip_proto
= key
->ip_proto
;
216 flow
->l4_mask
.ip_proto
= mask
->ip_proto
;
220 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ETH_ADDRS
)) {
221 struct flow_dissector_key_eth_addrs
*key
=
222 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ETH_ADDRS
);
223 struct flow_dissector_key_eth_addrs
*mask
=
224 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ETH_ADDRS
);
226 flow
->flags
|= BNXT_TC_FLOW_FLAGS_ETH_ADDRS
;
227 ether_addr_copy(flow
->l2_key
.dmac
, key
->dst
);
228 ether_addr_copy(flow
->l2_mask
.dmac
, mask
->dst
);
229 ether_addr_copy(flow
->l2_key
.smac
, key
->src
);
230 ether_addr_copy(flow
->l2_mask
.smac
, mask
->src
);
233 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_VLAN
)) {
234 struct flow_dissector_key_vlan
*key
=
235 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_VLAN
);
236 struct flow_dissector_key_vlan
*mask
=
237 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_VLAN
);
239 flow
->l2_key
.inner_vlan_tci
=
240 cpu_to_be16(VLAN_TCI(key
->vlan_id
, key
->vlan_priority
));
241 flow
->l2_mask
.inner_vlan_tci
=
242 cpu_to_be16((VLAN_TCI(mask
->vlan_id
, mask
->vlan_priority
)));
243 flow
->l2_key
.inner_vlan_tpid
= htons(ETH_P_8021Q
);
244 flow
->l2_mask
.inner_vlan_tpid
= htons(0xffff);
245 flow
->l2_key
.num_vlans
= 1;
248 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
)) {
249 struct flow_dissector_key_ipv4_addrs
*key
=
250 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
);
251 struct flow_dissector_key_ipv4_addrs
*mask
=
252 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV4_ADDRS
);
254 flow
->flags
|= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS
;
255 flow
->l3_key
.ipv4
.daddr
.s_addr
= key
->dst
;
256 flow
->l3_mask
.ipv4
.daddr
.s_addr
= mask
->dst
;
257 flow
->l3_key
.ipv4
.saddr
.s_addr
= key
->src
;
258 flow
->l3_mask
.ipv4
.saddr
.s_addr
= mask
->src
;
259 } else if (dissector_uses_key(dissector
,
260 FLOW_DISSECTOR_KEY_IPV6_ADDRS
)) {
261 struct flow_dissector_key_ipv6_addrs
*key
=
262 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
);
263 struct flow_dissector_key_ipv6_addrs
*mask
=
264 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_IPV6_ADDRS
);
266 flow
->flags
|= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS
;
267 flow
->l3_key
.ipv6
.daddr
= key
->dst
;
268 flow
->l3_mask
.ipv6
.daddr
= mask
->dst
;
269 flow
->l3_key
.ipv6
.saddr
= key
->src
;
270 flow
->l3_mask
.ipv6
.saddr
= mask
->src
;
273 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_PORTS
)) {
274 struct flow_dissector_key_ports
*key
=
275 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_PORTS
);
276 struct flow_dissector_key_ports
*mask
=
277 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_PORTS
);
279 flow
->flags
|= BNXT_TC_FLOW_FLAGS_PORTS
;
280 flow
->l4_key
.ports
.dport
= key
->dst
;
281 flow
->l4_mask
.ports
.dport
= mask
->dst
;
282 flow
->l4_key
.ports
.sport
= key
->src
;
283 flow
->l4_mask
.ports
.sport
= mask
->src
;
286 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ICMP
)) {
287 struct flow_dissector_key_icmp
*key
=
288 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ICMP
);
289 struct flow_dissector_key_icmp
*mask
=
290 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ICMP
);
292 flow
->flags
|= BNXT_TC_FLOW_FLAGS_ICMP
;
293 flow
->l4_key
.icmp
.type
= key
->type
;
294 flow
->l4_key
.icmp
.code
= key
->code
;
295 flow
->l4_mask
.icmp
.type
= mask
->type
;
296 flow
->l4_mask
.icmp
.code
= mask
->code
;
299 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_CONTROL
)) {
300 struct flow_dissector_key_control
*key
=
301 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_CONTROL
);
303 addr_type
= key
->addr_type
;
306 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
)) {
307 struct flow_dissector_key_ipv4_addrs
*key
=
308 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
);
309 struct flow_dissector_key_ipv4_addrs
*mask
=
310 GET_MASK(tc_flow_cmd
,
311 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS
);
313 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS
;
314 flow
->tun_key
.u
.ipv4
.dst
= key
->dst
;
315 flow
->tun_mask
.u
.ipv4
.dst
= mask
->dst
;
316 flow
->tun_key
.u
.ipv4
.src
= key
->src
;
317 flow
->tun_mask
.u
.ipv4
.src
= mask
->src
;
318 } else if (dissector_uses_key(dissector
,
319 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS
)) {
323 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_KEYID
)) {
324 struct flow_dissector_key_keyid
*key
=
325 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_KEYID
);
326 struct flow_dissector_key_keyid
*mask
=
327 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_KEYID
);
329 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_ID
;
330 flow
->tun_key
.tun_id
= key32_to_tunnel_id(key
->keyid
);
331 flow
->tun_mask
.tun_id
= key32_to_tunnel_id(mask
->keyid
);
334 if (dissector_uses_key(dissector
, FLOW_DISSECTOR_KEY_ENC_PORTS
)) {
335 struct flow_dissector_key_ports
*key
=
336 GET_KEY(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_PORTS
);
337 struct flow_dissector_key_ports
*mask
=
338 GET_MASK(tc_flow_cmd
, FLOW_DISSECTOR_KEY_ENC_PORTS
);
340 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_PORTS
;
341 flow
->tun_key
.tp_dst
= key
->dst
;
342 flow
->tun_mask
.tp_dst
= mask
->dst
;
343 flow
->tun_key
.tp_src
= key
->src
;
344 flow
->tun_mask
.tp_src
= mask
->src
;
347 return bnxt_tc_parse_actions(bp
, &flow
->actions
, tc_flow_cmd
->exts
);
350 static int bnxt_hwrm_cfa_flow_free(struct bnxt
*bp
, __le16 flow_handle
)
352 struct hwrm_cfa_flow_free_input req
= { 0 };
355 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_FREE
, -1, -1);
356 req
.flow_handle
= flow_handle
;
358 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
360 netdev_info(bp
->dev
, "Error: %s: flow_handle=0x%x rc=%d",
361 __func__
, flow_handle
, rc
);
365 static int ipv6_mask_len(struct in6_addr
*mask
)
369 for (i
= 0; i
< 4; i
++)
370 mask_len
+= inet_mask_len(mask
->s6_addr32
[i
]);
375 static bool is_wildcard(void *mask
, int len
)
380 for (i
= 0; i
< len
; i
++) {
387 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
388 __le16 ref_flow_handle
,
389 __le32 tunnel_handle
, __le16
*flow_handle
)
391 struct hwrm_cfa_flow_alloc_output
*resp
= bp
->hwrm_cmd_resp_addr
;
392 struct bnxt_tc_actions
*actions
= &flow
->actions
;
393 struct bnxt_tc_l3_key
*l3_mask
= &flow
->l3_mask
;
394 struct bnxt_tc_l3_key
*l3_key
= &flow
->l3_key
;
395 struct hwrm_cfa_flow_alloc_input req
= { 0 };
396 u16 flow_flags
= 0, action_flags
= 0;
399 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_ALLOC
, -1, -1);
401 req
.src_fid
= cpu_to_le16(flow
->src_fid
);
402 req
.ref_flow_handle
= ref_flow_handle
;
404 if (actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
||
405 actions
->flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
) {
406 req
.tunnel_handle
= tunnel_handle
;
407 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL
;
408 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL
;
411 req
.ethertype
= flow
->l2_key
.ether_type
;
412 req
.ip_proto
= flow
->l4_key
.ip_proto
;
414 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_ETH_ADDRS
) {
415 memcpy(req
.dmac
, flow
->l2_key
.dmac
, ETH_ALEN
);
416 memcpy(req
.smac
, flow
->l2_key
.smac
, ETH_ALEN
);
419 if (flow
->l2_key
.num_vlans
> 0) {
420 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE
;
421 /* FW expects the inner_vlan_tci value to be set
422 * in outer_vlan_tci when num_vlans is 1 (which is
423 * always the case in TC.)
425 req
.outer_vlan_tci
= flow
->l2_key
.inner_vlan_tci
;
428 /* If all IP and L4 fields are wildcarded then this is an L2 flow */
429 if (is_wildcard(&l3_mask
, sizeof(l3_mask
)) &&
430 is_wildcard(&flow
->l4_mask
, sizeof(flow
->l4_mask
))) {
431 flow_flags
|= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2
;
433 flow_flags
|= flow
->l2_key
.ether_type
== htons(ETH_P_IP
) ?
434 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4
:
435 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
;
437 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_IPV4_ADDRS
) {
438 req
.ip_dst
[0] = l3_key
->ipv4
.daddr
.s_addr
;
439 req
.ip_dst_mask_len
=
440 inet_mask_len(l3_mask
->ipv4
.daddr
.s_addr
);
441 req
.ip_src
[0] = l3_key
->ipv4
.saddr
.s_addr
;
442 req
.ip_src_mask_len
=
443 inet_mask_len(l3_mask
->ipv4
.saddr
.s_addr
);
444 } else if (flow
->flags
& BNXT_TC_FLOW_FLAGS_IPV6_ADDRS
) {
445 memcpy(req
.ip_dst
, l3_key
->ipv6
.daddr
.s6_addr32
,
447 req
.ip_dst_mask_len
=
448 ipv6_mask_len(&l3_mask
->ipv6
.daddr
);
449 memcpy(req
.ip_src
, l3_key
->ipv6
.saddr
.s6_addr32
,
451 req
.ip_src_mask_len
=
452 ipv6_mask_len(&l3_mask
->ipv6
.saddr
);
456 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_PORTS
) {
457 req
.l4_src_port
= flow
->l4_key
.ports
.sport
;
458 req
.l4_src_port_mask
= flow
->l4_mask
.ports
.sport
;
459 req
.l4_dst_port
= flow
->l4_key
.ports
.dport
;
460 req
.l4_dst_port_mask
= flow
->l4_mask
.ports
.dport
;
461 } else if (flow
->flags
& BNXT_TC_FLOW_FLAGS_ICMP
) {
462 /* l4 ports serve as type/code when ip_proto is ICMP */
463 req
.l4_src_port
= htons(flow
->l4_key
.icmp
.type
);
464 req
.l4_src_port_mask
= htons(flow
->l4_mask
.icmp
.type
);
465 req
.l4_dst_port
= htons(flow
->l4_key
.icmp
.code
);
466 req
.l4_dst_port_mask
= htons(flow
->l4_mask
.icmp
.code
);
468 req
.flags
= cpu_to_le16(flow_flags
);
470 if (actions
->flags
& BNXT_TC_ACTION_FLAG_DROP
) {
471 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP
;
473 if (actions
->flags
& BNXT_TC_ACTION_FLAG_FWD
) {
474 action_flags
|= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD
;
475 req
.dst_fid
= cpu_to_le16(actions
->dst_fid
);
477 if (actions
->flags
& BNXT_TC_ACTION_FLAG_PUSH_VLAN
) {
479 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE
;
480 req
.l2_rewrite_vlan_tpid
= actions
->push_vlan_tpid
;
481 req
.l2_rewrite_vlan_tci
= actions
->push_vlan_tci
;
482 memcpy(&req
.l2_rewrite_dmac
, &req
.dmac
, ETH_ALEN
);
483 memcpy(&req
.l2_rewrite_smac
, &req
.smac
, ETH_ALEN
);
485 if (actions
->flags
& BNXT_TC_ACTION_FLAG_POP_VLAN
) {
487 CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE
;
488 /* Rewrite config with tpid = 0 implies vlan pop */
489 req
.l2_rewrite_vlan_tpid
= 0;
490 memcpy(&req
.l2_rewrite_dmac
, &req
.dmac
, ETH_ALEN
);
491 memcpy(&req
.l2_rewrite_smac
, &req
.smac
, ETH_ALEN
);
494 req
.action_flags
= cpu_to_le16(action_flags
);
496 mutex_lock(&bp
->hwrm_cmd_lock
);
498 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
500 *flow_handle
= resp
->flow_handle
;
502 mutex_unlock(&bp
->hwrm_cmd_lock
);
507 static int hwrm_cfa_decap_filter_alloc(struct bnxt
*bp
,
508 struct bnxt_tc_flow
*flow
,
509 struct bnxt_tc_l2_key
*l2_info
,
510 __le32 ref_decap_handle
,
511 __le32
*decap_filter_handle
)
513 struct hwrm_cfa_decap_filter_alloc_output
*resp
=
514 bp
->hwrm_cmd_resp_addr
;
515 struct hwrm_cfa_decap_filter_alloc_input req
= { 0 };
516 struct ip_tunnel_key
*tun_key
= &flow
->tun_key
;
520 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_DECAP_FILTER_ALLOC
, -1, -1);
522 req
.flags
= cpu_to_le32(CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL
);
523 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
|
524 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL
;
525 req
.tunnel_type
= CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN
;
526 req
.ip_protocol
= CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP
;
528 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_ID
) {
529 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID
;
530 /* tunnel_id is wrongly defined in hsi defn. as __le32 */
531 req
.tunnel_id
= tunnel_id_to_key32(tun_key
->tun_id
);
534 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS
) {
535 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR
|
536 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR
;
537 ether_addr_copy(req
.dst_macaddr
, l2_info
->dmac
);
538 ether_addr_copy(req
.src_macaddr
, l2_info
->smac
);
540 if (l2_info
->num_vlans
) {
541 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID
;
542 req
.t_ivlan_vid
= l2_info
->inner_vlan_tci
;
545 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE
;
546 req
.ethertype
= htons(ETH_P_IP
);
548 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS
) {
549 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR
|
550 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR
|
551 CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE
;
552 req
.ip_addr_type
= CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4
;
553 req
.dst_ipaddr
[0] = tun_key
->u
.ipv4
.dst
;
554 req
.src_ipaddr
[0] = tun_key
->u
.ipv4
.src
;
557 if (flow
->flags
& BNXT_TC_FLOW_FLAGS_TUNL_PORTS
) {
558 enables
|= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT
;
559 req
.dst_port
= tun_key
->tp_dst
;
562 /* Eventhough the decap_handle returned by hwrm_cfa_decap_filter_alloc
563 * is defined as __le32, l2_ctxt_ref_id is defined in HSI as __le16.
565 req
.l2_ctxt_ref_id
= (__force __le16
)ref_decap_handle
;
566 req
.enables
= cpu_to_le32(enables
);
568 mutex_lock(&bp
->hwrm_cmd_lock
);
569 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
571 *decap_filter_handle
= resp
->decap_filter_id
;
573 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
574 mutex_unlock(&bp
->hwrm_cmd_lock
);
579 static int hwrm_cfa_decap_filter_free(struct bnxt
*bp
,
580 __le32 decap_filter_handle
)
582 struct hwrm_cfa_decap_filter_free_input req
= { 0 };
585 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_DECAP_FILTER_FREE
, -1, -1);
586 req
.decap_filter_id
= decap_filter_handle
;
588 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
590 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
594 static int hwrm_cfa_encap_record_alloc(struct bnxt
*bp
,
595 struct ip_tunnel_key
*encap_key
,
596 struct bnxt_tc_l2_key
*l2_info
,
597 __le32
*encap_record_handle
)
599 struct hwrm_cfa_encap_record_alloc_output
*resp
=
600 bp
->hwrm_cmd_resp_addr
;
601 struct hwrm_cfa_encap_record_alloc_input req
= { 0 };
602 struct hwrm_cfa_encap_data_vxlan
*encap
=
603 (struct hwrm_cfa_encap_data_vxlan
*)&req
.encap_data
;
604 struct hwrm_vxlan_ipv4_hdr
*encap_ipv4
=
605 (struct hwrm_vxlan_ipv4_hdr
*)encap
->l3
;
608 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_ENCAP_RECORD_ALLOC
, -1, -1);
610 req
.encap_type
= CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN
;
612 ether_addr_copy(encap
->dst_mac_addr
, l2_info
->dmac
);
613 ether_addr_copy(encap
->src_mac_addr
, l2_info
->smac
);
614 if (l2_info
->num_vlans
) {
615 encap
->num_vlan_tags
= l2_info
->num_vlans
;
616 encap
->ovlan_tci
= l2_info
->inner_vlan_tci
;
617 encap
->ovlan_tpid
= l2_info
->inner_vlan_tpid
;
620 encap_ipv4
->ver_hlen
= 4 << VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT
;
621 encap_ipv4
->ver_hlen
|= 5 << VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT
;
622 encap_ipv4
->ttl
= encap_key
->ttl
;
624 encap_ipv4
->dest_ip_addr
= encap_key
->u
.ipv4
.dst
;
625 encap_ipv4
->src_ip_addr
= encap_key
->u
.ipv4
.src
;
626 encap_ipv4
->protocol
= IPPROTO_UDP
;
628 encap
->dst_port
= encap_key
->tp_dst
;
629 encap
->vni
= tunnel_id_to_key32(encap_key
->tun_id
);
631 mutex_lock(&bp
->hwrm_cmd_lock
);
632 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
634 *encap_record_handle
= resp
->encap_record_id
;
636 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
637 mutex_unlock(&bp
->hwrm_cmd_lock
);
642 static int hwrm_cfa_encap_record_free(struct bnxt
*bp
,
643 __le32 encap_record_handle
)
645 struct hwrm_cfa_encap_record_free_input req
= { 0 };
648 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_ENCAP_RECORD_FREE
, -1, -1);
649 req
.encap_record_id
= encap_record_handle
;
651 rc
= hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
653 netdev_info(bp
->dev
, "%s: Error rc=%d", __func__
, rc
);
657 static int bnxt_tc_put_l2_node(struct bnxt
*bp
,
658 struct bnxt_tc_flow_node
*flow_node
)
660 struct bnxt_tc_l2_node
*l2_node
= flow_node
->l2_node
;
661 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
664 /* remove flow_node from the L2 shared flow list */
665 list_del(&flow_node
->l2_list_node
);
666 if (--l2_node
->refcount
== 0) {
667 rc
= rhashtable_remove_fast(&tc_info
->l2_table
, &l2_node
->node
,
668 tc_info
->l2_ht_params
);
671 "Error: %s: rhashtable_remove_fast: %d",
673 kfree_rcu(l2_node
, rcu
);
678 static struct bnxt_tc_l2_node
*
679 bnxt_tc_get_l2_node(struct bnxt
*bp
, struct rhashtable
*l2_table
,
680 struct rhashtable_params ht_params
,
681 struct bnxt_tc_l2_key
*l2_key
)
683 struct bnxt_tc_l2_node
*l2_node
;
686 l2_node
= rhashtable_lookup_fast(l2_table
, l2_key
, ht_params
);
688 l2_node
= kzalloc(sizeof(*l2_node
), GFP_KERNEL
);
694 l2_node
->key
= *l2_key
;
695 rc
= rhashtable_insert_fast(l2_table
, &l2_node
->node
,
698 kfree_rcu(l2_node
, rcu
);
700 "Error: %s: rhashtable_insert_fast: %d",
704 INIT_LIST_HEAD(&l2_node
->common_l2_flows
);
709 /* Get the ref_flow_handle for a flow by checking if there are any other
710 * flows that share the same L2 key as this flow.
713 bnxt_tc_get_ref_flow_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
714 struct bnxt_tc_flow_node
*flow_node
,
715 __le16
*ref_flow_handle
)
717 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
718 struct bnxt_tc_flow_node
*ref_flow_node
;
719 struct bnxt_tc_l2_node
*l2_node
;
721 l2_node
= bnxt_tc_get_l2_node(bp
, &tc_info
->l2_table
,
722 tc_info
->l2_ht_params
,
727 /* If any other flow is using this l2_node, use it's flow_handle
728 * as the ref_flow_handle
730 if (l2_node
->refcount
> 0) {
731 ref_flow_node
= list_first_entry(&l2_node
->common_l2_flows
,
732 struct bnxt_tc_flow_node
,
734 *ref_flow_handle
= ref_flow_node
->flow_handle
;
736 *ref_flow_handle
= cpu_to_le16(0xffff);
739 /* Insert the l2_node into the flow_node so that subsequent flows
740 * with a matching l2 key can use the flow_handle of this flow
741 * as their ref_flow_handle
743 flow_node
->l2_node
= l2_node
;
744 list_add(&flow_node
->l2_list_node
, &l2_node
->common_l2_flows
);
749 /* After the flow parsing is done, this routine is used for checking
750 * if there are any aspects of the flow that prevent it from being
753 static bool bnxt_tc_can_offload(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
)
755 /* If L4 ports are specified then ip_proto must be TCP or UDP */
756 if ((flow
->flags
& BNXT_TC_FLOW_FLAGS_PORTS
) &&
757 (flow
->l4_key
.ip_proto
!= IPPROTO_TCP
&&
758 flow
->l4_key
.ip_proto
!= IPPROTO_UDP
)) {
759 netdev_info(bp
->dev
, "Cannot offload non-TCP/UDP (%d) ports",
760 flow
->l4_key
.ip_proto
);
767 /* Returns the final refcount of the node on success
768 * or a -ve error code on failure
770 static int bnxt_tc_put_tunnel_node(struct bnxt
*bp
,
771 struct rhashtable
*tunnel_table
,
772 struct rhashtable_params
*ht_params
,
773 struct bnxt_tc_tunnel_node
*tunnel_node
)
777 if (--tunnel_node
->refcount
== 0) {
778 rc
= rhashtable_remove_fast(tunnel_table
, &tunnel_node
->node
,
781 netdev_err(bp
->dev
, "rhashtable_remove_fast rc=%d", rc
);
784 kfree_rcu(tunnel_node
, rcu
);
787 return tunnel_node
->refcount
;
791 /* Get (or add) either encap or decap tunnel node from/to the supplied
794 static struct bnxt_tc_tunnel_node
*
795 bnxt_tc_get_tunnel_node(struct bnxt
*bp
, struct rhashtable
*tunnel_table
,
796 struct rhashtable_params
*ht_params
,
797 struct ip_tunnel_key
*tun_key
)
799 struct bnxt_tc_tunnel_node
*tunnel_node
;
802 tunnel_node
= rhashtable_lookup_fast(tunnel_table
, tun_key
, *ht_params
);
804 tunnel_node
= kzalloc(sizeof(*tunnel_node
), GFP_KERNEL
);
810 tunnel_node
->key
= *tun_key
;
811 tunnel_node
->tunnel_handle
= INVALID_TUNNEL_HANDLE
;
812 rc
= rhashtable_insert_fast(tunnel_table
, &tunnel_node
->node
,
815 kfree_rcu(tunnel_node
, rcu
);
819 tunnel_node
->refcount
++;
822 netdev_info(bp
->dev
, "error rc=%d", rc
);
826 static int bnxt_tc_get_ref_decap_handle(struct bnxt
*bp
,
827 struct bnxt_tc_flow
*flow
,
828 struct bnxt_tc_l2_key
*l2_key
,
829 struct bnxt_tc_flow_node
*flow_node
,
830 __le32
*ref_decap_handle
)
832 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
833 struct bnxt_tc_flow_node
*ref_flow_node
;
834 struct bnxt_tc_l2_node
*decap_l2_node
;
836 decap_l2_node
= bnxt_tc_get_l2_node(bp
, &tc_info
->decap_l2_table
,
837 tc_info
->decap_l2_ht_params
,
842 /* If any other flow is using this decap_l2_node, use it's decap_handle
843 * as the ref_decap_handle
845 if (decap_l2_node
->refcount
> 0) {
847 list_first_entry(&decap_l2_node
->common_l2_flows
,
848 struct bnxt_tc_flow_node
,
850 *ref_decap_handle
= ref_flow_node
->decap_node
->tunnel_handle
;
852 *ref_decap_handle
= INVALID_TUNNEL_HANDLE
;
855 /* Insert the l2_node into the flow_node so that subsequent flows
856 * with a matching decap l2 key can use the decap_filter_handle of
857 * this flow as their ref_decap_handle
859 flow_node
->decap_l2_node
= decap_l2_node
;
860 list_add(&flow_node
->decap_l2_list_node
,
861 &decap_l2_node
->common_l2_flows
);
862 decap_l2_node
->refcount
++;
866 static void bnxt_tc_put_decap_l2_node(struct bnxt
*bp
,
867 struct bnxt_tc_flow_node
*flow_node
)
869 struct bnxt_tc_l2_node
*decap_l2_node
= flow_node
->decap_l2_node
;
870 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
873 /* remove flow_node from the decap L2 sharing flow list */
874 list_del(&flow_node
->decap_l2_list_node
);
875 if (--decap_l2_node
->refcount
== 0) {
876 rc
= rhashtable_remove_fast(&tc_info
->decap_l2_table
,
877 &decap_l2_node
->node
,
878 tc_info
->decap_l2_ht_params
);
880 netdev_err(bp
->dev
, "rhashtable_remove_fast rc=%d", rc
);
881 kfree_rcu(decap_l2_node
, rcu
);
885 static void bnxt_tc_put_decap_handle(struct bnxt
*bp
,
886 struct bnxt_tc_flow_node
*flow_node
)
888 __le32 decap_handle
= flow_node
->decap_node
->tunnel_handle
;
889 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
892 if (flow_node
->decap_l2_node
)
893 bnxt_tc_put_decap_l2_node(bp
, flow_node
);
895 rc
= bnxt_tc_put_tunnel_node(bp
, &tc_info
->decap_table
,
896 &tc_info
->decap_ht_params
,
897 flow_node
->decap_node
);
898 if (!rc
&& decap_handle
!= INVALID_TUNNEL_HANDLE
)
899 hwrm_cfa_decap_filter_free(bp
, decap_handle
);
902 static int bnxt_tc_resolve_tunnel_hdrs(struct bnxt
*bp
,
903 struct ip_tunnel_key
*tun_key
,
904 struct bnxt_tc_l2_key
*l2_info
,
905 struct net_device
*real_dst_dev
)
908 struct flowi4 flow
= { {0} };
909 struct net_device
*dst_dev
;
910 struct neighbour
*nbr
;
914 flow
.flowi4_proto
= IPPROTO_UDP
;
915 flow
.fl4_dport
= tun_key
->tp_dst
;
916 flow
.daddr
= tun_key
->u
.ipv4
.dst
;
918 rt
= ip_route_output_key(dev_net(real_dst_dev
), &flow
);
920 netdev_info(bp
->dev
, "no route to %pI4b", &flow
.daddr
);
924 /* The route must either point to the real_dst_dev or a dst_dev that
925 * uses the real_dst_dev.
927 dst_dev
= rt
->dst
.dev
;
928 if (is_vlan_dev(dst_dev
)) {
929 #if IS_ENABLED(CONFIG_VLAN_8021Q)
930 struct vlan_dev_priv
*vlan
= vlan_dev_priv(dst_dev
);
932 if (vlan
->real_dev
!= real_dst_dev
) {
934 "dst_dev(%s) doesn't use PF-if(%s)",
935 netdev_name(dst_dev
),
936 netdev_name(real_dst_dev
));
940 l2_info
->inner_vlan_tci
= htons(vlan
->vlan_id
);
941 l2_info
->inner_vlan_tpid
= vlan
->vlan_proto
;
942 l2_info
->num_vlans
= 1;
944 } else if (dst_dev
!= real_dst_dev
) {
946 "dst_dev(%s) for %pI4b is not PF-if(%s)",
947 netdev_name(dst_dev
), &flow
.daddr
,
948 netdev_name(real_dst_dev
));
953 nbr
= dst_neigh_lookup(&rt
->dst
, &flow
.daddr
);
955 netdev_info(bp
->dev
, "can't lookup neighbor for %pI4b",
961 tun_key
->u
.ipv4
.src
= flow
.saddr
;
962 tun_key
->ttl
= ip4_dst_hoplimit(&rt
->dst
);
963 neigh_ha_snapshot(l2_info
->dmac
, nbr
, dst_dev
);
964 ether_addr_copy(l2_info
->smac
, dst_dev
->dev_addr
);
977 static int bnxt_tc_get_decap_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
978 struct bnxt_tc_flow_node
*flow_node
,
979 __le32
*decap_filter_handle
)
981 struct ip_tunnel_key
*decap_key
= &flow
->tun_key
;
982 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
983 struct bnxt_tc_l2_key l2_info
= { {0} };
984 struct bnxt_tc_tunnel_node
*decap_node
;
985 struct ip_tunnel_key tun_key
= { 0 };
986 struct bnxt_tc_l2_key
*decap_l2_info
;
987 __le32 ref_decap_handle
;
990 /* Check if there's another flow using the same tunnel decap.
991 * If not, add this tunnel to the table and resolve the other
992 * tunnel header fileds
994 decap_node
= bnxt_tc_get_tunnel_node(bp
, &tc_info
->decap_table
,
995 &tc_info
->decap_ht_params
,
1000 flow_node
->decap_node
= decap_node
;
1002 if (decap_node
->tunnel_handle
!= INVALID_TUNNEL_HANDLE
)
1005 /* Resolve the L2 fields for tunnel decap
1006 * Resolve the route for remote vtep (saddr) of the decap key
1007 * Find it's next-hop mac addrs
1009 tun_key
.u
.ipv4
.dst
= flow
->tun_key
.u
.ipv4
.src
;
1010 tun_key
.tp_dst
= flow
->tun_key
.tp_dst
;
1011 rc
= bnxt_tc_resolve_tunnel_hdrs(bp
, &tun_key
, &l2_info
, bp
->dev
);
1015 decap_key
->ttl
= tun_key
.ttl
;
1016 decap_l2_info
= &decap_node
->l2_info
;
1017 ether_addr_copy(decap_l2_info
->dmac
, l2_info
.smac
);
1018 ether_addr_copy(decap_l2_info
->smac
, l2_info
.dmac
);
1019 if (l2_info
.num_vlans
) {
1020 decap_l2_info
->num_vlans
= l2_info
.num_vlans
;
1021 decap_l2_info
->inner_vlan_tpid
= l2_info
.inner_vlan_tpid
;
1022 decap_l2_info
->inner_vlan_tci
= l2_info
.inner_vlan_tci
;
1024 flow
->flags
|= BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS
;
1026 /* For getting a decap_filter_handle we first need to check if
1027 * there are any other decap flows that share the same tunnel L2
1028 * key and if so, pass that flow's decap_filter_handle as the
1029 * ref_decap_handle for this flow.
1031 rc
= bnxt_tc_get_ref_decap_handle(bp
, flow
, decap_l2_info
, flow_node
,
1036 /* Issue the hwrm cmd to allocate a decap filter handle */
1037 rc
= hwrm_cfa_decap_filter_alloc(bp
, flow
, decap_l2_info
,
1039 &decap_node
->tunnel_handle
);
1044 *decap_filter_handle
= decap_node
->tunnel_handle
;
1048 bnxt_tc_put_decap_l2_node(bp
, flow_node
);
1050 bnxt_tc_put_tunnel_node(bp
, &tc_info
->decap_table
,
1051 &tc_info
->decap_ht_params
,
1052 flow_node
->decap_node
);
1056 static void bnxt_tc_put_encap_handle(struct bnxt
*bp
,
1057 struct bnxt_tc_tunnel_node
*encap_node
)
1059 __le32 encap_handle
= encap_node
->tunnel_handle
;
1060 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1063 rc
= bnxt_tc_put_tunnel_node(bp
, &tc_info
->encap_table
,
1064 &tc_info
->encap_ht_params
, encap_node
);
1065 if (!rc
&& encap_handle
!= INVALID_TUNNEL_HANDLE
)
1066 hwrm_cfa_encap_record_free(bp
, encap_handle
);
1069 /* Lookup the tunnel encap table and check if there's an encap_handle
1071 * If not, query L2 info via a route lookup and issue an encap_record_alloc
1074 static int bnxt_tc_get_encap_handle(struct bnxt
*bp
, struct bnxt_tc_flow
*flow
,
1075 struct bnxt_tc_flow_node
*flow_node
,
1076 __le32
*encap_handle
)
1078 struct ip_tunnel_key
*encap_key
= &flow
->actions
.tun_encap_key
;
1079 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1080 struct bnxt_tc_tunnel_node
*encap_node
;
1083 /* Check if there's another flow using the same tunnel encap.
1084 * If not, add this tunnel to the table and resolve the other
1085 * tunnel header fileds
1087 encap_node
= bnxt_tc_get_tunnel_node(bp
, &tc_info
->encap_table
,
1088 &tc_info
->encap_ht_params
,
1093 flow_node
->encap_node
= encap_node
;
1095 if (encap_node
->tunnel_handle
!= INVALID_TUNNEL_HANDLE
)
1098 rc
= bnxt_tc_resolve_tunnel_hdrs(bp
, encap_key
, &encap_node
->l2_info
,
1099 flow
->actions
.dst_dev
);
1103 /* Allocate a new tunnel encap record */
1104 rc
= hwrm_cfa_encap_record_alloc(bp
, encap_key
, &encap_node
->l2_info
,
1105 &encap_node
->tunnel_handle
);
1110 *encap_handle
= encap_node
->tunnel_handle
;
1114 bnxt_tc_put_tunnel_node(bp
, &tc_info
->encap_table
,
1115 &tc_info
->encap_ht_params
, encap_node
);
1119 static void bnxt_tc_put_tunnel_handle(struct bnxt
*bp
,
1120 struct bnxt_tc_flow
*flow
,
1121 struct bnxt_tc_flow_node
*flow_node
)
1123 if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
)
1124 bnxt_tc_put_decap_handle(bp
, flow_node
);
1125 else if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
)
1126 bnxt_tc_put_encap_handle(bp
, flow_node
->encap_node
);
1129 static int bnxt_tc_get_tunnel_handle(struct bnxt
*bp
,
1130 struct bnxt_tc_flow
*flow
,
1131 struct bnxt_tc_flow_node
*flow_node
,
1132 __le32
*tunnel_handle
)
1134 if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_DECAP
)
1135 return bnxt_tc_get_decap_handle(bp
, flow
, flow_node
,
1137 else if (flow
->actions
.flags
& BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP
)
1138 return bnxt_tc_get_encap_handle(bp
, flow
, flow_node
,
1143 static int __bnxt_tc_del_flow(struct bnxt
*bp
,
1144 struct bnxt_tc_flow_node
*flow_node
)
1146 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1149 /* send HWRM cmd to free the flow-id */
1150 bnxt_hwrm_cfa_flow_free(bp
, flow_node
->flow_handle
);
1152 mutex_lock(&tc_info
->lock
);
1154 /* release references to any tunnel encap/decap nodes */
1155 bnxt_tc_put_tunnel_handle(bp
, &flow_node
->flow
, flow_node
);
1157 /* release reference to l2 node */
1158 bnxt_tc_put_l2_node(bp
, flow_node
);
1160 mutex_unlock(&tc_info
->lock
);
1162 rc
= rhashtable_remove_fast(&tc_info
->flow_table
, &flow_node
->node
,
1163 tc_info
->flow_ht_params
);
1165 netdev_err(bp
->dev
, "Error: %s: rhashtable_remove_fast rc=%d",
1168 kfree_rcu(flow_node
, rcu
);
1172 /* Add a new flow or replace an existing flow.
1174 * There are essentially two critical sections here.
1175 * 1. while adding a new flow
1177 * b) issue HWRM cmd and get flow_handle
1178 * c) link l2-key with flow
1179 * 2. while deleting a flow
1180 * a) unlinking l2-key from flow
1181 * A lock is needed to protect these two critical sections.
1183 * The hash-tables are already protected by the rhashtable API.
1185 static int bnxt_tc_add_flow(struct bnxt
*bp
, u16 src_fid
,
1186 struct tc_cls_flower_offload
*tc_flow_cmd
)
1188 struct bnxt_tc_flow_node
*new_node
, *old_node
;
1189 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1190 struct bnxt_tc_flow
*flow
;
1191 __le32 tunnel_handle
= 0;
1192 __le16 ref_flow_handle
;
1195 /* allocate memory for the new flow and it's node */
1196 new_node
= kzalloc(sizeof(*new_node
), GFP_KERNEL
);
1201 new_node
->cookie
= tc_flow_cmd
->cookie
;
1202 flow
= &new_node
->flow
;
1204 rc
= bnxt_tc_parse_flow(bp
, tc_flow_cmd
, flow
);
1207 flow
->src_fid
= src_fid
;
1209 if (!bnxt_tc_can_offload(bp
, flow
)) {
1214 /* If a flow exists with the same cookie, delete it */
1215 old_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
1216 &tc_flow_cmd
->cookie
,
1217 tc_info
->flow_ht_params
);
1219 __bnxt_tc_del_flow(bp
, old_node
);
1221 /* Check if the L2 part of the flow has been offloaded already.
1222 * If so, bump up it's refcnt and get it's reference handle.
1224 mutex_lock(&tc_info
->lock
);
1225 rc
= bnxt_tc_get_ref_flow_handle(bp
, flow
, new_node
, &ref_flow_handle
);
1229 /* If the flow involves tunnel encap/decap, get tunnel_handle */
1230 rc
= bnxt_tc_get_tunnel_handle(bp
, flow
, new_node
, &tunnel_handle
);
1234 /* send HWRM cmd to alloc the flow */
1235 rc
= bnxt_hwrm_cfa_flow_alloc(bp
, flow
, ref_flow_handle
,
1236 tunnel_handle
, &new_node
->flow_handle
);
1240 flow
->lastused
= jiffies
;
1241 spin_lock_init(&flow
->stats_lock
);
1242 /* add new flow to flow-table */
1243 rc
= rhashtable_insert_fast(&tc_info
->flow_table
, &new_node
->node
,
1244 tc_info
->flow_ht_params
);
1246 goto hwrm_flow_free
;
1248 mutex_unlock(&tc_info
->lock
);
1252 bnxt_hwrm_cfa_flow_free(bp
, new_node
->flow_handle
);
1254 bnxt_tc_put_tunnel_handle(bp
, flow
, new_node
);
1256 bnxt_tc_put_l2_node(bp
, new_node
);
1258 mutex_unlock(&tc_info
->lock
);
1260 kfree_rcu(new_node
, rcu
);
1262 netdev_err(bp
->dev
, "Error: %s: cookie=0x%lx error=%d",
1263 __func__
, tc_flow_cmd
->cookie
, rc
);
1267 static int bnxt_tc_del_flow(struct bnxt
*bp
,
1268 struct tc_cls_flower_offload
*tc_flow_cmd
)
1270 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1271 struct bnxt_tc_flow_node
*flow_node
;
1273 flow_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
1274 &tc_flow_cmd
->cookie
,
1275 tc_info
->flow_ht_params
);
1277 netdev_info(bp
->dev
, "ERROR: no flow_node for cookie %lx",
1278 tc_flow_cmd
->cookie
);
1282 return __bnxt_tc_del_flow(bp
, flow_node
);
1285 static int bnxt_tc_get_flow_stats(struct bnxt
*bp
,
1286 struct tc_cls_flower_offload
*tc_flow_cmd
)
1288 struct bnxt_tc_flow_stats stats
, *curr_stats
, *prev_stats
;
1289 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1290 struct bnxt_tc_flow_node
*flow_node
;
1291 struct bnxt_tc_flow
*flow
;
1292 unsigned long lastused
;
1294 flow_node
= rhashtable_lookup_fast(&tc_info
->flow_table
,
1295 &tc_flow_cmd
->cookie
,
1296 tc_info
->flow_ht_params
);
1298 netdev_info(bp
->dev
, "Error: no flow_node for cookie %lx",
1299 tc_flow_cmd
->cookie
);
1303 flow
= &flow_node
->flow
;
1304 curr_stats
= &flow
->stats
;
1305 prev_stats
= &flow
->prev_stats
;
1307 spin_lock(&flow
->stats_lock
);
1308 stats
.packets
= curr_stats
->packets
- prev_stats
->packets
;
1309 stats
.bytes
= curr_stats
->bytes
- prev_stats
->bytes
;
1310 *prev_stats
= *curr_stats
;
1311 lastused
= flow
->lastused
;
1312 spin_unlock(&flow
->stats_lock
);
1314 tcf_exts_stats_update(tc_flow_cmd
->exts
, stats
.bytes
, stats
.packets
,
1320 bnxt_hwrm_cfa_flow_stats_get(struct bnxt
*bp
, int num_flows
,
1321 struct bnxt_tc_stats_batch stats_batch
[])
1323 struct hwrm_cfa_flow_stats_output
*resp
= bp
->hwrm_cmd_resp_addr
;
1324 struct hwrm_cfa_flow_stats_input req
= { 0 };
1325 __le16
*req_flow_handles
= &req
.flow_handle_0
;
1328 bnxt_hwrm_cmd_hdr_init(bp
, &req
, HWRM_CFA_FLOW_STATS
, -1, -1);
1329 req
.num_flows
= cpu_to_le16(num_flows
);
1330 for (i
= 0; i
< num_flows
; i
++) {
1331 struct bnxt_tc_flow_node
*flow_node
= stats_batch
[i
].flow_node
;
1333 req_flow_handles
[i
] = flow_node
->flow_handle
;
1336 mutex_lock(&bp
->hwrm_cmd_lock
);
1337 rc
= _hwrm_send_message(bp
, &req
, sizeof(req
), HWRM_CMD_TIMEOUT
);
1339 __le64
*resp_packets
= &resp
->packet_0
;
1340 __le64
*resp_bytes
= &resp
->byte_0
;
1342 for (i
= 0; i
< num_flows
; i
++) {
1343 stats_batch
[i
].hw_stats
.packets
=
1344 le64_to_cpu(resp_packets
[i
]);
1345 stats_batch
[i
].hw_stats
.bytes
=
1346 le64_to_cpu(resp_bytes
[i
]);
1349 netdev_info(bp
->dev
, "error rc=%d", rc
);
1352 mutex_unlock(&bp
->hwrm_cmd_lock
);
1356 /* Add val to accum while handling a possible wraparound
1357 * of val. Eventhough val is of type u64, its actual width
1358 * is denoted by mask and will wrap-around beyond that width.
1360 static void accumulate_val(u64
*accum
, u64 val
, u64 mask
)
1362 #define low_bits(x, mask) ((x) & (mask))
1363 #define high_bits(x, mask) ((x) & ~(mask))
1364 bool wrapped
= val
< low_bits(*accum
, mask
);
1366 *accum
= high_bits(*accum
, mask
) + val
;
1368 *accum
+= (mask
+ 1);
1371 /* The HW counters' width is much less than 64bits.
1372 * Handle possible wrap-around while updating the stat counters
1374 static void bnxt_flow_stats_accum(struct bnxt_tc_info
*tc_info
,
1375 struct bnxt_tc_flow_stats
*acc_stats
,
1376 struct bnxt_tc_flow_stats
*hw_stats
)
1378 accumulate_val(&acc_stats
->bytes
, hw_stats
->bytes
, tc_info
->bytes_mask
);
1379 accumulate_val(&acc_stats
->packets
, hw_stats
->packets
,
1380 tc_info
->packets_mask
);
1384 bnxt_tc_flow_stats_batch_update(struct bnxt
*bp
, int num_flows
,
1385 struct bnxt_tc_stats_batch stats_batch
[])
1387 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1390 rc
= bnxt_hwrm_cfa_flow_stats_get(bp
, num_flows
, stats_batch
);
1394 for (i
= 0; i
< num_flows
; i
++) {
1395 struct bnxt_tc_flow_node
*flow_node
= stats_batch
[i
].flow_node
;
1396 struct bnxt_tc_flow
*flow
= &flow_node
->flow
;
1398 spin_lock(&flow
->stats_lock
);
1399 bnxt_flow_stats_accum(tc_info
, &flow
->stats
,
1400 &stats_batch
[i
].hw_stats
);
1401 if (flow
->stats
.packets
!= flow
->prev_stats
.packets
)
1402 flow
->lastused
= jiffies
;
1403 spin_unlock(&flow
->stats_lock
);
1410 bnxt_tc_flow_stats_batch_prep(struct bnxt
*bp
,
1411 struct bnxt_tc_stats_batch stats_batch
[],
1414 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1415 struct rhashtable_iter
*iter
= &tc_info
->iter
;
1419 rc
= rhashtable_walk_start(iter
);
1420 if (rc
&& rc
!= -EAGAIN
) {
1426 for (i
= 0; i
< BNXT_FLOW_STATS_BATCH_MAX
; i
++) {
1427 flow_node
= rhashtable_walk_next(iter
);
1428 if (IS_ERR(flow_node
)) {
1430 if (PTR_ERR(flow_node
) == -EAGAIN
) {
1433 rc
= PTR_ERR(flow_node
);
1442 stats_batch
[i
].flow_node
= flow_node
;
1445 rhashtable_walk_stop(iter
);
1450 void bnxt_tc_flow_stats_work(struct bnxt
*bp
)
1452 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1455 num_flows
= atomic_read(&tc_info
->flow_table
.nelems
);
1459 rhashtable_walk_enter(&tc_info
->flow_table
, &tc_info
->iter
);
1462 rc
= bnxt_tc_flow_stats_batch_prep(bp
, tc_info
->stats_batch
,
1473 bnxt_tc_flow_stats_batch_update(bp
, num_flows
,
1474 tc_info
->stats_batch
);
1477 rhashtable_walk_exit(&tc_info
->iter
);
1480 int bnxt_tc_setup_flower(struct bnxt
*bp
, u16 src_fid
,
1481 struct tc_cls_flower_offload
*cls_flower
)
1485 if (cls_flower
->common
.chain_index
)
1488 switch (cls_flower
->command
) {
1489 case TC_CLSFLOWER_REPLACE
:
1490 rc
= bnxt_tc_add_flow(bp
, src_fid
, cls_flower
);
1493 case TC_CLSFLOWER_DESTROY
:
1494 rc
= bnxt_tc_del_flow(bp
, cls_flower
);
1497 case TC_CLSFLOWER_STATS
:
1498 rc
= bnxt_tc_get_flow_stats(bp
, cls_flower
);
1504 static const struct rhashtable_params bnxt_tc_flow_ht_params
= {
1505 .head_offset
= offsetof(struct bnxt_tc_flow_node
, node
),
1506 .key_offset
= offsetof(struct bnxt_tc_flow_node
, cookie
),
1507 .key_len
= sizeof(((struct bnxt_tc_flow_node
*)0)->cookie
),
1508 .automatic_shrinking
= true
1511 static const struct rhashtable_params bnxt_tc_l2_ht_params
= {
1512 .head_offset
= offsetof(struct bnxt_tc_l2_node
, node
),
1513 .key_offset
= offsetof(struct bnxt_tc_l2_node
, key
),
1514 .key_len
= BNXT_TC_L2_KEY_LEN
,
1515 .automatic_shrinking
= true
1518 static const struct rhashtable_params bnxt_tc_decap_l2_ht_params
= {
1519 .head_offset
= offsetof(struct bnxt_tc_l2_node
, node
),
1520 .key_offset
= offsetof(struct bnxt_tc_l2_node
, key
),
1521 .key_len
= BNXT_TC_L2_KEY_LEN
,
1522 .automatic_shrinking
= true
1525 static const struct rhashtable_params bnxt_tc_tunnel_ht_params
= {
1526 .head_offset
= offsetof(struct bnxt_tc_tunnel_node
, node
),
1527 .key_offset
= offsetof(struct bnxt_tc_tunnel_node
, key
),
1528 .key_len
= sizeof(struct ip_tunnel_key
),
1529 .automatic_shrinking
= true
1532 /* convert counter width in bits to a mask */
1533 #define mask(width) ((u64)~0 >> (64 - (width)))
1535 int bnxt_init_tc(struct bnxt
*bp
)
1537 struct bnxt_tc_info
*tc_info
;
1540 if (bp
->hwrm_spec_code
< 0x10803) {
1541 netdev_warn(bp
->dev
,
1542 "Firmware does not support TC flower offload.\n");
1546 tc_info
= kzalloc(sizeof(*tc_info
), GFP_KERNEL
);
1549 mutex_init(&tc_info
->lock
);
1551 /* Counter widths are programmed by FW */
1552 tc_info
->bytes_mask
= mask(36);
1553 tc_info
->packets_mask
= mask(28);
1555 tc_info
->flow_ht_params
= bnxt_tc_flow_ht_params
;
1556 rc
= rhashtable_init(&tc_info
->flow_table
, &tc_info
->flow_ht_params
);
1560 tc_info
->l2_ht_params
= bnxt_tc_l2_ht_params
;
1561 rc
= rhashtable_init(&tc_info
->l2_table
, &tc_info
->l2_ht_params
);
1563 goto destroy_flow_table
;
1565 tc_info
->decap_l2_ht_params
= bnxt_tc_decap_l2_ht_params
;
1566 rc
= rhashtable_init(&tc_info
->decap_l2_table
,
1567 &tc_info
->decap_l2_ht_params
);
1569 goto destroy_l2_table
;
1571 tc_info
->decap_ht_params
= bnxt_tc_tunnel_ht_params
;
1572 rc
= rhashtable_init(&tc_info
->decap_table
,
1573 &tc_info
->decap_ht_params
);
1575 goto destroy_decap_l2_table
;
1577 tc_info
->encap_ht_params
= bnxt_tc_tunnel_ht_params
;
1578 rc
= rhashtable_init(&tc_info
->encap_table
,
1579 &tc_info
->encap_ht_params
);
1581 goto destroy_decap_table
;
1583 tc_info
->enabled
= true;
1584 bp
->dev
->hw_features
|= NETIF_F_HW_TC
;
1585 bp
->dev
->features
|= NETIF_F_HW_TC
;
1586 bp
->tc_info
= tc_info
;
1589 destroy_decap_table
:
1590 rhashtable_destroy(&tc_info
->decap_table
);
1591 destroy_decap_l2_table
:
1592 rhashtable_destroy(&tc_info
->decap_l2_table
);
1594 rhashtable_destroy(&tc_info
->l2_table
);
1596 rhashtable_destroy(&tc_info
->flow_table
);
1602 void bnxt_shutdown_tc(struct bnxt
*bp
)
1604 struct bnxt_tc_info
*tc_info
= bp
->tc_info
;
1606 if (!bnxt_tc_flower_enabled(bp
))
1609 rhashtable_destroy(&tc_info
->flow_table
);
1610 rhashtable_destroy(&tc_info
->l2_table
);
1611 rhashtable_destroy(&tc_info
->decap_l2_table
);
1612 rhashtable_destroy(&tc_info
->decap_table
);
1613 rhashtable_destroy(&tc_info
->encap_table
);