2 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
10 #include <linux/module.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/skbuff.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/filter.h>
16 #include <linux/bpf.h>
18 #include <net/netlink.h>
19 #include <net/pkt_sched.h>
21 #include <linux/tc_act/tc_bpf.h>
22 #include <net/tc_act/tc_bpf.h>
24 #define BPF_TAB_MASK 15
25 #define ACT_BPF_NAME_LEN 256
28 struct bpf_prog
*filter
;
29 struct sock_filter
*bpf_ops
;
36 static int bpf_net_id
;
37 static struct tc_action_ops act_bpf_ops
;
39 static int tcf_bpf(struct sk_buff
*skb
, const struct tc_action
*act
,
40 struct tcf_result
*res
)
42 bool at_ingress
= skb_at_tc_ingress(skb
);
43 struct tcf_bpf
*prog
= to_bpf(act
);
44 struct bpf_prog
*filter
;
45 int action
, filter_res
;
47 tcf_lastuse_update(&prog
->tcf_tm
);
48 bstats_cpu_update(this_cpu_ptr(prog
->common
.cpu_bstats
), skb
);
51 filter
= rcu_dereference(prog
->filter
);
53 __skb_push(skb
, skb
->mac_len
);
54 bpf_compute_data_end(skb
);
55 filter_res
= BPF_PROG_RUN(filter
, skb
);
56 __skb_pull(skb
, skb
->mac_len
);
58 bpf_compute_data_end(skb
);
59 filter_res
= BPF_PROG_RUN(filter
, skb
);
63 /* A BPF program may overwrite the default action opcode.
64 * Similarly as in cls_bpf, if filter_res == -1 we use the
65 * default action specified from tc.
67 * In case a different well-known TC_ACT opcode has been
68 * returned, it will overwrite the default one.
70 * For everything else that is unkown, TC_ACT_UNSPEC is
75 case TC_ACT_RECLASSIFY
:
82 qstats_drop_inc(this_cpu_ptr(prog
->common
.cpu_qstats
));
85 action
= prog
->tcf_action
;
88 action
= TC_ACT_UNSPEC
;
95 static bool tcf_bpf_is_ebpf(const struct tcf_bpf
*prog
)
97 return !prog
->bpf_ops
;
100 static int tcf_bpf_dump_bpf_info(const struct tcf_bpf
*prog
,
105 if (nla_put_u16(skb
, TCA_ACT_BPF_OPS_LEN
, prog
->bpf_num_ops
))
108 nla
= nla_reserve(skb
, TCA_ACT_BPF_OPS
, prog
->bpf_num_ops
*
109 sizeof(struct sock_filter
));
113 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
118 static int tcf_bpf_dump_ebpf_info(const struct tcf_bpf
*prog
,
121 if (nla_put_u32(skb
, TCA_ACT_BPF_FD
, prog
->bpf_fd
))
124 if (prog
->bpf_name
&&
125 nla_put_string(skb
, TCA_ACT_BPF_NAME
, prog
->bpf_name
))
131 static int tcf_bpf_dump(struct sk_buff
*skb
, struct tc_action
*act
,
134 unsigned char *tp
= skb_tail_pointer(skb
);
135 struct tcf_bpf
*prog
= to_bpf(act
);
136 struct tc_act_bpf opt
= {
137 .index
= prog
->tcf_index
,
138 .refcnt
= prog
->tcf_refcnt
- ref
,
139 .bindcnt
= prog
->tcf_bindcnt
- bind
,
140 .action
= prog
->tcf_action
,
145 if (nla_put(skb
, TCA_ACT_BPF_PARMS
, sizeof(opt
), &opt
))
146 goto nla_put_failure
;
148 if (tcf_bpf_is_ebpf(prog
))
149 ret
= tcf_bpf_dump_ebpf_info(prog
, skb
);
151 ret
= tcf_bpf_dump_bpf_info(prog
, skb
);
153 goto nla_put_failure
;
155 tcf_tm_dump(&tm
, &prog
->tcf_tm
);
156 if (nla_put_64bit(skb
, TCA_ACT_BPF_TM
, sizeof(tm
), &tm
,
158 goto nla_put_failure
;
167 static const struct nla_policy act_bpf_policy
[TCA_ACT_BPF_MAX
+ 1] = {
168 [TCA_ACT_BPF_PARMS
] = { .len
= sizeof(struct tc_act_bpf
) },
169 [TCA_ACT_BPF_FD
] = { .type
= NLA_U32
},
170 [TCA_ACT_BPF_NAME
] = { .type
= NLA_NUL_STRING
,
171 .len
= ACT_BPF_NAME_LEN
},
172 [TCA_ACT_BPF_OPS_LEN
] = { .type
= NLA_U16
},
173 [TCA_ACT_BPF_OPS
] = { .type
= NLA_BINARY
,
174 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
177 static int tcf_bpf_init_from_ops(struct nlattr
**tb
, struct tcf_bpf_cfg
*cfg
)
179 struct sock_filter
*bpf_ops
;
180 struct sock_fprog_kern fprog_tmp
;
182 u16 bpf_size
, bpf_num_ops
;
185 bpf_num_ops
= nla_get_u16(tb
[TCA_ACT_BPF_OPS_LEN
]);
186 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
189 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
190 if (bpf_size
!= nla_len(tb
[TCA_ACT_BPF_OPS
]))
193 bpf_ops
= kzalloc(bpf_size
, GFP_KERNEL
);
197 memcpy(bpf_ops
, nla_data(tb
[TCA_ACT_BPF_OPS
]), bpf_size
);
199 fprog_tmp
.len
= bpf_num_ops
;
200 fprog_tmp
.filter
= bpf_ops
;
202 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
208 cfg
->bpf_ops
= bpf_ops
;
209 cfg
->bpf_num_ops
= bpf_num_ops
;
211 cfg
->is_ebpf
= false;
216 static int tcf_bpf_init_from_efd(struct nlattr
**tb
, struct tcf_bpf_cfg
*cfg
)
222 bpf_fd
= nla_get_u32(tb
[TCA_ACT_BPF_FD
]);
224 fp
= bpf_prog_get_type(bpf_fd
, BPF_PROG_TYPE_SCHED_ACT
);
228 if (tb
[TCA_ACT_BPF_NAME
]) {
229 name
= kmemdup(nla_data(tb
[TCA_ACT_BPF_NAME
]),
230 nla_len(tb
[TCA_ACT_BPF_NAME
]),
238 cfg
->bpf_fd
= bpf_fd
;
239 cfg
->bpf_name
= name
;
246 static void tcf_bpf_cfg_cleanup(const struct tcf_bpf_cfg
*cfg
)
249 bpf_prog_put(cfg
->filter
);
251 bpf_prog_destroy(cfg
->filter
);
254 kfree(cfg
->bpf_name
);
257 static void tcf_bpf_prog_fill_cfg(const struct tcf_bpf
*prog
,
258 struct tcf_bpf_cfg
*cfg
)
260 cfg
->is_ebpf
= tcf_bpf_is_ebpf(prog
);
261 /* updates to prog->filter are prevented, since it's called either
262 * with rtnl lock or during final cleanup in rcu callback
264 cfg
->filter
= rcu_dereference_protected(prog
->filter
, 1);
266 cfg
->bpf_ops
= prog
->bpf_ops
;
267 cfg
->bpf_name
= prog
->bpf_name
;
270 static int tcf_bpf_init(struct net
*net
, struct nlattr
*nla
,
271 struct nlattr
*est
, struct tc_action
**act
,
272 int replace
, int bind
)
274 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
275 struct nlattr
*tb
[TCA_ACT_BPF_MAX
+ 1];
276 struct tcf_bpf_cfg cfg
, old
;
277 struct tc_act_bpf
*parm
;
278 struct tcf_bpf
*prog
;
279 bool is_bpf
, is_ebpf
;
285 ret
= nla_parse_nested(tb
, TCA_ACT_BPF_MAX
, nla
, act_bpf_policy
);
289 if (!tb
[TCA_ACT_BPF_PARMS
])
292 parm
= nla_data(tb
[TCA_ACT_BPF_PARMS
]);
294 if (!tcf_hash_check(tn
, parm
->index
, act
, bind
)) {
295 ret
= tcf_hash_create(tn
, parm
->index
, est
, act
,
296 &act_bpf_ops
, bind
, true);
302 /* Don't override defaults. */
306 tcf_hash_release(*act
, bind
);
311 is_bpf
= tb
[TCA_ACT_BPF_OPS_LEN
] && tb
[TCA_ACT_BPF_OPS
];
312 is_ebpf
= tb
[TCA_ACT_BPF_FD
];
314 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
)) {
319 memset(&cfg
, 0, sizeof(cfg
));
321 ret
= is_bpf
? tcf_bpf_init_from_ops(tb
, &cfg
) :
322 tcf_bpf_init_from_efd(tb
, &cfg
);
329 if (res
!= ACT_P_CREATED
)
330 tcf_bpf_prog_fill_cfg(prog
, &old
);
332 prog
->bpf_ops
= cfg
.bpf_ops
;
333 prog
->bpf_name
= cfg
.bpf_name
;
336 prog
->bpf_num_ops
= cfg
.bpf_num_ops
;
338 prog
->bpf_fd
= cfg
.bpf_fd
;
340 prog
->tcf_action
= parm
->action
;
341 rcu_assign_pointer(prog
->filter
, cfg
.filter
);
343 if (res
== ACT_P_CREATED
) {
344 tcf_hash_insert(tn
, *act
);
346 /* make sure the program being replaced is no longer executing */
348 tcf_bpf_cfg_cleanup(&old
);
353 if (res
== ACT_P_CREATED
)
354 tcf_hash_cleanup(*act
, est
);
359 static void tcf_bpf_cleanup(struct tc_action
*act
, int bind
)
361 struct tcf_bpf_cfg tmp
;
363 tcf_bpf_prog_fill_cfg(to_bpf(act
), &tmp
);
364 tcf_bpf_cfg_cleanup(&tmp
);
367 static int tcf_bpf_walker(struct net
*net
, struct sk_buff
*skb
,
368 struct netlink_callback
*cb
, int type
,
369 const struct tc_action_ops
*ops
)
371 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
373 return tcf_generic_walker(tn
, skb
, cb
, type
, ops
);
376 static int tcf_bpf_search(struct net
*net
, struct tc_action
**a
, u32 index
)
378 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
380 return tcf_hash_search(tn
, a
, index
);
383 static struct tc_action_ops act_bpf_ops __read_mostly
= {
386 .owner
= THIS_MODULE
,
388 .dump
= tcf_bpf_dump
,
389 .cleanup
= tcf_bpf_cleanup
,
390 .init
= tcf_bpf_init
,
391 .walk
= tcf_bpf_walker
,
392 .lookup
= tcf_bpf_search
,
393 .size
= sizeof(struct tcf_bpf
),
396 static __net_init
int bpf_init_net(struct net
*net
)
398 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
400 return tc_action_net_init(tn
, &act_bpf_ops
, BPF_TAB_MASK
);
403 static void __net_exit
bpf_exit_net(struct net
*net
)
405 struct tc_action_net
*tn
= net_generic(net
, bpf_net_id
);
407 tc_action_net_exit(tn
);
410 static struct pernet_operations bpf_net_ops
= {
411 .init
= bpf_init_net
,
412 .exit
= bpf_exit_net
,
414 .size
= sizeof(struct tc_action_net
),
417 static int __init
bpf_init_module(void)
419 return tcf_register_action(&act_bpf_ops
, &bpf_net_ops
);
422 static void __exit
bpf_cleanup_module(void)
424 tcf_unregister_action(&act_bpf_ops
, &bpf_net_ops
);
427 module_init(bpf_init_module
);
428 module_exit(bpf_cleanup_module
);
430 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
431 MODULE_DESCRIPTION("TC BPF based action");
432 MODULE_LICENSE("GPL v2");