2 * Berkeley Packet Filter based traffic classifier
4 * Might be used to classify traffic through flexible, user-defined and
5 * possibly JIT-ed BPF filters for traffic control as an alternative to
8 * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/skbuff.h>
18 #include <linux/filter.h>
19 #include <linux/bpf.h>
21 #include <net/rtnetlink.h>
22 #include <net/pkt_cls.h>
25 MODULE_LICENSE("GPL");
26 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
27 MODULE_DESCRIPTION("TC BPF based classifier");
29 #define CLS_BPF_NAME_LEN 256
32 struct list_head plist
;
38 struct bpf_prog
*filter
;
39 struct list_head link
;
40 struct tcf_result res
;
48 struct sock_filter
*bpf_ops
;
54 static const struct nla_policy bpf_policy
[TCA_BPF_MAX
+ 1] = {
55 [TCA_BPF_CLASSID
] = { .type
= NLA_U32
},
56 [TCA_BPF_FLAGS
] = { .type
= NLA_U32
},
57 [TCA_BPF_FD
] = { .type
= NLA_U32
},
58 [TCA_BPF_NAME
] = { .type
= NLA_NUL_STRING
,
59 .len
= CLS_BPF_NAME_LEN
},
60 [TCA_BPF_OPS_LEN
] = { .type
= NLA_U16
},
61 [TCA_BPF_OPS
] = { .type
= NLA_BINARY
,
62 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
65 static int cls_bpf_exec_opcode(int code
)
79 static int cls_bpf_classify(struct sk_buff
*skb
, const struct tcf_proto
*tp
,
80 struct tcf_result
*res
)
82 struct cls_bpf_head
*head
= rcu_dereference_bh(tp
->root
);
83 bool at_ingress
= skb_at_tc_ingress(skb
);
84 struct cls_bpf_prog
*prog
;
87 /* Needed here for accessing maps. */
89 list_for_each_entry_rcu(prog
, &head
->plist
, link
) {
92 qdisc_skb_cb(skb
)->tc_classid
= prog
->res
.classid
;
95 /* It is safe to push/pull even if skb_shared() */
96 __skb_push(skb
, skb
->mac_len
);
97 bpf_compute_data_end(skb
);
98 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
99 __skb_pull(skb
, skb
->mac_len
);
101 bpf_compute_data_end(skb
);
102 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
105 if (prog
->exts_integrated
) {
107 res
->classid
= TC_H_MAJ(prog
->res
.classid
) |
108 qdisc_skb_cb(skb
)->tc_classid
;
110 ret
= cls_bpf_exec_opcode(filter_res
);
111 if (ret
== TC_ACT_UNSPEC
)
118 if (filter_res
!= -1) {
120 res
->classid
= filter_res
;
125 ret
= tcf_exts_exec(skb
, &prog
->exts
, res
);
136 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog
*prog
)
138 return !prog
->bpf_ops
;
141 static int cls_bpf_init(struct tcf_proto
*tp
)
143 struct cls_bpf_head
*head
;
145 head
= kzalloc(sizeof(*head
), GFP_KERNEL
);
149 INIT_LIST_HEAD_RCU(&head
->plist
);
150 rcu_assign_pointer(tp
->root
, head
);
155 static void cls_bpf_delete_prog(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
)
157 tcf_exts_destroy(&prog
->exts
);
159 if (cls_bpf_is_ebpf(prog
))
160 bpf_prog_put(prog
->filter
);
162 bpf_prog_destroy(prog
->filter
);
164 kfree(prog
->bpf_name
);
165 kfree(prog
->bpf_ops
);
169 static void __cls_bpf_delete_prog(struct rcu_head
*rcu
)
171 struct cls_bpf_prog
*prog
= container_of(rcu
, struct cls_bpf_prog
, rcu
);
173 cls_bpf_delete_prog(prog
->tp
, prog
);
176 static int cls_bpf_delete(struct tcf_proto
*tp
, unsigned long arg
)
178 struct cls_bpf_prog
*prog
= (struct cls_bpf_prog
*) arg
;
180 list_del_rcu(&prog
->link
);
181 tcf_unbind_filter(tp
, &prog
->res
);
182 call_rcu(&prog
->rcu
, __cls_bpf_delete_prog
);
187 static bool cls_bpf_destroy(struct tcf_proto
*tp
, bool force
)
189 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
190 struct cls_bpf_prog
*prog
, *tmp
;
192 if (!force
&& !list_empty(&head
->plist
))
195 list_for_each_entry_safe(prog
, tmp
, &head
->plist
, link
) {
196 list_del_rcu(&prog
->link
);
197 tcf_unbind_filter(tp
, &prog
->res
);
198 call_rcu(&prog
->rcu
, __cls_bpf_delete_prog
);
201 RCU_INIT_POINTER(tp
->root
, NULL
);
202 kfree_rcu(head
, rcu
);
206 static unsigned long cls_bpf_get(struct tcf_proto
*tp
, u32 handle
)
208 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
209 struct cls_bpf_prog
*prog
;
210 unsigned long ret
= 0UL;
215 list_for_each_entry(prog
, &head
->plist
, link
) {
216 if (prog
->handle
== handle
) {
217 ret
= (unsigned long) prog
;
225 static int cls_bpf_prog_from_ops(struct nlattr
**tb
, struct cls_bpf_prog
*prog
)
227 struct sock_filter
*bpf_ops
;
228 struct sock_fprog_kern fprog_tmp
;
230 u16 bpf_size
, bpf_num_ops
;
233 bpf_num_ops
= nla_get_u16(tb
[TCA_BPF_OPS_LEN
]);
234 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
237 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
238 if (bpf_size
!= nla_len(tb
[TCA_BPF_OPS
]))
241 bpf_ops
= kzalloc(bpf_size
, GFP_KERNEL
);
245 memcpy(bpf_ops
, nla_data(tb
[TCA_BPF_OPS
]), bpf_size
);
247 fprog_tmp
.len
= bpf_num_ops
;
248 fprog_tmp
.filter
= bpf_ops
;
250 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
256 prog
->bpf_ops
= bpf_ops
;
257 prog
->bpf_num_ops
= bpf_num_ops
;
258 prog
->bpf_name
= NULL
;
264 static int cls_bpf_prog_from_efd(struct nlattr
**tb
, struct cls_bpf_prog
*prog
,
265 const struct tcf_proto
*tp
)
271 bpf_fd
= nla_get_u32(tb
[TCA_BPF_FD
]);
273 fp
= bpf_prog_get_type(bpf_fd
, BPF_PROG_TYPE_SCHED_CLS
);
277 if (tb
[TCA_BPF_NAME
]) {
278 name
= kmemdup(nla_data(tb
[TCA_BPF_NAME
]),
279 nla_len(tb
[TCA_BPF_NAME
]),
287 prog
->bpf_ops
= NULL
;
288 prog
->bpf_fd
= bpf_fd
;
289 prog
->bpf_name
= name
;
292 if (fp
->dst_needed
&& !(tp
->q
->flags
& TCQ_F_INGRESS
))
293 netif_keep_dst(qdisc_dev(tp
->q
));
298 static int cls_bpf_modify_existing(struct net
*net
, struct tcf_proto
*tp
,
299 struct cls_bpf_prog
*prog
,
300 unsigned long base
, struct nlattr
**tb
,
301 struct nlattr
*est
, bool ovr
)
303 bool is_bpf
, is_ebpf
, have_exts
= false;
304 struct tcf_exts exts
;
307 is_bpf
= tb
[TCA_BPF_OPS_LEN
] && tb
[TCA_BPF_OPS
];
308 is_ebpf
= tb
[TCA_BPF_FD
];
309 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
))
312 ret
= tcf_exts_init(&exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
315 ret
= tcf_exts_validate(net
, tp
, tb
, est
, &exts
, ovr
);
319 if (tb
[TCA_BPF_FLAGS
]) {
320 u32 bpf_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS
]);
322 if (bpf_flags
& ~TCA_BPF_FLAG_ACT_DIRECT
) {
327 have_exts
= bpf_flags
& TCA_BPF_FLAG_ACT_DIRECT
;
330 prog
->exts_integrated
= have_exts
;
332 ret
= is_bpf
? cls_bpf_prog_from_ops(tb
, prog
) :
333 cls_bpf_prog_from_efd(tb
, prog
, tp
);
337 if (tb
[TCA_BPF_CLASSID
]) {
338 prog
->res
.classid
= nla_get_u32(tb
[TCA_BPF_CLASSID
]);
339 tcf_bind_filter(tp
, &prog
->res
, base
);
342 tcf_exts_change(tp
, &prog
->exts
, &exts
);
346 tcf_exts_destroy(&exts
);
350 static u32
cls_bpf_grab_new_handle(struct tcf_proto
*tp
,
351 struct cls_bpf_head
*head
)
353 unsigned int i
= 0x80000000;
357 if (++head
->hgen
== 0x7FFFFFFF)
359 } while (--i
> 0 && cls_bpf_get(tp
, head
->hgen
));
361 if (unlikely(i
== 0)) {
362 pr_err("Insufficient number of handles\n");
371 static int cls_bpf_change(struct net
*net
, struct sk_buff
*in_skb
,
372 struct tcf_proto
*tp
, unsigned long base
,
373 u32 handle
, struct nlattr
**tca
,
374 unsigned long *arg
, bool ovr
)
376 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
377 struct cls_bpf_prog
*oldprog
= (struct cls_bpf_prog
*) *arg
;
378 struct nlattr
*tb
[TCA_BPF_MAX
+ 1];
379 struct cls_bpf_prog
*prog
;
382 if (tca
[TCA_OPTIONS
] == NULL
)
385 ret
= nla_parse_nested(tb
, TCA_BPF_MAX
, tca
[TCA_OPTIONS
], bpf_policy
);
389 prog
= kzalloc(sizeof(*prog
), GFP_KERNEL
);
393 ret
= tcf_exts_init(&prog
->exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
398 if (handle
&& oldprog
->handle
!= handle
) {
405 prog
->handle
= cls_bpf_grab_new_handle(tp
, head
);
407 prog
->handle
= handle
;
408 if (prog
->handle
== 0) {
413 ret
= cls_bpf_modify_existing(net
, tp
, prog
, base
, tb
, tca
[TCA_RATE
],
419 list_replace_rcu(&oldprog
->link
, &prog
->link
);
420 tcf_unbind_filter(tp
, &oldprog
->res
);
421 call_rcu(&oldprog
->rcu
, __cls_bpf_delete_prog
);
423 list_add_rcu(&prog
->link
, &head
->plist
);
426 *arg
= (unsigned long) prog
;
430 tcf_exts_destroy(&prog
->exts
);
435 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog
*prog
,
440 if (nla_put_u16(skb
, TCA_BPF_OPS_LEN
, prog
->bpf_num_ops
))
443 nla
= nla_reserve(skb
, TCA_BPF_OPS
, prog
->bpf_num_ops
*
444 sizeof(struct sock_filter
));
448 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
453 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog
*prog
,
456 if (nla_put_u32(skb
, TCA_BPF_FD
, prog
->bpf_fd
))
459 if (prog
->bpf_name
&&
460 nla_put_string(skb
, TCA_BPF_NAME
, prog
->bpf_name
))
466 static int cls_bpf_dump(struct net
*net
, struct tcf_proto
*tp
, unsigned long fh
,
467 struct sk_buff
*skb
, struct tcmsg
*tm
)
469 struct cls_bpf_prog
*prog
= (struct cls_bpf_prog
*) fh
;
477 tm
->tcm_handle
= prog
->handle
;
479 nest
= nla_nest_start(skb
, TCA_OPTIONS
);
481 goto nla_put_failure
;
483 if (prog
->res
.classid
&&
484 nla_put_u32(skb
, TCA_BPF_CLASSID
, prog
->res
.classid
))
485 goto nla_put_failure
;
487 if (cls_bpf_is_ebpf(prog
))
488 ret
= cls_bpf_dump_ebpf_info(prog
, skb
);
490 ret
= cls_bpf_dump_bpf_info(prog
, skb
);
492 goto nla_put_failure
;
494 if (tcf_exts_dump(skb
, &prog
->exts
) < 0)
495 goto nla_put_failure
;
497 if (prog
->exts_integrated
)
498 bpf_flags
|= TCA_BPF_FLAG_ACT_DIRECT
;
499 if (bpf_flags
&& nla_put_u32(skb
, TCA_BPF_FLAGS
, bpf_flags
))
500 goto nla_put_failure
;
502 nla_nest_end(skb
, nest
);
504 if (tcf_exts_dump_stats(skb
, &prog
->exts
) < 0)
505 goto nla_put_failure
;
510 nla_nest_cancel(skb
, nest
);
514 static void cls_bpf_walk(struct tcf_proto
*tp
, struct tcf_walker
*arg
)
516 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
517 struct cls_bpf_prog
*prog
;
519 list_for_each_entry(prog
, &head
->plist
, link
) {
520 if (arg
->count
< arg
->skip
)
522 if (arg
->fn(tp
, (unsigned long) prog
, arg
) < 0) {
531 static struct tcf_proto_ops cls_bpf_ops __read_mostly
= {
533 .owner
= THIS_MODULE
,
534 .classify
= cls_bpf_classify
,
535 .init
= cls_bpf_init
,
536 .destroy
= cls_bpf_destroy
,
538 .change
= cls_bpf_change
,
539 .delete = cls_bpf_delete
,
540 .walk
= cls_bpf_walk
,
541 .dump
= cls_bpf_dump
,
544 static int __init
cls_bpf_init_mod(void)
546 return register_tcf_proto_ops(&cls_bpf_ops
);
549 static void __exit
cls_bpf_exit_mod(void)
551 unregister_tcf_proto_ops(&cls_bpf_ops
);
554 module_init(cls_bpf_init_mod
);
555 module_exit(cls_bpf_exit_mod
);