2 * Berkeley Packet Filter based traffic classifier
4 * Might be used to classify traffic through flexible, user-defined and
5 * possibly JIT-ed BPF filters for traffic control as an alternative to
8 * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/skbuff.h>
18 #include <linux/filter.h>
19 #include <linux/bpf.h>
21 #include <net/rtnetlink.h>
22 #include <net/pkt_cls.h>
25 MODULE_LICENSE("GPL");
26 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
27 MODULE_DESCRIPTION("TC BPF based classifier");
29 #define CLS_BPF_NAME_LEN 256
32 struct list_head plist
;
38 struct bpf_prog
*filter
;
39 struct list_head link
;
40 struct tcf_result res
;
48 struct sock_filter
*bpf_ops
;
54 static const struct nla_policy bpf_policy
[TCA_BPF_MAX
+ 1] = {
55 [TCA_BPF_CLASSID
] = { .type
= NLA_U32
},
56 [TCA_BPF_FLAGS
] = { .type
= NLA_U32
},
57 [TCA_BPF_FD
] = { .type
= NLA_U32
},
58 [TCA_BPF_NAME
] = { .type
= NLA_NUL_STRING
, .len
= CLS_BPF_NAME_LEN
},
59 [TCA_BPF_OPS_LEN
] = { .type
= NLA_U16
},
60 [TCA_BPF_OPS
] = { .type
= NLA_BINARY
,
61 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
64 static int cls_bpf_exec_opcode(int code
)
68 case TC_ACT_RECLASSIFY
:
80 static int cls_bpf_classify(struct sk_buff
*skb
, const struct tcf_proto
*tp
,
81 struct tcf_result
*res
)
83 struct cls_bpf_head
*head
= rcu_dereference_bh(tp
->root
);
84 struct cls_bpf_prog
*prog
;
85 #ifdef CONFIG_NET_CLS_ACT
86 bool at_ingress
= G_TC_AT(skb
->tc_verd
) & AT_INGRESS
;
88 bool at_ingress
= false;
92 if (unlikely(!skb_mac_header_was_set(skb
)))
95 /* Needed here for accessing maps. */
97 list_for_each_entry_rcu(prog
, &head
->plist
, link
) {
100 qdisc_skb_cb(skb
)->tc_classid
= prog
->res
.classid
;
103 /* It is safe to push/pull even if skb_shared() */
104 __skb_push(skb
, skb
->mac_len
);
105 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
106 __skb_pull(skb
, skb
->mac_len
);
108 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
111 if (prog
->exts_integrated
) {
112 res
->class = prog
->res
.class;
113 res
->classid
= qdisc_skb_cb(skb
)->tc_classid
;
115 ret
= cls_bpf_exec_opcode(filter_res
);
116 if (ret
== TC_ACT_UNSPEC
)
125 if (filter_res
!= -1)
126 res
->classid
= filter_res
;
128 ret
= tcf_exts_exec(skb
, &prog
->exts
, res
);
139 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog
*prog
)
141 return !prog
->bpf_ops
;
144 static int cls_bpf_init(struct tcf_proto
*tp
)
146 struct cls_bpf_head
*head
;
148 head
= kzalloc(sizeof(*head
), GFP_KERNEL
);
152 INIT_LIST_HEAD_RCU(&head
->plist
);
153 rcu_assign_pointer(tp
->root
, head
);
158 static void cls_bpf_delete_prog(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
)
160 tcf_exts_destroy(&prog
->exts
);
162 if (cls_bpf_is_ebpf(prog
))
163 bpf_prog_put(prog
->filter
);
165 bpf_prog_destroy(prog
->filter
);
167 kfree(prog
->bpf_name
);
168 kfree(prog
->bpf_ops
);
172 static void __cls_bpf_delete_prog(struct rcu_head
*rcu
)
174 struct cls_bpf_prog
*prog
= container_of(rcu
, struct cls_bpf_prog
, rcu
);
176 cls_bpf_delete_prog(prog
->tp
, prog
);
179 static int cls_bpf_delete(struct tcf_proto
*tp
, unsigned long arg
)
181 struct cls_bpf_prog
*prog
= (struct cls_bpf_prog
*) arg
;
183 list_del_rcu(&prog
->link
);
184 tcf_unbind_filter(tp
, &prog
->res
);
185 call_rcu(&prog
->rcu
, __cls_bpf_delete_prog
);
190 static bool cls_bpf_destroy(struct tcf_proto
*tp
, bool force
)
192 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
193 struct cls_bpf_prog
*prog
, *tmp
;
195 if (!force
&& !list_empty(&head
->plist
))
198 list_for_each_entry_safe(prog
, tmp
, &head
->plist
, link
) {
199 list_del_rcu(&prog
->link
);
200 tcf_unbind_filter(tp
, &prog
->res
);
201 call_rcu(&prog
->rcu
, __cls_bpf_delete_prog
);
204 RCU_INIT_POINTER(tp
->root
, NULL
);
205 kfree_rcu(head
, rcu
);
209 static unsigned long cls_bpf_get(struct tcf_proto
*tp
, u32 handle
)
211 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
212 struct cls_bpf_prog
*prog
;
213 unsigned long ret
= 0UL;
218 list_for_each_entry(prog
, &head
->plist
, link
) {
219 if (prog
->handle
== handle
) {
220 ret
= (unsigned long) prog
;
228 static int cls_bpf_prog_from_ops(struct nlattr
**tb
, struct cls_bpf_prog
*prog
)
230 struct sock_filter
*bpf_ops
;
231 struct sock_fprog_kern fprog_tmp
;
233 u16 bpf_size
, bpf_num_ops
;
236 bpf_num_ops
= nla_get_u16(tb
[TCA_BPF_OPS_LEN
]);
237 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
240 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
241 if (bpf_size
!= nla_len(tb
[TCA_BPF_OPS
]))
244 bpf_ops
= kzalloc(bpf_size
, GFP_KERNEL
);
248 memcpy(bpf_ops
, nla_data(tb
[TCA_BPF_OPS
]), bpf_size
);
250 fprog_tmp
.len
= bpf_num_ops
;
251 fprog_tmp
.filter
= bpf_ops
;
253 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
259 prog
->bpf_ops
= bpf_ops
;
260 prog
->bpf_num_ops
= bpf_num_ops
;
261 prog
->bpf_name
= NULL
;
267 static int cls_bpf_prog_from_efd(struct nlattr
**tb
, struct cls_bpf_prog
*prog
,
268 const struct tcf_proto
*tp
)
274 bpf_fd
= nla_get_u32(tb
[TCA_BPF_FD
]);
276 fp
= bpf_prog_get(bpf_fd
);
280 if (fp
->type
!= BPF_PROG_TYPE_SCHED_CLS
) {
285 if (tb
[TCA_BPF_NAME
]) {
286 name
= kmemdup(nla_data(tb
[TCA_BPF_NAME
]),
287 nla_len(tb
[TCA_BPF_NAME
]),
295 prog
->bpf_ops
= NULL
;
296 prog
->bpf_fd
= bpf_fd
;
297 prog
->bpf_name
= name
;
303 static int cls_bpf_modify_existing(struct net
*net
, struct tcf_proto
*tp
,
304 struct cls_bpf_prog
*prog
,
305 unsigned long base
, struct nlattr
**tb
,
306 struct nlattr
*est
, bool ovr
)
308 bool is_bpf
, is_ebpf
, have_exts
= false;
309 struct tcf_exts exts
;
313 is_bpf
= tb
[TCA_BPF_OPS_LEN
] && tb
[TCA_BPF_OPS
];
314 is_ebpf
= tb
[TCA_BPF_FD
];
316 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
) ||
317 !tb
[TCA_BPF_CLASSID
])
320 tcf_exts_init(&exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
321 ret
= tcf_exts_validate(net
, tp
, tb
, est
, &exts
, ovr
);
325 classid
= nla_get_u32(tb
[TCA_BPF_CLASSID
]);
326 if (tb
[TCA_BPF_FLAGS
]) {
327 u32 bpf_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS
]);
329 if (bpf_flags
& ~TCA_BPF_FLAG_ACT_DIRECT
) {
330 tcf_exts_destroy(&exts
);
334 have_exts
= bpf_flags
& TCA_BPF_FLAG_ACT_DIRECT
;
337 prog
->res
.classid
= classid
;
338 prog
->exts_integrated
= have_exts
;
340 ret
= is_bpf
? cls_bpf_prog_from_ops(tb
, prog
) :
341 cls_bpf_prog_from_efd(tb
, prog
, tp
);
343 tcf_exts_destroy(&exts
);
347 tcf_bind_filter(tp
, &prog
->res
, base
);
348 tcf_exts_change(tp
, &prog
->exts
, &exts
);
353 static u32
cls_bpf_grab_new_handle(struct tcf_proto
*tp
,
354 struct cls_bpf_head
*head
)
356 unsigned int i
= 0x80000000;
360 if (++head
->hgen
== 0x7FFFFFFF)
362 } while (--i
> 0 && cls_bpf_get(tp
, head
->hgen
));
364 if (unlikely(i
== 0)) {
365 pr_err("Insufficient number of handles\n");
374 static int cls_bpf_change(struct net
*net
, struct sk_buff
*in_skb
,
375 struct tcf_proto
*tp
, unsigned long base
,
376 u32 handle
, struct nlattr
**tca
,
377 unsigned long *arg
, bool ovr
)
379 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
380 struct cls_bpf_prog
*oldprog
= (struct cls_bpf_prog
*) *arg
;
381 struct nlattr
*tb
[TCA_BPF_MAX
+ 1];
382 struct cls_bpf_prog
*prog
;
385 if (tca
[TCA_OPTIONS
] == NULL
)
388 ret
= nla_parse_nested(tb
, TCA_BPF_MAX
, tca
[TCA_OPTIONS
], bpf_policy
);
392 prog
= kzalloc(sizeof(*prog
), GFP_KERNEL
);
396 tcf_exts_init(&prog
->exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
399 if (handle
&& oldprog
->handle
!= handle
) {
406 prog
->handle
= cls_bpf_grab_new_handle(tp
, head
);
408 prog
->handle
= handle
;
409 if (prog
->handle
== 0) {
414 ret
= cls_bpf_modify_existing(net
, tp
, prog
, base
, tb
, tca
[TCA_RATE
], ovr
);
419 list_replace_rcu(&oldprog
->link
, &prog
->link
);
420 tcf_unbind_filter(tp
, &oldprog
->res
);
421 call_rcu(&oldprog
->rcu
, __cls_bpf_delete_prog
);
423 list_add_rcu(&prog
->link
, &head
->plist
);
426 *arg
= (unsigned long) prog
;
434 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog
*prog
,
439 if (nla_put_u16(skb
, TCA_BPF_OPS_LEN
, prog
->bpf_num_ops
))
442 nla
= nla_reserve(skb
, TCA_BPF_OPS
, prog
->bpf_num_ops
*
443 sizeof(struct sock_filter
));
447 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
452 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog
*prog
,
455 if (nla_put_u32(skb
, TCA_BPF_FD
, prog
->bpf_fd
))
458 if (prog
->bpf_name
&&
459 nla_put_string(skb
, TCA_BPF_NAME
, prog
->bpf_name
))
465 static int cls_bpf_dump(struct net
*net
, struct tcf_proto
*tp
, unsigned long fh
,
466 struct sk_buff
*skb
, struct tcmsg
*tm
)
468 struct cls_bpf_prog
*prog
= (struct cls_bpf_prog
*) fh
;
475 tm
->tcm_handle
= prog
->handle
;
477 nest
= nla_nest_start(skb
, TCA_OPTIONS
);
479 goto nla_put_failure
;
481 if (nla_put_u32(skb
, TCA_BPF_CLASSID
, prog
->res
.classid
))
482 goto nla_put_failure
;
484 if (cls_bpf_is_ebpf(prog
))
485 ret
= cls_bpf_dump_ebpf_info(prog
, skb
);
487 ret
= cls_bpf_dump_bpf_info(prog
, skb
);
489 goto nla_put_failure
;
491 if (tcf_exts_dump(skb
, &prog
->exts
) < 0)
492 goto nla_put_failure
;
494 nla_nest_end(skb
, nest
);
496 if (tcf_exts_dump_stats(skb
, &prog
->exts
) < 0)
497 goto nla_put_failure
;
502 nla_nest_cancel(skb
, nest
);
506 static void cls_bpf_walk(struct tcf_proto
*tp
, struct tcf_walker
*arg
)
508 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
509 struct cls_bpf_prog
*prog
;
511 list_for_each_entry(prog
, &head
->plist
, link
) {
512 if (arg
->count
< arg
->skip
)
514 if (arg
->fn(tp
, (unsigned long) prog
, arg
) < 0) {
523 static struct tcf_proto_ops cls_bpf_ops __read_mostly
= {
525 .owner
= THIS_MODULE
,
526 .classify
= cls_bpf_classify
,
527 .init
= cls_bpf_init
,
528 .destroy
= cls_bpf_destroy
,
530 .change
= cls_bpf_change
,
531 .delete = cls_bpf_delete
,
532 .walk
= cls_bpf_walk
,
533 .dump
= cls_bpf_dump
,
536 static int __init
cls_bpf_init_mod(void)
538 return register_tcf_proto_ops(&cls_bpf_ops
);
541 static void __exit
cls_bpf_exit_mod(void)
543 unregister_tcf_proto_ops(&cls_bpf_ops
);
546 module_init(cls_bpf_init_mod
);
547 module_exit(cls_bpf_exit_mod
);