2 * Berkeley Packet Filter based traffic classifier
4 * Might be used to classify traffic through flexible, user-defined and
5 * possibly JIT-ed BPF filters for traffic control as an alternative to
8 * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/skbuff.h>
18 #include <linux/filter.h>
19 #include <linux/bpf.h>
20 #include <linux/idr.h>
22 #include <net/rtnetlink.h>
23 #include <net/pkt_cls.h>
26 MODULE_LICENSE("GPL");
27 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
28 MODULE_DESCRIPTION("TC BPF based classifier");
30 #define CLS_BPF_NAME_LEN 256
31 #define CLS_BPF_SUPPORTED_GEN_FLAGS \
32 (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)
35 struct list_head plist
;
36 struct idr handle_idr
;
41 struct bpf_prog
*filter
;
42 struct list_head link
;
43 struct tcf_result res
;
49 struct sock_filter
*bpf_ops
;
53 struct work_struct work
;
58 static const struct nla_policy bpf_policy
[TCA_BPF_MAX
+ 1] = {
59 [TCA_BPF_CLASSID
] = { .type
= NLA_U32
},
60 [TCA_BPF_FLAGS
] = { .type
= NLA_U32
},
61 [TCA_BPF_FLAGS_GEN
] = { .type
= NLA_U32
},
62 [TCA_BPF_FD
] = { .type
= NLA_U32
},
63 [TCA_BPF_NAME
] = { .type
= NLA_NUL_STRING
,
64 .len
= CLS_BPF_NAME_LEN
},
65 [TCA_BPF_OPS_LEN
] = { .type
= NLA_U16
},
66 [TCA_BPF_OPS
] = { .type
= NLA_BINARY
,
67 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
70 static int cls_bpf_exec_opcode(int code
)
85 static int cls_bpf_classify(struct sk_buff
*skb
, const struct tcf_proto
*tp
,
86 struct tcf_result
*res
)
88 struct cls_bpf_head
*head
= rcu_dereference_bh(tp
->root
);
89 bool at_ingress
= skb_at_tc_ingress(skb
);
90 struct cls_bpf_prog
*prog
;
93 /* Needed here for accessing maps. */
95 list_for_each_entry_rcu(prog
, &head
->plist
, link
) {
98 qdisc_skb_cb(skb
)->tc_classid
= prog
->res
.classid
;
100 if (tc_skip_sw(prog
->gen_flags
)) {
101 filter_res
= prog
->exts_integrated
? TC_ACT_UNSPEC
: 0;
102 } else if (at_ingress
) {
103 /* It is safe to push/pull even if skb_shared() */
104 __skb_push(skb
, skb
->mac_len
);
105 bpf_compute_data_pointers(skb
);
106 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
107 __skb_pull(skb
, skb
->mac_len
);
109 bpf_compute_data_pointers(skb
);
110 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
113 if (prog
->exts_integrated
) {
115 res
->classid
= TC_H_MAJ(prog
->res
.classid
) |
116 qdisc_skb_cb(skb
)->tc_classid
;
118 ret
= cls_bpf_exec_opcode(filter_res
);
119 if (ret
== TC_ACT_UNSPEC
)
126 if (filter_res
!= -1) {
128 res
->classid
= filter_res
;
133 ret
= tcf_exts_exec(skb
, &prog
->exts
, res
);
144 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog
*prog
)
146 return !prog
->bpf_ops
;
149 static int cls_bpf_offload_cmd(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
,
150 struct cls_bpf_prog
*oldprog
)
152 struct tcf_block
*block
= tp
->chain
->block
;
153 struct tc_cls_bpf_offload cls_bpf
= {};
154 struct cls_bpf_prog
*obj
;
158 skip_sw
= prog
&& tc_skip_sw(prog
->gen_flags
);
159 obj
= prog
?: oldprog
;
161 tc_cls_common_offload_init(&cls_bpf
.common
, tp
);
162 cls_bpf
.command
= TC_CLSBPF_OFFLOAD
;
163 cls_bpf
.exts
= &obj
->exts
;
164 cls_bpf
.prog
= prog
? prog
->filter
: NULL
;
165 cls_bpf
.oldprog
= oldprog
? oldprog
->filter
: NULL
;
166 cls_bpf
.name
= obj
->bpf_name
;
167 cls_bpf
.exts_integrated
= obj
->exts_integrated
;
168 cls_bpf
.gen_flags
= obj
->gen_flags
;
170 err
= tc_setup_cb_call(block
, NULL
, TC_SETUP_CLSBPF
, &cls_bpf
, skip_sw
);
173 cls_bpf_offload_cmd(tp
, oldprog
, prog
);
175 } else if (err
> 0) {
176 prog
->gen_flags
|= TCA_CLS_FLAGS_IN_HW
;
180 if (prog
&& skip_sw
&& !(prog
->gen_flags
& TCA_CLS_FLAGS_IN_HW
))
186 static u32
cls_bpf_flags(u32 flags
)
188 return flags
& CLS_BPF_SUPPORTED_GEN_FLAGS
;
191 static int cls_bpf_offload(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
,
192 struct cls_bpf_prog
*oldprog
)
194 if (prog
&& oldprog
&&
195 cls_bpf_flags(prog
->gen_flags
) !=
196 cls_bpf_flags(oldprog
->gen_flags
))
199 if (prog
&& tc_skip_hw(prog
->gen_flags
))
201 if (oldprog
&& tc_skip_hw(oldprog
->gen_flags
))
203 if (!prog
&& !oldprog
)
206 return cls_bpf_offload_cmd(tp
, prog
, oldprog
);
209 static void cls_bpf_stop_offload(struct tcf_proto
*tp
,
210 struct cls_bpf_prog
*prog
)
214 err
= cls_bpf_offload_cmd(tp
, NULL
, prog
);
216 pr_err("Stopping hardware offload failed: %d\n", err
);
219 static void cls_bpf_offload_update_stats(struct tcf_proto
*tp
,
220 struct cls_bpf_prog
*prog
)
222 struct tcf_block
*block
= tp
->chain
->block
;
223 struct tc_cls_bpf_offload cls_bpf
= {};
225 tc_cls_common_offload_init(&cls_bpf
.common
, tp
);
226 cls_bpf
.command
= TC_CLSBPF_STATS
;
227 cls_bpf
.exts
= &prog
->exts
;
228 cls_bpf
.prog
= prog
->filter
;
229 cls_bpf
.name
= prog
->bpf_name
;
230 cls_bpf
.exts_integrated
= prog
->exts_integrated
;
231 cls_bpf
.gen_flags
= prog
->gen_flags
;
233 tc_setup_cb_call(block
, NULL
, TC_SETUP_CLSBPF
, &cls_bpf
, false);
236 static int cls_bpf_init(struct tcf_proto
*tp
)
238 struct cls_bpf_head
*head
;
240 head
= kzalloc(sizeof(*head
), GFP_KERNEL
);
244 INIT_LIST_HEAD_RCU(&head
->plist
);
245 idr_init(&head
->handle_idr
);
246 rcu_assign_pointer(tp
->root
, head
);
251 static void cls_bpf_free_parms(struct cls_bpf_prog
*prog
)
253 if (cls_bpf_is_ebpf(prog
))
254 bpf_prog_put(prog
->filter
);
256 bpf_prog_destroy(prog
->filter
);
258 kfree(prog
->bpf_name
);
259 kfree(prog
->bpf_ops
);
262 static void __cls_bpf_delete_prog(struct cls_bpf_prog
*prog
)
264 tcf_exts_destroy(&prog
->exts
);
265 tcf_exts_put_net(&prog
->exts
);
267 cls_bpf_free_parms(prog
);
271 static void cls_bpf_delete_prog_work(struct work_struct
*work
)
273 struct cls_bpf_prog
*prog
= container_of(work
, struct cls_bpf_prog
, work
);
276 __cls_bpf_delete_prog(prog
);
280 static void cls_bpf_delete_prog_rcu(struct rcu_head
*rcu
)
282 struct cls_bpf_prog
*prog
= container_of(rcu
, struct cls_bpf_prog
, rcu
);
284 INIT_WORK(&prog
->work
, cls_bpf_delete_prog_work
);
285 tcf_queue_work(&prog
->work
);
288 static void __cls_bpf_delete(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
)
290 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
292 idr_remove_ext(&head
->handle_idr
, prog
->handle
);
293 cls_bpf_stop_offload(tp
, prog
);
294 list_del_rcu(&prog
->link
);
295 tcf_unbind_filter(tp
, &prog
->res
);
296 if (tcf_exts_get_net(&prog
->exts
))
297 call_rcu(&prog
->rcu
, cls_bpf_delete_prog_rcu
);
299 __cls_bpf_delete_prog(prog
);
302 static int cls_bpf_delete(struct tcf_proto
*tp
, void *arg
, bool *last
)
304 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
306 __cls_bpf_delete(tp
, arg
);
307 *last
= list_empty(&head
->plist
);
311 static void cls_bpf_destroy(struct tcf_proto
*tp
)
313 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
314 struct cls_bpf_prog
*prog
, *tmp
;
316 list_for_each_entry_safe(prog
, tmp
, &head
->plist
, link
)
317 __cls_bpf_delete(tp
, prog
);
319 idr_destroy(&head
->handle_idr
);
320 kfree_rcu(head
, rcu
);
323 static void *cls_bpf_get(struct tcf_proto
*tp
, u32 handle
)
325 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
326 struct cls_bpf_prog
*prog
;
328 list_for_each_entry(prog
, &head
->plist
, link
) {
329 if (prog
->handle
== handle
)
336 static int cls_bpf_prog_from_ops(struct nlattr
**tb
, struct cls_bpf_prog
*prog
)
338 struct sock_filter
*bpf_ops
;
339 struct sock_fprog_kern fprog_tmp
;
341 u16 bpf_size
, bpf_num_ops
;
344 bpf_num_ops
= nla_get_u16(tb
[TCA_BPF_OPS_LEN
]);
345 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
348 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
349 if (bpf_size
!= nla_len(tb
[TCA_BPF_OPS
]))
352 bpf_ops
= kzalloc(bpf_size
, GFP_KERNEL
);
356 memcpy(bpf_ops
, nla_data(tb
[TCA_BPF_OPS
]), bpf_size
);
358 fprog_tmp
.len
= bpf_num_ops
;
359 fprog_tmp
.filter
= bpf_ops
;
361 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
367 prog
->bpf_ops
= bpf_ops
;
368 prog
->bpf_num_ops
= bpf_num_ops
;
369 prog
->bpf_name
= NULL
;
375 static int cls_bpf_prog_from_efd(struct nlattr
**tb
, struct cls_bpf_prog
*prog
,
376 u32 gen_flags
, const struct tcf_proto
*tp
)
383 bpf_fd
= nla_get_u32(tb
[TCA_BPF_FD
]);
384 skip_sw
= gen_flags
& TCA_CLS_FLAGS_SKIP_SW
;
386 fp
= bpf_prog_get_type_dev(bpf_fd
, BPF_PROG_TYPE_SCHED_CLS
, skip_sw
);
390 if (tb
[TCA_BPF_NAME
]) {
391 name
= nla_memdup(tb
[TCA_BPF_NAME
], GFP_KERNEL
);
398 prog
->bpf_ops
= NULL
;
399 prog
->bpf_name
= name
;
402 if (fp
->dst_needed
&& !(tp
->q
->flags
& TCQ_F_INGRESS
))
403 netif_keep_dst(qdisc_dev(tp
->q
));
408 static int cls_bpf_set_parms(struct net
*net
, struct tcf_proto
*tp
,
409 struct cls_bpf_prog
*prog
, unsigned long base
,
410 struct nlattr
**tb
, struct nlattr
*est
, bool ovr
)
412 bool is_bpf
, is_ebpf
, have_exts
= false;
416 is_bpf
= tb
[TCA_BPF_OPS_LEN
] && tb
[TCA_BPF_OPS
];
417 is_ebpf
= tb
[TCA_BPF_FD
];
418 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
))
421 ret
= tcf_exts_validate(net
, tp
, tb
, est
, &prog
->exts
, ovr
);
425 if (tb
[TCA_BPF_FLAGS
]) {
426 u32 bpf_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS
]);
428 if (bpf_flags
& ~TCA_BPF_FLAG_ACT_DIRECT
)
431 have_exts
= bpf_flags
& TCA_BPF_FLAG_ACT_DIRECT
;
433 if (tb
[TCA_BPF_FLAGS_GEN
]) {
434 gen_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS_GEN
]);
435 if (gen_flags
& ~CLS_BPF_SUPPORTED_GEN_FLAGS
||
436 !tc_flags_valid(gen_flags
))
440 prog
->exts_integrated
= have_exts
;
441 prog
->gen_flags
= gen_flags
;
443 ret
= is_bpf
? cls_bpf_prog_from_ops(tb
, prog
) :
444 cls_bpf_prog_from_efd(tb
, prog
, gen_flags
, tp
);
448 if (tb
[TCA_BPF_CLASSID
]) {
449 prog
->res
.classid
= nla_get_u32(tb
[TCA_BPF_CLASSID
]);
450 tcf_bind_filter(tp
, &prog
->res
, base
);
456 static int cls_bpf_change(struct net
*net
, struct sk_buff
*in_skb
,
457 struct tcf_proto
*tp
, unsigned long base
,
458 u32 handle
, struct nlattr
**tca
,
459 void **arg
, bool ovr
)
461 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
462 struct cls_bpf_prog
*oldprog
= *arg
;
463 struct nlattr
*tb
[TCA_BPF_MAX
+ 1];
464 struct cls_bpf_prog
*prog
;
465 unsigned long idr_index
;
468 if (tca
[TCA_OPTIONS
] == NULL
)
471 ret
= nla_parse_nested(tb
, TCA_BPF_MAX
, tca
[TCA_OPTIONS
], bpf_policy
,
476 prog
= kzalloc(sizeof(*prog
), GFP_KERNEL
);
480 ret
= tcf_exts_init(&prog
->exts
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
485 if (handle
&& oldprog
->handle
!= handle
) {
492 ret
= idr_alloc_ext(&head
->handle_idr
, prog
, &idr_index
,
493 1, 0x7FFFFFFF, GFP_KERNEL
);
496 prog
->handle
= idr_index
;
499 ret
= idr_alloc_ext(&head
->handle_idr
, prog
, &idr_index
,
500 handle
, handle
+ 1, GFP_KERNEL
);
504 prog
->handle
= handle
;
507 ret
= cls_bpf_set_parms(net
, tp
, prog
, base
, tb
, tca
[TCA_RATE
], ovr
);
511 ret
= cls_bpf_offload(tp
, prog
, oldprog
);
515 if (!tc_in_hw(prog
->gen_flags
))
516 prog
->gen_flags
|= TCA_CLS_FLAGS_NOT_IN_HW
;
519 idr_replace_ext(&head
->handle_idr
, prog
, handle
);
520 list_replace_rcu(&oldprog
->link
, &prog
->link
);
521 tcf_unbind_filter(tp
, &oldprog
->res
);
522 tcf_exts_get_net(&oldprog
->exts
);
523 call_rcu(&oldprog
->rcu
, cls_bpf_delete_prog_rcu
);
525 list_add_rcu(&prog
->link
, &head
->plist
);
532 cls_bpf_free_parms(prog
);
535 idr_remove_ext(&head
->handle_idr
, prog
->handle
);
537 tcf_exts_destroy(&prog
->exts
);
542 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog
*prog
,
547 if (nla_put_u16(skb
, TCA_BPF_OPS_LEN
, prog
->bpf_num_ops
))
550 nla
= nla_reserve(skb
, TCA_BPF_OPS
, prog
->bpf_num_ops
*
551 sizeof(struct sock_filter
));
555 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
560 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog
*prog
,
565 if (prog
->bpf_name
&&
566 nla_put_string(skb
, TCA_BPF_NAME
, prog
->bpf_name
))
569 if (nla_put_u32(skb
, TCA_BPF_ID
, prog
->filter
->aux
->id
))
572 nla
= nla_reserve(skb
, TCA_BPF_TAG
, sizeof(prog
->filter
->tag
));
576 memcpy(nla_data(nla
), prog
->filter
->tag
, nla_len(nla
));
581 static int cls_bpf_dump(struct net
*net
, struct tcf_proto
*tp
, void *fh
,
582 struct sk_buff
*skb
, struct tcmsg
*tm
)
584 struct cls_bpf_prog
*prog
= fh
;
592 tm
->tcm_handle
= prog
->handle
;
594 cls_bpf_offload_update_stats(tp
, prog
);
596 nest
= nla_nest_start(skb
, TCA_OPTIONS
);
598 goto nla_put_failure
;
600 if (prog
->res
.classid
&&
601 nla_put_u32(skb
, TCA_BPF_CLASSID
, prog
->res
.classid
))
602 goto nla_put_failure
;
604 if (cls_bpf_is_ebpf(prog
))
605 ret
= cls_bpf_dump_ebpf_info(prog
, skb
);
607 ret
= cls_bpf_dump_bpf_info(prog
, skb
);
609 goto nla_put_failure
;
611 if (tcf_exts_dump(skb
, &prog
->exts
) < 0)
612 goto nla_put_failure
;
614 if (prog
->exts_integrated
)
615 bpf_flags
|= TCA_BPF_FLAG_ACT_DIRECT
;
616 if (bpf_flags
&& nla_put_u32(skb
, TCA_BPF_FLAGS
, bpf_flags
))
617 goto nla_put_failure
;
618 if (prog
->gen_flags
&&
619 nla_put_u32(skb
, TCA_BPF_FLAGS_GEN
, prog
->gen_flags
))
620 goto nla_put_failure
;
622 nla_nest_end(skb
, nest
);
624 if (tcf_exts_dump_stats(skb
, &prog
->exts
) < 0)
625 goto nla_put_failure
;
630 nla_nest_cancel(skb
, nest
);
634 static void cls_bpf_bind_class(void *fh
, u32 classid
, unsigned long cl
)
636 struct cls_bpf_prog
*prog
= fh
;
638 if (prog
&& prog
->res
.classid
== classid
)
639 prog
->res
.class = cl
;
642 static void cls_bpf_walk(struct tcf_proto
*tp
, struct tcf_walker
*arg
)
644 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
645 struct cls_bpf_prog
*prog
;
647 list_for_each_entry(prog
, &head
->plist
, link
) {
648 if (arg
->count
< arg
->skip
)
650 if (arg
->fn(tp
, prog
, arg
) < 0) {
659 static struct tcf_proto_ops cls_bpf_ops __read_mostly
= {
661 .owner
= THIS_MODULE
,
662 .classify
= cls_bpf_classify
,
663 .init
= cls_bpf_init
,
664 .destroy
= cls_bpf_destroy
,
666 .change
= cls_bpf_change
,
667 .delete = cls_bpf_delete
,
668 .walk
= cls_bpf_walk
,
669 .dump
= cls_bpf_dump
,
670 .bind_class
= cls_bpf_bind_class
,
673 static int __init
cls_bpf_init_mod(void)
675 return register_tcf_proto_ops(&cls_bpf_ops
);
678 static void __exit
cls_bpf_exit_mod(void)
680 unregister_tcf_proto_ops(&cls_bpf_ops
);
683 module_init(cls_bpf_init_mod
);
684 module_exit(cls_bpf_exit_mod
);