2 * Berkeley Packet Filter based traffic classifier
4 * Might be used to classify traffic through flexible, user-defined and
5 * possibly JIT-ed BPF filters for traffic control as an alternative to
8 * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
15 #include <linux/module.h>
16 #include <linux/types.h>
17 #include <linux/skbuff.h>
18 #include <linux/filter.h>
19 #include <linux/bpf.h>
20 #include <linux/idr.h>
22 #include <net/rtnetlink.h>
23 #include <net/pkt_cls.h>
26 MODULE_LICENSE("GPL");
27 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
28 MODULE_DESCRIPTION("TC BPF based classifier");
30 #define CLS_BPF_NAME_LEN 256
31 #define CLS_BPF_SUPPORTED_GEN_FLAGS \
32 (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW)
35 struct list_head plist
;
36 struct idr handle_idr
;
41 struct bpf_prog
*filter
;
42 struct list_head link
;
43 struct tcf_result res
;
49 struct sock_filter
*bpf_ops
;
52 struct rcu_work rwork
;
55 static const struct nla_policy bpf_policy
[TCA_BPF_MAX
+ 1] = {
56 [TCA_BPF_CLASSID
] = { .type
= NLA_U32
},
57 [TCA_BPF_FLAGS
] = { .type
= NLA_U32
},
58 [TCA_BPF_FLAGS_GEN
] = { .type
= NLA_U32
},
59 [TCA_BPF_FD
] = { .type
= NLA_U32
},
60 [TCA_BPF_NAME
] = { .type
= NLA_NUL_STRING
,
61 .len
= CLS_BPF_NAME_LEN
},
62 [TCA_BPF_OPS_LEN
] = { .type
= NLA_U16
},
63 [TCA_BPF_OPS
] = { .type
= NLA_BINARY
,
64 .len
= sizeof(struct sock_filter
) * BPF_MAXINSNS
},
67 static int cls_bpf_exec_opcode(int code
)
82 static int cls_bpf_classify(struct sk_buff
*skb
, const struct tcf_proto
*tp
,
83 struct tcf_result
*res
)
85 struct cls_bpf_head
*head
= rcu_dereference_bh(tp
->root
);
86 bool at_ingress
= skb_at_tc_ingress(skb
);
87 struct cls_bpf_prog
*prog
;
90 /* Needed here for accessing maps. */
92 list_for_each_entry_rcu(prog
, &head
->plist
, link
) {
95 qdisc_skb_cb(skb
)->tc_classid
= prog
->res
.classid
;
97 if (tc_skip_sw(prog
->gen_flags
)) {
98 filter_res
= prog
->exts_integrated
? TC_ACT_UNSPEC
: 0;
99 } else if (at_ingress
) {
100 /* It is safe to push/pull even if skb_shared() */
101 __skb_push(skb
, skb
->mac_len
);
102 bpf_compute_data_pointers(skb
);
103 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
104 __skb_pull(skb
, skb
->mac_len
);
106 bpf_compute_data_pointers(skb
);
107 filter_res
= BPF_PROG_RUN(prog
->filter
, skb
);
110 if (prog
->exts_integrated
) {
112 res
->classid
= TC_H_MAJ(prog
->res
.classid
) |
113 qdisc_skb_cb(skb
)->tc_classid
;
115 ret
= cls_bpf_exec_opcode(filter_res
);
116 if (ret
== TC_ACT_UNSPEC
)
123 if (filter_res
!= -1) {
125 res
->classid
= filter_res
;
130 ret
= tcf_exts_exec(skb
, &prog
->exts
, res
);
141 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog
*prog
)
143 return !prog
->bpf_ops
;
146 static int cls_bpf_offload_cmd(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
,
147 struct cls_bpf_prog
*oldprog
)
149 struct tcf_block
*block
= tp
->chain
->block
;
150 struct tc_cls_bpf_offload cls_bpf
= {};
151 struct cls_bpf_prog
*obj
;
155 skip_sw
= prog
&& tc_skip_sw(prog
->gen_flags
);
156 obj
= prog
?: oldprog
;
158 tc_cls_common_offload_init(&cls_bpf
.common
, tp
);
159 cls_bpf
.command
= TC_CLSBPF_OFFLOAD
;
160 cls_bpf
.exts
= &obj
->exts
;
161 cls_bpf
.prog
= prog
? prog
->filter
: NULL
;
162 cls_bpf
.oldprog
= oldprog
? oldprog
->filter
: NULL
;
163 cls_bpf
.name
= obj
->bpf_name
;
164 cls_bpf
.exts_integrated
= obj
->exts_integrated
;
165 cls_bpf
.gen_flags
= obj
->gen_flags
;
167 err
= tc_setup_cb_call(block
, NULL
, TC_SETUP_CLSBPF
, &cls_bpf
, skip_sw
);
170 cls_bpf_offload_cmd(tp
, oldprog
, prog
);
172 } else if (err
> 0) {
173 prog
->gen_flags
|= TCA_CLS_FLAGS_IN_HW
;
177 if (prog
&& skip_sw
&& !(prog
->gen_flags
& TCA_CLS_FLAGS_IN_HW
))
183 static u32
cls_bpf_flags(u32 flags
)
185 return flags
& CLS_BPF_SUPPORTED_GEN_FLAGS
;
188 static int cls_bpf_offload(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
,
189 struct cls_bpf_prog
*oldprog
)
191 if (prog
&& oldprog
&&
192 cls_bpf_flags(prog
->gen_flags
) !=
193 cls_bpf_flags(oldprog
->gen_flags
))
196 if (prog
&& tc_skip_hw(prog
->gen_flags
))
198 if (oldprog
&& tc_skip_hw(oldprog
->gen_flags
))
200 if (!prog
&& !oldprog
)
203 return cls_bpf_offload_cmd(tp
, prog
, oldprog
);
206 static void cls_bpf_stop_offload(struct tcf_proto
*tp
,
207 struct cls_bpf_prog
*prog
)
211 err
= cls_bpf_offload_cmd(tp
, NULL
, prog
);
213 pr_err("Stopping hardware offload failed: %d\n", err
);
216 static void cls_bpf_offload_update_stats(struct tcf_proto
*tp
,
217 struct cls_bpf_prog
*prog
)
219 struct tcf_block
*block
= tp
->chain
->block
;
220 struct tc_cls_bpf_offload cls_bpf
= {};
222 tc_cls_common_offload_init(&cls_bpf
.common
, tp
);
223 cls_bpf
.command
= TC_CLSBPF_STATS
;
224 cls_bpf
.exts
= &prog
->exts
;
225 cls_bpf
.prog
= prog
->filter
;
226 cls_bpf
.name
= prog
->bpf_name
;
227 cls_bpf
.exts_integrated
= prog
->exts_integrated
;
228 cls_bpf
.gen_flags
= prog
->gen_flags
;
230 tc_setup_cb_call(block
, NULL
, TC_SETUP_CLSBPF
, &cls_bpf
, false);
233 static int cls_bpf_init(struct tcf_proto
*tp
)
235 struct cls_bpf_head
*head
;
237 head
= kzalloc(sizeof(*head
), GFP_KERNEL
);
241 INIT_LIST_HEAD_RCU(&head
->plist
);
242 idr_init(&head
->handle_idr
);
243 rcu_assign_pointer(tp
->root
, head
);
248 static void cls_bpf_free_parms(struct cls_bpf_prog
*prog
)
250 if (cls_bpf_is_ebpf(prog
))
251 bpf_prog_put(prog
->filter
);
253 bpf_prog_destroy(prog
->filter
);
255 kfree(prog
->bpf_name
);
256 kfree(prog
->bpf_ops
);
259 static void __cls_bpf_delete_prog(struct cls_bpf_prog
*prog
)
261 tcf_exts_destroy(&prog
->exts
);
262 tcf_exts_put_net(&prog
->exts
);
264 cls_bpf_free_parms(prog
);
268 static void cls_bpf_delete_prog_work(struct work_struct
*work
)
270 struct cls_bpf_prog
*prog
= container_of(to_rcu_work(work
),
274 __cls_bpf_delete_prog(prog
);
278 static void __cls_bpf_delete(struct tcf_proto
*tp
, struct cls_bpf_prog
*prog
)
280 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
282 idr_remove_ext(&head
->handle_idr
, prog
->handle
);
283 cls_bpf_stop_offload(tp
, prog
);
284 list_del_rcu(&prog
->link
);
285 tcf_unbind_filter(tp
, &prog
->res
);
286 if (tcf_exts_get_net(&prog
->exts
))
287 tcf_queue_work(&prog
->rwork
, cls_bpf_delete_prog_work
);
289 __cls_bpf_delete_prog(prog
);
292 static int cls_bpf_delete(struct tcf_proto
*tp
, void *arg
, bool *last
)
294 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
296 __cls_bpf_delete(tp
, arg
);
297 *last
= list_empty(&head
->plist
);
301 static void cls_bpf_destroy(struct tcf_proto
*tp
)
303 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
304 struct cls_bpf_prog
*prog
, *tmp
;
306 list_for_each_entry_safe(prog
, tmp
, &head
->plist
, link
)
307 __cls_bpf_delete(tp
, prog
);
309 idr_destroy(&head
->handle_idr
);
310 kfree_rcu(head
, rcu
);
313 static void *cls_bpf_get(struct tcf_proto
*tp
, u32 handle
)
315 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
316 struct cls_bpf_prog
*prog
;
318 list_for_each_entry(prog
, &head
->plist
, link
) {
319 if (prog
->handle
== handle
)
326 static int cls_bpf_prog_from_ops(struct nlattr
**tb
, struct cls_bpf_prog
*prog
)
328 struct sock_filter
*bpf_ops
;
329 struct sock_fprog_kern fprog_tmp
;
331 u16 bpf_size
, bpf_num_ops
;
334 bpf_num_ops
= nla_get_u16(tb
[TCA_BPF_OPS_LEN
]);
335 if (bpf_num_ops
> BPF_MAXINSNS
|| bpf_num_ops
== 0)
338 bpf_size
= bpf_num_ops
* sizeof(*bpf_ops
);
339 if (bpf_size
!= nla_len(tb
[TCA_BPF_OPS
]))
342 bpf_ops
= kzalloc(bpf_size
, GFP_KERNEL
);
346 memcpy(bpf_ops
, nla_data(tb
[TCA_BPF_OPS
]), bpf_size
);
348 fprog_tmp
.len
= bpf_num_ops
;
349 fprog_tmp
.filter
= bpf_ops
;
351 ret
= bpf_prog_create(&fp
, &fprog_tmp
);
357 prog
->bpf_ops
= bpf_ops
;
358 prog
->bpf_num_ops
= bpf_num_ops
;
359 prog
->bpf_name
= NULL
;
365 static int cls_bpf_prog_from_efd(struct nlattr
**tb
, struct cls_bpf_prog
*prog
,
366 u32 gen_flags
, const struct tcf_proto
*tp
)
373 bpf_fd
= nla_get_u32(tb
[TCA_BPF_FD
]);
374 skip_sw
= gen_flags
& TCA_CLS_FLAGS_SKIP_SW
;
376 fp
= bpf_prog_get_type_dev(bpf_fd
, BPF_PROG_TYPE_SCHED_CLS
, skip_sw
);
380 if (tb
[TCA_BPF_NAME
]) {
381 name
= nla_memdup(tb
[TCA_BPF_NAME
], GFP_KERNEL
);
388 prog
->bpf_ops
= NULL
;
389 prog
->bpf_name
= name
;
392 if (fp
->dst_needed
&& !(tp
->q
->flags
& TCQ_F_INGRESS
))
393 netif_keep_dst(qdisc_dev(tp
->q
));
398 static int cls_bpf_set_parms(struct net
*net
, struct tcf_proto
*tp
,
399 struct cls_bpf_prog
*prog
, unsigned long base
,
400 struct nlattr
**tb
, struct nlattr
*est
, bool ovr
)
402 bool is_bpf
, is_ebpf
, have_exts
= false;
406 is_bpf
= tb
[TCA_BPF_OPS_LEN
] && tb
[TCA_BPF_OPS
];
407 is_ebpf
= tb
[TCA_BPF_FD
];
408 if ((!is_bpf
&& !is_ebpf
) || (is_bpf
&& is_ebpf
))
411 ret
= tcf_exts_validate(net
, tp
, tb
, est
, &prog
->exts
, ovr
);
415 if (tb
[TCA_BPF_FLAGS
]) {
416 u32 bpf_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS
]);
418 if (bpf_flags
& ~TCA_BPF_FLAG_ACT_DIRECT
)
421 have_exts
= bpf_flags
& TCA_BPF_FLAG_ACT_DIRECT
;
423 if (tb
[TCA_BPF_FLAGS_GEN
]) {
424 gen_flags
= nla_get_u32(tb
[TCA_BPF_FLAGS_GEN
]);
425 if (gen_flags
& ~CLS_BPF_SUPPORTED_GEN_FLAGS
||
426 !tc_flags_valid(gen_flags
))
430 prog
->exts_integrated
= have_exts
;
431 prog
->gen_flags
= gen_flags
;
433 ret
= is_bpf
? cls_bpf_prog_from_ops(tb
, prog
) :
434 cls_bpf_prog_from_efd(tb
, prog
, gen_flags
, tp
);
438 if (tb
[TCA_BPF_CLASSID
]) {
439 prog
->res
.classid
= nla_get_u32(tb
[TCA_BPF_CLASSID
]);
440 tcf_bind_filter(tp
, &prog
->res
, base
);
446 static int cls_bpf_change(struct net
*net
, struct sk_buff
*in_skb
,
447 struct tcf_proto
*tp
, unsigned long base
,
448 u32 handle
, struct nlattr
**tca
,
449 void **arg
, bool ovr
)
451 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
452 struct cls_bpf_prog
*oldprog
= *arg
;
453 struct nlattr
*tb
[TCA_BPF_MAX
+ 1];
454 struct cls_bpf_prog
*prog
;
455 unsigned long idr_index
;
458 if (tca
[TCA_OPTIONS
] == NULL
)
461 ret
= nla_parse_nested(tb
, TCA_BPF_MAX
, tca
[TCA_OPTIONS
], bpf_policy
,
466 prog
= kzalloc(sizeof(*prog
), GFP_KERNEL
);
470 ret
= tcf_exts_init(&prog
->exts
, net
, TCA_BPF_ACT
, TCA_BPF_POLICE
);
475 if (handle
&& oldprog
->handle
!= handle
) {
482 ret
= idr_alloc_ext(&head
->handle_idr
, prog
, &idr_index
,
483 1, 0x7FFFFFFF, GFP_KERNEL
);
486 prog
->handle
= idr_index
;
489 ret
= idr_alloc_ext(&head
->handle_idr
, prog
, &idr_index
,
490 handle
, handle
+ 1, GFP_KERNEL
);
494 prog
->handle
= handle
;
497 ret
= cls_bpf_set_parms(net
, tp
, prog
, base
, tb
, tca
[TCA_RATE
], ovr
);
501 ret
= cls_bpf_offload(tp
, prog
, oldprog
);
505 if (!tc_in_hw(prog
->gen_flags
))
506 prog
->gen_flags
|= TCA_CLS_FLAGS_NOT_IN_HW
;
509 idr_replace_ext(&head
->handle_idr
, prog
, handle
);
510 list_replace_rcu(&oldprog
->link
, &prog
->link
);
511 tcf_unbind_filter(tp
, &oldprog
->res
);
512 tcf_exts_get_net(&oldprog
->exts
);
513 tcf_queue_work(&oldprog
->rwork
, cls_bpf_delete_prog_work
);
515 list_add_rcu(&prog
->link
, &head
->plist
);
522 cls_bpf_free_parms(prog
);
525 idr_remove_ext(&head
->handle_idr
, prog
->handle
);
527 tcf_exts_destroy(&prog
->exts
);
532 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog
*prog
,
537 if (nla_put_u16(skb
, TCA_BPF_OPS_LEN
, prog
->bpf_num_ops
))
540 nla
= nla_reserve(skb
, TCA_BPF_OPS
, prog
->bpf_num_ops
*
541 sizeof(struct sock_filter
));
545 memcpy(nla_data(nla
), prog
->bpf_ops
, nla_len(nla
));
550 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog
*prog
,
555 if (prog
->bpf_name
&&
556 nla_put_string(skb
, TCA_BPF_NAME
, prog
->bpf_name
))
559 if (nla_put_u32(skb
, TCA_BPF_ID
, prog
->filter
->aux
->id
))
562 nla
= nla_reserve(skb
, TCA_BPF_TAG
, sizeof(prog
->filter
->tag
));
566 memcpy(nla_data(nla
), prog
->filter
->tag
, nla_len(nla
));
571 static int cls_bpf_dump(struct net
*net
, struct tcf_proto
*tp
, void *fh
,
572 struct sk_buff
*skb
, struct tcmsg
*tm
)
574 struct cls_bpf_prog
*prog
= fh
;
582 tm
->tcm_handle
= prog
->handle
;
584 cls_bpf_offload_update_stats(tp
, prog
);
586 nest
= nla_nest_start(skb
, TCA_OPTIONS
);
588 goto nla_put_failure
;
590 if (prog
->res
.classid
&&
591 nla_put_u32(skb
, TCA_BPF_CLASSID
, prog
->res
.classid
))
592 goto nla_put_failure
;
594 if (cls_bpf_is_ebpf(prog
))
595 ret
= cls_bpf_dump_ebpf_info(prog
, skb
);
597 ret
= cls_bpf_dump_bpf_info(prog
, skb
);
599 goto nla_put_failure
;
601 if (tcf_exts_dump(skb
, &prog
->exts
) < 0)
602 goto nla_put_failure
;
604 if (prog
->exts_integrated
)
605 bpf_flags
|= TCA_BPF_FLAG_ACT_DIRECT
;
606 if (bpf_flags
&& nla_put_u32(skb
, TCA_BPF_FLAGS
, bpf_flags
))
607 goto nla_put_failure
;
608 if (prog
->gen_flags
&&
609 nla_put_u32(skb
, TCA_BPF_FLAGS_GEN
, prog
->gen_flags
))
610 goto nla_put_failure
;
612 nla_nest_end(skb
, nest
);
614 if (tcf_exts_dump_stats(skb
, &prog
->exts
) < 0)
615 goto nla_put_failure
;
620 nla_nest_cancel(skb
, nest
);
624 static void cls_bpf_bind_class(void *fh
, u32 classid
, unsigned long cl
,
625 void *q
, unsigned long base
)
627 struct cls_bpf_prog
*prog
= fh
;
629 if (prog
&& prog
->res
.classid
== classid
) {
631 __tcf_bind_filter(q
, &prog
->res
, base
);
633 __tcf_unbind_filter(q
, &prog
->res
);
637 static void cls_bpf_walk(struct tcf_proto
*tp
, struct tcf_walker
*arg
)
639 struct cls_bpf_head
*head
= rtnl_dereference(tp
->root
);
640 struct cls_bpf_prog
*prog
;
642 list_for_each_entry(prog
, &head
->plist
, link
) {
643 if (arg
->count
< arg
->skip
)
645 if (arg
->fn(tp
, prog
, arg
) < 0) {
654 static struct tcf_proto_ops cls_bpf_ops __read_mostly
= {
656 .owner
= THIS_MODULE
,
657 .classify
= cls_bpf_classify
,
658 .init
= cls_bpf_init
,
659 .destroy
= cls_bpf_destroy
,
661 .change
= cls_bpf_change
,
662 .delete = cls_bpf_delete
,
663 .walk
= cls_bpf_walk
,
664 .dump
= cls_bpf_dump
,
665 .bind_class
= cls_bpf_bind_class
,
668 static int __init
cls_bpf_init_mod(void)
670 return register_tcf_proto_ops(&cls_bpf_ops
);
673 static void __exit
cls_bpf_exit_mod(void)
675 unregister_tcf_proto_ops(&cls_bpf_ops
);
678 module_init(cls_bpf_init_mod
);
679 module_exit(cls_bpf_exit_mod
);