net/sched/cls_bpf.c

   1 /*
   2  * Berkeley Packet Filter based traffic classifier
   3  *
   4  * Might be used to classify traffic through flexible, user-defined and
   5  * possibly JIT-ed BPF filters for traffic control as an alternative to
   6  * ematches.
   7  *
   8  * (C) 2013 Daniel Borkmann <dborkman@redhat.com>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  */
  14
  15 #include <linux/module.h>
  16 #include <linux/types.h>
  17 #include <linux/skbuff.h>
  18 #include <linux/filter.h>
  19 #include <linux/bpf.h>
  20
  21 #include <net/rtnetlink.h>
  22 #include <net/pkt_cls.h>
  23 #include <net/sock.h>
  24
  25 MODULE_LICENSE("GPL");
  26 MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
  27 MODULE_DESCRIPTION("TC BPF based classifier");
  28
  29 #define CLS_BPF_NAME_LEN        256
  30
  31 struct cls_bpf_head {
  32         struct list_head plist;
  33         u32 hgen;
  34         struct rcu_head rcu;
  35 };
  36
  37 struct cls_bpf_prog {
  38         struct bpf_prog *filter;
  39         struct list_head link;
  40         struct tcf_result res;
  41         bool exts_integrated;
  42         struct tcf_exts exts;
  43         u32 handle;
  44         union {
  45                 u32 bpf_fd;
  46                 u16 bpf_num_ops;
  47         };
  48         struct sock_filter *bpf_ops;
  49         const char *bpf_name;
  50         struct tcf_proto *tp;
  51         struct rcu_head rcu;
  52 };
  53
  54 static const struct nla_policy bpf_policy[TCA_BPF_MAX + 1] = {
  55         [TCA_BPF_CLASSID]       = { .type = NLA_U32 },
  56         [TCA_BPF_FLAGS]         = { .type = NLA_U32 },
  57         [TCA_BPF_FD]            = { .type = NLA_U32 },
  58         [TCA_BPF_NAME]          = { .type = NLA_NUL_STRING, .len = CLS_BPF_NAME_LEN },
  59         [TCA_BPF_OPS_LEN]       = { .type = NLA_U16 },
  60         [TCA_BPF_OPS]           = { .type = NLA_BINARY,
  61                                     .len = sizeof(struct sock_filter) * BPF_MAXINSNS },
  62 };
  63
  64 static int cls_bpf_exec_opcode(int code)
  65 {
  66         switch (code) {
  67         case TC_ACT_OK:
  68         case TC_ACT_RECLASSIFY:
  69         case TC_ACT_SHOT:
  70         case TC_ACT_PIPE:
  71         case TC_ACT_STOLEN:
  72         case TC_ACT_QUEUED:
  73         case TC_ACT_UNSPEC:
  74                 return code;
  75         default:
  76                 return TC_ACT_UNSPEC;
  77         }
  78 }
  79
  80 static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
  81                             struct tcf_result *res)
  82 {
  83         struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
  84         struct cls_bpf_prog *prog;
  85 #ifdef CONFIG_NET_CLS_ACT
  86         bool at_ingress = G_TC_AT(skb->tc_verd) & AT_INGRESS;
  87 #else
  88         bool at_ingress = false;
  89 #endif
  90         int ret = -1;
  91
  92         if (unlikely(!skb_mac_header_was_set(skb)))
  93                 return -1;
  94
  95         /* Needed here for accessing maps. */
  96         rcu_read_lock();
  97         list_for_each_entry_rcu(prog, &head->plist, link) {
  98                 int filter_res;
  99
 100                 qdisc_skb_cb(skb)->tc_classid = prog->res.classid;
 101
 102                 if (at_ingress) {
 103                         /* It is safe to push/pull even if skb_shared() */
 104                         __skb_push(skb, skb->mac_len);
 105                         filter_res = BPF_PROG_RUN(prog->filter, skb);
 106                         __skb_pull(skb, skb->mac_len);
 107                 } else {
 108                         filter_res = BPF_PROG_RUN(prog->filter, skb);
 109                 }
 110
 111                 if (prog->exts_integrated) {
 112                         res->class = prog->res.class;
 113                         res->classid = qdisc_skb_cb(skb)->tc_classid;
 114
 115                         ret = cls_bpf_exec_opcode(filter_res);
 116                         if (ret == TC_ACT_UNSPEC)
 117                                 continue;
 118                         break;
 119                 }
 120
 121                 if (filter_res == 0)
 122                         continue;
 123
 124                 *res = prog->res;
 125                 if (filter_res != -1)
 126                         res->classid = filter_res;
 127
 128                 ret = tcf_exts_exec(skb, &prog->exts, res);
 129                 if (ret < 0)
 130                         continue;
 131
 132                 break;
 133         }
 134         rcu_read_unlock();
 135
 136         return ret;
 137 }
 138
 139 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog)
 140 {
 141         return !prog->bpf_ops;
 142 }
 143
 144 static int cls_bpf_init(struct tcf_proto *tp)
 145 {
 146         struct cls_bpf_head *head;
 147
 148         head = kzalloc(sizeof(*head), GFP_KERNEL);
 149         if (head == NULL)
 150                 return -ENOBUFS;
 151
 152         INIT_LIST_HEAD_RCU(&head->plist);
 153         rcu_assign_pointer(tp->root, head);
 154
 155         return 0;
 156 }
 157
 158 static void cls_bpf_delete_prog(struct tcf_proto *tp, struct cls_bpf_prog *prog)
 159 {
 160         tcf_exts_destroy(&prog->exts);
 161
 162         if (cls_bpf_is_ebpf(prog))
 163                 bpf_prog_put(prog->filter);
 164         else
 165                 bpf_prog_destroy(prog->filter);
 166
 167         kfree(prog->bpf_name);
 168         kfree(prog->bpf_ops);
 169         kfree(prog);
 170 }
 171
 172 static void __cls_bpf_delete_prog(struct rcu_head *rcu)
 173 {
 174         struct cls_bpf_prog *prog = container_of(rcu, struct cls_bpf_prog, rcu);
 175
 176         cls_bpf_delete_prog(prog->tp, prog);
 177 }
 178
 179 static int cls_bpf_delete(struct tcf_proto *tp, unsigned long arg)
 180 {
 181         struct cls_bpf_prog *prog = (struct cls_bpf_prog *) arg;
 182
 183         list_del_rcu(&prog->link);
 184         tcf_unbind_filter(tp, &prog->res);
 185         call_rcu(&prog->rcu, __cls_bpf_delete_prog);
 186
 187         return 0;
 188 }
 189
 190 static bool cls_bpf_destroy(struct tcf_proto *tp, bool force)
 191 {
 192         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 193         struct cls_bpf_prog *prog, *tmp;
 194
 195         if (!force && !list_empty(&head->plist))
 196                 return false;
 197
 198         list_for_each_entry_safe(prog, tmp, &head->plist, link) {
 199                 list_del_rcu(&prog->link);
 200                 tcf_unbind_filter(tp, &prog->res);
 201                 call_rcu(&prog->rcu, __cls_bpf_delete_prog);
 202         }
 203
 204         RCU_INIT_POINTER(tp->root, NULL);
 205         kfree_rcu(head, rcu);
 206         return true;
 207 }
 208
 209 static unsigned long cls_bpf_get(struct tcf_proto *tp, u32 handle)
 210 {
 211         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 212         struct cls_bpf_prog *prog;
 213         unsigned long ret = 0UL;
 214
 215         if (head == NULL)
 216                 return 0UL;
 217
 218         list_for_each_entry(prog, &head->plist, link) {
 219                 if (prog->handle == handle) {
 220                         ret = (unsigned long) prog;
 221                         break;
 222                 }
 223         }
 224
 225         return ret;
 226 }
 227
 228 static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog)
 229 {
 230         struct sock_filter *bpf_ops;
 231         struct sock_fprog_kern fprog_tmp;
 232         struct bpf_prog *fp;
 233         u16 bpf_size, bpf_num_ops;
 234         int ret;
 235
 236         bpf_num_ops = nla_get_u16(tb[TCA_BPF_OPS_LEN]);
 237         if (bpf_num_ops > BPF_MAXINSNS || bpf_num_ops == 0)
 238                 return -EINVAL;
 239
 240         bpf_size = bpf_num_ops * sizeof(*bpf_ops);
 241         if (bpf_size != nla_len(tb[TCA_BPF_OPS]))
 242                 return -EINVAL;
 243
 244         bpf_ops = kzalloc(bpf_size, GFP_KERNEL);
 245         if (bpf_ops == NULL)
 246                 return -ENOMEM;
 247
 248         memcpy(bpf_ops, nla_data(tb[TCA_BPF_OPS]), bpf_size);
 249
 250         fprog_tmp.len = bpf_num_ops;
 251         fprog_tmp.filter = bpf_ops;
 252
 253         ret = bpf_prog_create(&fp, &fprog_tmp);
 254         if (ret < 0) {
 255                 kfree(bpf_ops);
 256                 return ret;
 257         }
 258
 259         prog->bpf_ops = bpf_ops;
 260         prog->bpf_num_ops = bpf_num_ops;
 261         prog->bpf_name = NULL;
 262         prog->filter = fp;
 263
 264         return 0;
 265 }
 266
 267 static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog,
 268                                  const struct tcf_proto *tp)
 269 {
 270         struct bpf_prog *fp;
 271         char *name = NULL;
 272         u32 bpf_fd;
 273
 274         bpf_fd = nla_get_u32(tb[TCA_BPF_FD]);
 275
 276         fp = bpf_prog_get(bpf_fd);
 277         if (IS_ERR(fp))
 278                 return PTR_ERR(fp);
 279
 280         if (fp->type != BPF_PROG_TYPE_SCHED_CLS) {
 281                 bpf_prog_put(fp);
 282                 return -EINVAL;
 283         }
 284
 285         if (tb[TCA_BPF_NAME]) {
 286                 name = kmemdup(nla_data(tb[TCA_BPF_NAME]),
 287                                nla_len(tb[TCA_BPF_NAME]),
 288                                GFP_KERNEL);
 289                 if (!name) {
 290                         bpf_prog_put(fp);
 291                         return -ENOMEM;
 292                 }
 293         }
 294
 295         prog->bpf_ops = NULL;
 296         prog->bpf_fd = bpf_fd;
 297         prog->bpf_name = name;
 298         prog->filter = fp;
 299
 300         return 0;
 301 }
 302
 303 static int cls_bpf_modify_existing(struct net *net, struct tcf_proto *tp,
 304                                    struct cls_bpf_prog *prog,
 305                                    unsigned long base, struct nlattr **tb,
 306                                    struct nlattr *est, bool ovr)
 307 {
 308         bool is_bpf, is_ebpf, have_exts = false;
 309         struct tcf_exts exts;
 310         u32 classid;
 311         int ret;
 312
 313         is_bpf = tb[TCA_BPF_OPS_LEN] && tb[TCA_BPF_OPS];
 314         is_ebpf = tb[TCA_BPF_FD];
 315
 316         if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf) ||
 317             !tb[TCA_BPF_CLASSID])
 318                 return -EINVAL;
 319
 320         tcf_exts_init(&exts, TCA_BPF_ACT, TCA_BPF_POLICE);
 321         ret = tcf_exts_validate(net, tp, tb, est, &exts, ovr);
 322         if (ret < 0)
 323                 return ret;
 324
 325         classid = nla_get_u32(tb[TCA_BPF_CLASSID]);
 326         if (tb[TCA_BPF_FLAGS]) {
 327                 u32 bpf_flags = nla_get_u32(tb[TCA_BPF_FLAGS]);
 328
 329                 if (bpf_flags & ~TCA_BPF_FLAG_ACT_DIRECT) {
 330                         tcf_exts_destroy(&exts);
 331                         return -EINVAL;
 332                 }
 333
 334                 have_exts = bpf_flags & TCA_BPF_FLAG_ACT_DIRECT;
 335         }
 336
 337         prog->res.classid = classid;
 338         prog->exts_integrated = have_exts;
 339
 340         ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog) :
 341                        cls_bpf_prog_from_efd(tb, prog, tp);
 342         if (ret < 0) {
 343                 tcf_exts_destroy(&exts);
 344                 return ret;
 345         }
 346
 347         tcf_bind_filter(tp, &prog->res, base);
 348         tcf_exts_change(tp, &prog->exts, &exts);
 349
 350         return 0;
 351 }
 352
 353 static u32 cls_bpf_grab_new_handle(struct tcf_proto *tp,
 354                                    struct cls_bpf_head *head)
 355 {
 356         unsigned int i = 0x80000000;
 357         u32 handle;
 358
 359         do {
 360                 if (++head->hgen == 0x7FFFFFFF)
 361                         head->hgen = 1;
 362         } while (--i > 0 && cls_bpf_get(tp, head->hgen));
 363
 364         if (unlikely(i == 0)) {
 365                 pr_err("Insufficient number of handles\n");
 366                 handle = 0;
 367         } else {
 368                 handle = head->hgen;
 369         }
 370
 371         return handle;
 372 }
 373
 374 static int cls_bpf_change(struct net *net, struct sk_buff *in_skb,
 375                           struct tcf_proto *tp, unsigned long base,
 376                           u32 handle, struct nlattr **tca,
 377                           unsigned long *arg, bool ovr)
 378 {
 379         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 380         struct cls_bpf_prog *oldprog = (struct cls_bpf_prog *) *arg;
 381         struct nlattr *tb[TCA_BPF_MAX + 1];
 382         struct cls_bpf_prog *prog;
 383         int ret;
 384
 385         if (tca[TCA_OPTIONS] == NULL)
 386                 return -EINVAL;
 387
 388         ret = nla_parse_nested(tb, TCA_BPF_MAX, tca[TCA_OPTIONS], bpf_policy);
 389         if (ret < 0)
 390                 return ret;
 391
 392         prog = kzalloc(sizeof(*prog), GFP_KERNEL);
 393         if (!prog)
 394                 return -ENOBUFS;
 395
 396         tcf_exts_init(&prog->exts, TCA_BPF_ACT, TCA_BPF_POLICE);
 397
 398         if (oldprog) {
 399                 if (handle && oldprog->handle != handle) {
 400                         ret = -EINVAL;
 401                         goto errout;
 402                 }
 403         }
 404
 405         if (handle == 0)
 406                 prog->handle = cls_bpf_grab_new_handle(tp, head);
 407         else
 408                 prog->handle = handle;
 409         if (prog->handle == 0) {
 410                 ret = -EINVAL;
 411                 goto errout;
 412         }
 413
 414         ret = cls_bpf_modify_existing(net, tp, prog, base, tb, tca[TCA_RATE], ovr);
 415         if (ret < 0)
 416                 goto errout;
 417
 418         if (oldprog) {
 419                 list_replace_rcu(&oldprog->link, &prog->link);
 420                 tcf_unbind_filter(tp, &oldprog->res);
 421                 call_rcu(&oldprog->rcu, __cls_bpf_delete_prog);
 422         } else {
 423                 list_add_rcu(&prog->link, &head->plist);
 424         }
 425
 426         *arg = (unsigned long) prog;
 427         return 0;
 428 errout:
 429         kfree(prog);
 430
 431         return ret;
 432 }
 433
 434 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog *prog,
 435                                  struct sk_buff *skb)
 436 {
 437         struct nlattr *nla;
 438
 439         if (nla_put_u16(skb, TCA_BPF_OPS_LEN, prog->bpf_num_ops))
 440                 return -EMSGSIZE;
 441
 442         nla = nla_reserve(skb, TCA_BPF_OPS, prog->bpf_num_ops *
 443                           sizeof(struct sock_filter));
 444         if (nla == NULL)
 445                 return -EMSGSIZE;
 446
 447         memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));
 448
 449         return 0;
 450 }
 451
 452 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog *prog,
 453                                   struct sk_buff *skb)
 454 {
 455         if (nla_put_u32(skb, TCA_BPF_FD, prog->bpf_fd))
 456                 return -EMSGSIZE;
 457
 458         if (prog->bpf_name &&
 459             nla_put_string(skb, TCA_BPF_NAME, prog->bpf_name))
 460                 return -EMSGSIZE;
 461
 462         return 0;
 463 }
 464
 465 static int cls_bpf_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
 466                         struct sk_buff *skb, struct tcmsg *tm)
 467 {
 468         struct cls_bpf_prog *prog = (struct cls_bpf_prog *) fh;
 469         struct nlattr *nest;
 470         int ret;
 471
 472         if (prog == NULL)
 473                 return skb->len;
 474
 475         tm->tcm_handle = prog->handle;
 476
 477         nest = nla_nest_start(skb, TCA_OPTIONS);
 478         if (nest == NULL)
 479                 goto nla_put_failure;
 480
 481         if (nla_put_u32(skb, TCA_BPF_CLASSID, prog->res.classid))
 482                 goto nla_put_failure;
 483
 484         if (cls_bpf_is_ebpf(prog))
 485                 ret = cls_bpf_dump_ebpf_info(prog, skb);
 486         else
 487                 ret = cls_bpf_dump_bpf_info(prog, skb);
 488         if (ret)
 489                 goto nla_put_failure;
 490
 491         if (tcf_exts_dump(skb, &prog->exts) < 0)
 492                 goto nla_put_failure;
 493
 494         nla_nest_end(skb, nest);
 495
 496         if (tcf_exts_dump_stats(skb, &prog->exts) < 0)
 497                 goto nla_put_failure;
 498
 499         return skb->len;
 500
 501 nla_put_failure:
 502         nla_nest_cancel(skb, nest);
 503         return -1;
 504 }
 505
 506 static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg)
 507 {
 508         struct cls_bpf_head *head = rtnl_dereference(tp->root);
 509         struct cls_bpf_prog *prog;
 510
 511         list_for_each_entry(prog, &head->plist, link) {
 512                 if (arg->count < arg->skip)
 513                         goto skip;
 514                 if (arg->fn(tp, (unsigned long) prog, arg) < 0) {
 515                         arg->stop = 1;
 516                         break;
 517                 }
 518 skip:
 519                 arg->count++;
 520         }
 521 }
 522
 523 static struct tcf_proto_ops cls_bpf_ops __read_mostly = {
 524         .kind           =       "bpf",
 525         .owner          =       THIS_MODULE,
 526         .classify       =       cls_bpf_classify,
 527         .init           =       cls_bpf_init,
 528         .destroy        =       cls_bpf_destroy,
 529         .get            =       cls_bpf_get,
 530         .change         =       cls_bpf_change,
 531         .delete         =       cls_bpf_delete,
 532         .walk           =       cls_bpf_walk,
 533         .dump           =       cls_bpf_dump,
 534 };
 535
 536 static int __init cls_bpf_init_mod(void)
 537 {
 538         return register_tcf_proto_ops(&cls_bpf_ops);
 539 }
 540
 541 static void __exit cls_bpf_exit_mod(void)
 542 {
 543         unregister_tcf_proto_ops(&cls_bpf_ops);
 544 }
 545
 546 module_init(cls_bpf_init_mod);
 547 module_exit(cls_bpf_exit_mod);