kernel/bpf/syscall.c

   1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful, but
   8  * WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  10  * General Public License for more details.
  11  */
  12 #include <linux/bpf.h>
  13 #include <linux/syscalls.h>
  14 #include <linux/slab.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/mmzone.h>
  17 #include <linux/anon_inodes.h>
  18 #include <linux/file.h>
  19 #include <linux/license.h>
  20 #include <linux/filter.h>
  21 #include <linux/version.h>
  22 #include <linux/kernel.h>
  23 #include <linux/module.h>
  24
  25 DEFINE_PER_CPU(int, bpf_prog_active);
  26
  27 int sysctl_unprivileged_bpf_disabled __read_mostly;
  28
  29 static LIST_HEAD(bpf_map_types);
  30
  31 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
  32 {
  33         struct bpf_map_type_list *tl;
  34         struct bpf_map *map;
  35
  36         list_for_each_entry(tl, &bpf_map_types, list_node) {
  37                 if (tl->type == attr->map_type) {
  38                         map = tl->ops->map_alloc(attr);
  39                         if (IS_ERR(map))
  40                                 return map;
  41                         map->ops = tl->ops;
  42                         map->map_type = attr->map_type;
  43                         return map;
  44                 }
  45         }
  46         return ERR_PTR(-EINVAL);
  47 }
  48
  49 /* boot time registration of different map implementations */
  50 void bpf_register_map_type(struct bpf_map_type_list *tl)
  51 {
  52         list_add(&tl->list_node, &bpf_map_types);
  53 }
  54
  55 void *bpf_map_area_alloc(size_t size)
  56 {
  57         /* We definitely need __GFP_NORETRY, so OOM killer doesn't
  58          * trigger under memory pressure as we really just want to
  59          * fail instead.
  60          */
  61         const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO;
  62         void *area;
  63
  64         if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
  65                 area = kmalloc(size, GFP_USER | flags);
  66                 if (area != NULL)
  67                         return area;
  68         }
  69
  70         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | flags,
  71                          PAGE_KERNEL);
  72 }
  73
  74 void bpf_map_area_free(void *area)
  75 {
  76         kvfree(area);
  77 }
  78
  79 int bpf_map_precharge_memlock(u32 pages)
  80 {
  81         struct user_struct *user = get_current_user();
  82         unsigned long memlock_limit, cur;
  83
  84         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  85         cur = atomic_long_read(&user->locked_vm);
  86         free_uid(user);
  87         if (cur + pages > memlock_limit)
  88                 return -EPERM;
  89         return 0;
  90 }
  91
  92 static int bpf_map_charge_memlock(struct bpf_map *map)
  93 {
  94         struct user_struct *user = get_current_user();
  95         unsigned long memlock_limit;
  96
  97         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  98
  99         atomic_long_add(map->pages, &user->locked_vm);
 100
 101         if (atomic_long_read(&user->locked_vm) > memlock_limit) {
 102                 atomic_long_sub(map->pages, &user->locked_vm);
 103                 free_uid(user);
 104                 return -EPERM;
 105         }
 106         map->user = user;
 107         return 0;
 108 }
 109
 110 static void bpf_map_uncharge_memlock(struct bpf_map *map)
 111 {
 112         struct user_struct *user = map->user;
 113
 114         atomic_long_sub(map->pages, &user->locked_vm);
 115         free_uid(user);
 116 }
 117
 118 /* called from workqueue */
 119 static void bpf_map_free_deferred(struct work_struct *work)
 120 {
 121         struct bpf_map *map = container_of(work, struct bpf_map, work);
 122
 123         bpf_map_uncharge_memlock(map);
 124         /* implementation dependent freeing */
 125         map->ops->map_free(map);
 126 }
 127
 128 static void bpf_map_put_uref(struct bpf_map *map)
 129 {
 130         if (atomic_dec_and_test(&map->usercnt)) {
 131                 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
 132                         bpf_fd_array_map_clear(map);
 133         }
 134 }
 135
 136 /* decrement map refcnt and schedule it for freeing via workqueue
 137  * (unrelying map implementation ops->map_free() might sleep)
 138  */
 139 void bpf_map_put(struct bpf_map *map)
 140 {
 141         if (atomic_dec_and_test(&map->refcnt)) {
 142                 INIT_WORK(&map->work, bpf_map_free_deferred);
 143                 schedule_work(&map->work);
 144         }
 145 }
 146
 147 void bpf_map_put_with_uref(struct bpf_map *map)
 148 {
 149         bpf_map_put_uref(map);
 150         bpf_map_put(map);
 151 }
 152
 153 static int bpf_map_release(struct inode *inode, struct file *filp)
 154 {
 155         struct bpf_map *map = filp->private_data;
 156
 157         if (map->ops->map_release)
 158                 map->ops->map_release(map, filp);
 159
 160         bpf_map_put_with_uref(map);
 161         return 0;
 162 }
 163
 164 #ifdef CONFIG_PROC_FS
 165 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 166 {
 167         const struct bpf_map *map = filp->private_data;
 168         const struct bpf_array *array;
 169         u32 owner_prog_type = 0;
 170
 171         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 172                 array = container_of(map, struct bpf_array, map);
 173                 owner_prog_type = array->owner_prog_type;
 174         }
 175
 176         seq_printf(m,
 177                    "map_type:\t%u\n"
 178                    "key_size:\t%u\n"
 179                    "value_size:\t%u\n"
 180                    "max_entries:\t%u\n"
 181                    "map_flags:\t%#x\n"
 182                    "memlock:\t%llu\n",
 183                    map->map_type,
 184                    map->key_size,
 185                    map->value_size,
 186                    map->max_entries,
 187                    map->map_flags,
 188                    map->pages * 1ULL << PAGE_SHIFT);
 189
 190         if (owner_prog_type)
 191                 seq_printf(m, "owner_prog_type:\t%u\n",
 192                            owner_prog_type);
 193 }
 194 #endif
 195
 196 static const struct file_operations bpf_map_fops = {
 197 #ifdef CONFIG_PROC_FS
 198         .show_fdinfo    = bpf_map_show_fdinfo,
 199 #endif
 200         .release        = bpf_map_release,
 201 };
 202
 203 int bpf_map_new_fd(struct bpf_map *map)
 204 {
 205         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 206                                 O_RDWR | O_CLOEXEC);
 207 }
 208
 209 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 210 #define CHECK_ATTR(CMD) \
 211         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 212                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 213                    sizeof(*attr) - \
 214                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 215                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 216
 217 #define BPF_MAP_CREATE_LAST_FIELD map_flags
 218 /* called via syscall */
 219 static int map_create(union bpf_attr *attr)
 220 {
 221         struct bpf_map *map;
 222         int err;
 223
 224         err = CHECK_ATTR(BPF_MAP_CREATE);
 225         if (err)
 226                 return -EINVAL;
 227
 228         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 229         map = find_and_alloc_map(attr);
 230         if (IS_ERR(map))
 231                 return PTR_ERR(map);
 232
 233         atomic_set(&map->refcnt, 1);
 234         atomic_set(&map->usercnt, 1);
 235
 236         err = bpf_map_charge_memlock(map);
 237         if (err)
 238                 goto free_map_nouncharge;
 239
 240         err = bpf_map_new_fd(map);
 241         if (err < 0)
 242                 /* failed to allocate fd */
 243                 goto free_map;
 244
 245         return err;
 246
 247 free_map:
 248         bpf_map_uncharge_memlock(map);
 249 free_map_nouncharge:
 250         map->ops->map_free(map);
 251         return err;
 252 }
 253
 254 /* if error is returned, fd is released.
 255  * On success caller should complete fd access with matching fdput()
 256  */
 257 struct bpf_map *__bpf_map_get(struct fd f)
 258 {
 259         if (!f.file)
 260                 return ERR_PTR(-EBADF);
 261         if (f.file->f_op != &bpf_map_fops) {
 262                 fdput(f);
 263                 return ERR_PTR(-EINVAL);
 264         }
 265
 266         return f.file->private_data;
 267 }
 268
 269 /* prog's and map's refcnt limit */
 270 #define BPF_MAX_REFCNT 32768
 271
 272 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
 273 {
 274         if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
 275                 atomic_dec(&map->refcnt);
 276                 return ERR_PTR(-EBUSY);
 277         }
 278         if (uref)
 279                 atomic_inc(&map->usercnt);
 280         return map;
 281 }
 282
 283 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 284 {
 285         struct fd f = fdget(ufd);
 286         struct bpf_map *map;
 287
 288         map = __bpf_map_get(f);
 289         if (IS_ERR(map))
 290                 return map;
 291
 292         map = bpf_map_inc(map, true);
 293         fdput(f);
 294
 295         return map;
 296 }
 297
 298 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 299 {
 300         return -ENOTSUPP;
 301 }
 302
 303 /* last field in 'union bpf_attr' used by this command */
 304 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
 305
 306 static int map_lookup_elem(union bpf_attr *attr)
 307 {
 308         void __user *ukey = u64_to_user_ptr(attr->key);
 309         void __user *uvalue = u64_to_user_ptr(attr->value);
 310         int ufd = attr->map_fd;
 311         struct bpf_map *map;
 312         void *key, *value, *ptr;
 313         u32 value_size;
 314         struct fd f;
 315         int err;
 316
 317         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
 318                 return -EINVAL;
 319
 320         f = fdget(ufd);
 321         map = __bpf_map_get(f);
 322         if (IS_ERR(map))
 323                 return PTR_ERR(map);
 324
 325         err = -ENOMEM;
 326         key = kmalloc(map->key_size, GFP_USER);
 327         if (!key)
 328                 goto err_put;
 329
 330         err = -EFAULT;
 331         if (copy_from_user(key, ukey, map->key_size) != 0)
 332                 goto free_key;
 333
 334         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 335             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 336             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
 337                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 338         else
 339                 value_size = map->value_size;
 340
 341         err = -ENOMEM;
 342         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 343         if (!value)
 344                 goto free_key;
 345
 346         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 347             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 348                 err = bpf_percpu_hash_copy(map, key, value);
 349         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 350                 err = bpf_percpu_array_copy(map, key, value);
 351         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 352                 err = bpf_stackmap_copy(map, key, value);
 353         } else {
 354                 rcu_read_lock();
 355                 ptr = map->ops->map_lookup_elem(map, key);
 356                 if (ptr)
 357                         memcpy(value, ptr, value_size);
 358                 rcu_read_unlock();
 359                 err = ptr ? 0 : -ENOENT;
 360         }
 361
 362         if (err)
 363                 goto free_value;
 364
 365         err = -EFAULT;
 366         if (copy_to_user(uvalue, value, value_size) != 0)
 367                 goto free_value;
 368
 369         err = 0;
 370
 371 free_value:
 372         kfree(value);
 373 free_key:
 374         kfree(key);
 375 err_put:
 376         fdput(f);
 377         return err;
 378 }
 379
 380 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
 381
 382 static int map_update_elem(union bpf_attr *attr)
 383 {
 384         void __user *ukey = u64_to_user_ptr(attr->key);
 385         void __user *uvalue = u64_to_user_ptr(attr->value);
 386         int ufd = attr->map_fd;
 387         struct bpf_map *map;
 388         void *key, *value;
 389         u32 value_size;
 390         struct fd f;
 391         int err;
 392
 393         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
 394                 return -EINVAL;
 395
 396         f = fdget(ufd);
 397         map = __bpf_map_get(f);
 398         if (IS_ERR(map))
 399                 return PTR_ERR(map);
 400
 401         err = -ENOMEM;
 402         key = kmalloc(map->key_size, GFP_USER);
 403         if (!key)
 404                 goto err_put;
 405
 406         err = -EFAULT;
 407         if (copy_from_user(key, ukey, map->key_size) != 0)
 408                 goto free_key;
 409
 410         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 411             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 412             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
 413                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 414         else
 415                 value_size = map->value_size;
 416
 417         err = -ENOMEM;
 418         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 419         if (!value)
 420                 goto free_key;
 421
 422         err = -EFAULT;
 423         if (copy_from_user(value, uvalue, value_size) != 0)
 424                 goto free_value;
 425
 426         /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
 427          * inside bpf map update or delete otherwise deadlocks are possible
 428          */
 429         preempt_disable();
 430         __this_cpu_inc(bpf_prog_active);
 431         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 432             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 433                 err = bpf_percpu_hash_update(map, key, value, attr->flags);
 434         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 435                 err = bpf_percpu_array_update(map, key, value, attr->flags);
 436         } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
 437                    map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
 438                    map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY) {
 439                 rcu_read_lock();
 440                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 441                                                    attr->flags);
 442                 rcu_read_unlock();
 443         } else {
 444                 rcu_read_lock();
 445                 err = map->ops->map_update_elem(map, key, value, attr->flags);
 446                 rcu_read_unlock();
 447         }
 448         __this_cpu_dec(bpf_prog_active);
 449         preempt_enable();
 450
 451 free_value:
 452         kfree(value);
 453 free_key:
 454         kfree(key);
 455 err_put:
 456         fdput(f);
 457         return err;
 458 }
 459
 460 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
 461
 462 static int map_delete_elem(union bpf_attr *attr)
 463 {
 464         void __user *ukey = u64_to_user_ptr(attr->key);
 465         int ufd = attr->map_fd;
 466         struct bpf_map *map;
 467         struct fd f;
 468         void *key;
 469         int err;
 470
 471         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
 472                 return -EINVAL;
 473
 474         f = fdget(ufd);
 475         map = __bpf_map_get(f);
 476         if (IS_ERR(map))
 477                 return PTR_ERR(map);
 478
 479         err = -ENOMEM;
 480         key = kmalloc(map->key_size, GFP_USER);
 481         if (!key)
 482                 goto err_put;
 483
 484         err = -EFAULT;
 485         if (copy_from_user(key, ukey, map->key_size) != 0)
 486                 goto free_key;
 487
 488         preempt_disable();
 489         __this_cpu_inc(bpf_prog_active);
 490         rcu_read_lock();
 491         err = map->ops->map_delete_elem(map, key);
 492         rcu_read_unlock();
 493         __this_cpu_dec(bpf_prog_active);
 494         preempt_enable();
 495
 496 free_key:
 497         kfree(key);
 498 err_put:
 499         fdput(f);
 500         return err;
 501 }
 502
 503 /* last field in 'union bpf_attr' used by this command */
 504 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
 505
 506 static int map_get_next_key(union bpf_attr *attr)
 507 {
 508         void __user *ukey = u64_to_user_ptr(attr->key);
 509         void __user *unext_key = u64_to_user_ptr(attr->next_key);
 510         int ufd = attr->map_fd;
 511         struct bpf_map *map;
 512         void *key, *next_key;
 513         struct fd f;
 514         int err;
 515
 516         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
 517                 return -EINVAL;
 518
 519         f = fdget(ufd);
 520         map = __bpf_map_get(f);
 521         if (IS_ERR(map))
 522                 return PTR_ERR(map);
 523
 524         err = -ENOMEM;
 525         key = kmalloc(map->key_size, GFP_USER);
 526         if (!key)
 527                 goto err_put;
 528
 529         err = -EFAULT;
 530         if (copy_from_user(key, ukey, map->key_size) != 0)
 531                 goto free_key;
 532
 533         err = -ENOMEM;
 534         next_key = kmalloc(map->key_size, GFP_USER);
 535         if (!next_key)
 536                 goto free_key;
 537
 538         rcu_read_lock();
 539         err = map->ops->map_get_next_key(map, key, next_key);
 540         rcu_read_unlock();
 541         if (err)
 542                 goto free_next_key;
 543
 544         err = -EFAULT;
 545         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
 546                 goto free_next_key;
 547
 548         err = 0;
 549
 550 free_next_key:
 551         kfree(next_key);
 552 free_key:
 553         kfree(key);
 554 err_put:
 555         fdput(f);
 556         return err;
 557 }
 558
 559 static LIST_HEAD(bpf_prog_types);
 560
 561 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
 562 {
 563         struct bpf_prog_type_list *tl;
 564
 565         list_for_each_entry(tl, &bpf_prog_types, list_node) {
 566                 if (tl->type == type) {
 567                         prog->aux->ops = tl->ops;
 568                         prog->type = type;
 569                         return 0;
 570                 }
 571         }
 572
 573         return -EINVAL;
 574 }
 575
 576 void bpf_register_prog_type(struct bpf_prog_type_list *tl)
 577 {
 578         list_add(&tl->list_node, &bpf_prog_types);
 579 }
 580
 581 /* fixup insn->imm field of bpf_call instructions:
 582  * if (insn->imm == BPF_FUNC_map_lookup_elem)
 583  *      insn->imm = bpf_map_lookup_elem - __bpf_call_base;
 584  * else if (insn->imm == BPF_FUNC_map_update_elem)
 585  *      insn->imm = bpf_map_update_elem - __bpf_call_base;
 586  * else ...
 587  *
 588  * this function is called after eBPF program passed verification
 589  */
 590 static void fixup_bpf_calls(struct bpf_prog *prog)
 591 {
 592         const struct bpf_func_proto *fn;
 593         int i;
 594
 595         for (i = 0; i < prog->len; i++) {
 596                 struct bpf_insn *insn = &prog->insnsi[i];
 597
 598                 if (insn->code == (BPF_JMP | BPF_CALL)) {
 599                         /* we reach here when program has bpf_call instructions
 600                          * and it passed bpf_check(), means that
 601                          * ops->get_func_proto must have been supplied, check it
 602                          */
 603                         BUG_ON(!prog->aux->ops->get_func_proto);
 604
 605                         if (insn->imm == BPF_FUNC_get_route_realm)
 606                                 prog->dst_needed = 1;
 607                         if (insn->imm == BPF_FUNC_get_prandom_u32)
 608                                 bpf_user_rnd_init_once();
 609                         if (insn->imm == BPF_FUNC_xdp_adjust_head)
 610                                 prog->xdp_adjust_head = 1;
 611                         if (insn->imm == BPF_FUNC_tail_call) {
 612                                 /* mark bpf_tail_call as different opcode
 613                                  * to avoid conditional branch in
 614                                  * interpeter for every normal call
 615                                  * and to prevent accidental JITing by
 616                                  * JIT compiler that doesn't support
 617                                  * bpf_tail_call yet
 618                                  */
 619                                 insn->imm = 0;
 620                                 insn->code |= BPF_X;
 621                                 continue;
 622                         }
 623
 624                         fn = prog->aux->ops->get_func_proto(insn->imm);
 625                         /* all functions that have prototype and verifier allowed
 626                          * programs to call them, must be real in-kernel functions
 627                          */
 628                         BUG_ON(!fn->func);
 629                         insn->imm = fn->func - __bpf_call_base;
 630                 }
 631         }
 632 }
 633
 634 /* drop refcnt on maps used by eBPF program and free auxilary data */
 635 static void free_used_maps(struct bpf_prog_aux *aux)
 636 {
 637         int i;
 638
 639         for (i = 0; i < aux->used_map_cnt; i++)
 640                 bpf_map_put(aux->used_maps[i]);
 641
 642         kfree(aux->used_maps);
 643 }
 644
 645 int __bpf_prog_charge(struct user_struct *user, u32 pages)
 646 {
 647         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 648         unsigned long user_bufs;
 649
 650         if (user) {
 651                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
 652                 if (user_bufs > memlock_limit) {
 653                         atomic_long_sub(pages, &user->locked_vm);
 654                         return -EPERM;
 655                 }
 656         }
 657
 658         return 0;
 659 }
 660
 661 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
 662 {
 663         if (user)
 664                 atomic_long_sub(pages, &user->locked_vm);
 665 }
 666
 667 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
 668 {
 669         struct user_struct *user = get_current_user();
 670         int ret;
 671
 672         ret = __bpf_prog_charge(user, prog->pages);
 673         if (ret) {
 674                 free_uid(user);
 675                 return ret;
 676         }
 677
 678         prog->aux->user = user;
 679         return 0;
 680 }
 681
 682 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
 683 {
 684         struct user_struct *user = prog->aux->user;
 685
 686         __bpf_prog_uncharge(user, prog->pages);
 687         free_uid(user);
 688 }
 689
 690 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
 691 {
 692         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
 693
 694         free_used_maps(aux);
 695         bpf_prog_uncharge_memlock(aux->prog);
 696         bpf_prog_free(aux->prog);
 697 }
 698
 699 void bpf_prog_put(struct bpf_prog *prog)
 700 {
 701         if (atomic_dec_and_test(&prog->aux->refcnt))
 702                 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
 703 }
 704 EXPORT_SYMBOL_GPL(bpf_prog_put);
 705
 706 static int bpf_prog_release(struct inode *inode, struct file *filp)
 707 {
 708         struct bpf_prog *prog = filp->private_data;
 709
 710         bpf_prog_put(prog);
 711         return 0;
 712 }
 713
 714 #ifdef CONFIG_PROC_FS
 715 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
 716 {
 717         const struct bpf_prog *prog = filp->private_data;
 718         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
 719
 720         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
 721         seq_printf(m,
 722                    "prog_type:\t%u\n"
 723                    "prog_jited:\t%u\n"
 724                    "prog_tag:\t%s\n"
 725                    "memlock:\t%llu\n",
 726                    prog->type,
 727                    prog->jited,
 728                    prog_tag,
 729                    prog->pages * 1ULL << PAGE_SHIFT);
 730 }
 731 #endif
 732
 733 static const struct file_operations bpf_prog_fops = {
 734 #ifdef CONFIG_PROC_FS
 735         .show_fdinfo    = bpf_prog_show_fdinfo,
 736 #endif
 737         .release        = bpf_prog_release,
 738 };
 739
 740 int bpf_prog_new_fd(struct bpf_prog *prog)
 741 {
 742         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
 743                                 O_RDWR | O_CLOEXEC);
 744 }
 745
 746 static struct bpf_prog *____bpf_prog_get(struct fd f)
 747 {
 748         if (!f.file)
 749                 return ERR_PTR(-EBADF);
 750         if (f.file->f_op != &bpf_prog_fops) {
 751                 fdput(f);
 752                 return ERR_PTR(-EINVAL);
 753         }
 754
 755         return f.file->private_data;
 756 }
 757
 758 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
 759 {
 760         if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
 761                 atomic_sub(i, &prog->aux->refcnt);
 762                 return ERR_PTR(-EBUSY);
 763         }
 764         return prog;
 765 }
 766 EXPORT_SYMBOL_GPL(bpf_prog_add);
 767
 768 void bpf_prog_sub(struct bpf_prog *prog, int i)
 769 {
 770         /* Only to be used for undoing previous bpf_prog_add() in some
 771          * error path. We still know that another entity in our call
 772          * path holds a reference to the program, thus atomic_sub() can
 773          * be safely used in such cases!
 774          */
 775         WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
 776 }
 777 EXPORT_SYMBOL_GPL(bpf_prog_sub);
 778
 779 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
 780 {
 781         return bpf_prog_add(prog, 1);
 782 }
 783 EXPORT_SYMBOL_GPL(bpf_prog_inc);
 784
 785 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
 786 {
 787         struct fd f = fdget(ufd);
 788         struct bpf_prog *prog;
 789
 790         prog = ____bpf_prog_get(f);
 791         if (IS_ERR(prog))
 792                 return prog;
 793         if (type && prog->type != *type) {
 794                 prog = ERR_PTR(-EINVAL);
 795                 goto out;
 796         }
 797
 798         prog = bpf_prog_inc(prog);
 799 out:
 800         fdput(f);
 801         return prog;
 802 }
 803
 804 struct bpf_prog *bpf_prog_get(u32 ufd)
 805 {
 806         return __bpf_prog_get(ufd, NULL);
 807 }
 808
 809 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
 810 {
 811         return __bpf_prog_get(ufd, &type);
 812 }
 813 EXPORT_SYMBOL_GPL(bpf_prog_get_type);
 814
 815 /* last field in 'union bpf_attr' used by this command */
 816 #define BPF_PROG_LOAD_LAST_FIELD kern_version
 817
 818 static int bpf_prog_load(union bpf_attr *attr)
 819 {
 820         enum bpf_prog_type type = attr->prog_type;
 821         struct bpf_prog *prog;
 822         int err;
 823         char license[128];
 824         bool is_gpl;
 825
 826         if (CHECK_ATTR(BPF_PROG_LOAD))
 827                 return -EINVAL;
 828
 829         /* copy eBPF program license from user space */
 830         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
 831                               sizeof(license) - 1) < 0)
 832                 return -EFAULT;
 833         license[sizeof(license) - 1] = 0;
 834
 835         /* eBPF programs must be GPL compatible to use GPL-ed functions */
 836         is_gpl = license_is_gpl_compatible(license);
 837
 838         if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS)
 839                 return -E2BIG;
 840
 841         if (type == BPF_PROG_TYPE_KPROBE &&
 842             attr->kern_version != LINUX_VERSION_CODE)
 843                 return -EINVAL;
 844
 845         if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
 846                 return -EPERM;
 847
 848         /* plain bpf_prog allocation */
 849         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
 850         if (!prog)
 851                 return -ENOMEM;
 852
 853         err = bpf_prog_charge_memlock(prog);
 854         if (err)
 855                 goto free_prog_nouncharge;
 856
 857         prog->len = attr->insn_cnt;
 858
 859         err = -EFAULT;
 860         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
 861                            bpf_prog_insn_size(prog)) != 0)
 862                 goto free_prog;
 863
 864         prog->orig_prog = NULL;
 865         prog->jited = 0;
 866
 867         atomic_set(&prog->aux->refcnt, 1);
 868         prog->gpl_compatible = is_gpl ? 1 : 0;
 869
 870         /* find program type: socket_filter vs tracing_filter */
 871         err = find_prog_type(type, prog);
 872         if (err < 0)
 873                 goto free_prog;
 874
 875         /* run eBPF verifier */
 876         err = bpf_check(&prog, attr);
 877         if (err < 0)
 878                 goto free_used_maps;
 879
 880         /* fixup BPF_CALL->imm field */
 881         fixup_bpf_calls(prog);
 882
 883         /* eBPF program is ready to be JITed */
 884         prog = bpf_prog_select_runtime(prog, &err);
 885         if (err < 0)
 886                 goto free_used_maps;
 887
 888         err = bpf_prog_new_fd(prog);
 889         if (err < 0)
 890                 /* failed to allocate fd */
 891                 goto free_used_maps;
 892
 893         return err;
 894
 895 free_used_maps:
 896         free_used_maps(prog->aux);
 897 free_prog:
 898         bpf_prog_uncharge_memlock(prog);
 899 free_prog_nouncharge:
 900         bpf_prog_free(prog);
 901         return err;
 902 }
 903
 904 #define BPF_OBJ_LAST_FIELD bpf_fd
 905
 906 static int bpf_obj_pin(const union bpf_attr *attr)
 907 {
 908         if (CHECK_ATTR(BPF_OBJ))
 909                 return -EINVAL;
 910
 911         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
 912 }
 913
 914 static int bpf_obj_get(const union bpf_attr *attr)
 915 {
 916         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
 917                 return -EINVAL;
 918
 919         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
 920 }
 921
 922 #ifdef CONFIG_CGROUP_BPF
 923
 924 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
 925
 926 static int bpf_prog_attach(const union bpf_attr *attr)
 927 {
 928         enum bpf_prog_type ptype;
 929         struct bpf_prog *prog;
 930         struct cgroup *cgrp;
 931         int ret;
 932
 933         if (!capable(CAP_NET_ADMIN))
 934                 return -EPERM;
 935
 936         if (CHECK_ATTR(BPF_PROG_ATTACH))
 937                 return -EINVAL;
 938
 939         if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE)
 940                 return -EINVAL;
 941
 942         switch (attr->attach_type) {
 943         case BPF_CGROUP_INET_INGRESS:
 944         case BPF_CGROUP_INET_EGRESS:
 945                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
 946                 break;
 947         case BPF_CGROUP_INET_SOCK_CREATE:
 948                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
 949                 break;
 950         default:
 951                 return -EINVAL;
 952         }
 953
 954         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
 955         if (IS_ERR(prog))
 956                 return PTR_ERR(prog);
 957
 958         cgrp = cgroup_get_from_fd(attr->target_fd);
 959         if (IS_ERR(cgrp)) {
 960                 bpf_prog_put(prog);
 961                 return PTR_ERR(cgrp);
 962         }
 963
 964         ret = cgroup_bpf_update(cgrp, prog, attr->attach_type,
 965                                 attr->attach_flags & BPF_F_ALLOW_OVERRIDE);
 966         if (ret)
 967                 bpf_prog_put(prog);
 968         cgroup_put(cgrp);
 969
 970         return ret;
 971 }
 972
 973 #define BPF_PROG_DETACH_LAST_FIELD attach_type
 974
 975 static int bpf_prog_detach(const union bpf_attr *attr)
 976 {
 977         struct cgroup *cgrp;
 978         int ret;
 979
 980         if (!capable(CAP_NET_ADMIN))
 981                 return -EPERM;
 982
 983         if (CHECK_ATTR(BPF_PROG_DETACH))
 984                 return -EINVAL;
 985
 986         switch (attr->attach_type) {
 987         case BPF_CGROUP_INET_INGRESS:
 988         case BPF_CGROUP_INET_EGRESS:
 989         case BPF_CGROUP_INET_SOCK_CREATE:
 990                 cgrp = cgroup_get_from_fd(attr->target_fd);
 991                 if (IS_ERR(cgrp))
 992                         return PTR_ERR(cgrp);
 993
 994                 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false);
 995                 cgroup_put(cgrp);
 996                 break;
 997
 998         default:
 999                 return -EINVAL;
1000         }
1001
1002         return ret;
1003 }
1004 #endif /* CONFIG_CGROUP_BPF */
1005
1006 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
1007 {
1008         union bpf_attr attr = {};
1009         int err;
1010
1011         if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
1012                 return -EPERM;
1013
1014         if (!access_ok(VERIFY_READ, uattr, 1))
1015                 return -EFAULT;
1016
1017         if (size > PAGE_SIZE)   /* silly large */
1018                 return -E2BIG;
1019
1020         if (secure_modules())
1021                 return -EPERM;
1022
1023         /* If we're handed a bigger struct than we know of,
1024          * ensure all the unknown bits are 0 - i.e. new
1025          * user-space does not rely on any kernel feature
1026          * extensions we dont know about yet.
1027          */
1028         if (size > sizeof(attr)) {
1029                 unsigned char __user *addr;
1030                 unsigned char __user *end;
1031                 unsigned char val;
1032
1033                 addr = (void __user *)uattr + sizeof(attr);
1034                 end  = (void __user *)uattr + size;
1035
1036                 for (; addr < end; addr++) {
1037                         err = get_user(val, addr);
1038                         if (err)
1039                                 return err;
1040                         if (val)
1041                                 return -E2BIG;
1042                 }
1043                 size = sizeof(attr);
1044         }
1045
1046         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
1047         if (copy_from_user(&attr, uattr, size) != 0)
1048                 return -EFAULT;
1049
1050         switch (cmd) {
1051         case BPF_MAP_CREATE:
1052                 err = map_create(&attr);
1053                 break;
1054         case BPF_MAP_LOOKUP_ELEM:
1055                 err = map_lookup_elem(&attr);
1056                 break;
1057         case BPF_MAP_UPDATE_ELEM:
1058                 err = map_update_elem(&attr);
1059                 break;
1060         case BPF_MAP_DELETE_ELEM:
1061                 err = map_delete_elem(&attr);
1062                 break;
1063         case BPF_MAP_GET_NEXT_KEY:
1064                 err = map_get_next_key(&attr);
1065                 break;
1066         case BPF_PROG_LOAD:
1067                 err = bpf_prog_load(&attr);
1068                 break;
1069         case BPF_OBJ_PIN:
1070                 err = bpf_obj_pin(&attr);
1071                 break;
1072         case BPF_OBJ_GET:
1073                 err = bpf_obj_get(&attr);
1074                 break;
1075
1076 #ifdef CONFIG_CGROUP_BPF
1077         case BPF_PROG_ATTACH:
1078                 err = bpf_prog_attach(&attr);
1079                 break;
1080         case BPF_PROG_DETACH:
1081                 err = bpf_prog_detach(&attr);
1082                 break;
1083 #endif
1084
1085         default:
1086                 err = -EINVAL;
1087                 break;
1088         }
1089
1090         return err;
1091 }