kernel/bpf/syscall.c

   1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful, but
   8  * WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  10  * General Public License for more details.
  11  */
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_trace.h>
  14 #include <linux/syscalls.h>
  15 #include <linux/slab.h>
  16 #include <linux/sched/signal.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/mmzone.h>
  19 #include <linux/anon_inodes.h>
  20 #include <linux/file.h>
  21 #include <linux/license.h>
  22 #include <linux/filter.h>
  23 #include <linux/version.h>
  24 #include <linux/kernel.h>
  25 #include <linux/idr.h>
  26
  27 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
  28                            (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  29                            (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  30                            (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  31 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  32 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
  33
  34 DEFINE_PER_CPU(int, bpf_prog_active);
  35 static DEFINE_IDR(prog_idr);
  36 static DEFINE_SPINLOCK(prog_idr_lock);
  37 static DEFINE_IDR(map_idr);
  38 static DEFINE_SPINLOCK(map_idr_lock);
  39
  40 int sysctl_unprivileged_bpf_disabled __read_mostly;
  41
  42 static const struct bpf_map_ops * const bpf_map_types[] = {
  43 #define BPF_PROG_TYPE(_id, _ops)
  44 #define BPF_MAP_TYPE(_id, _ops) \
  45         [_id] = &_ops,
  46 #include <linux/bpf_types.h>
  47 #undef BPF_PROG_TYPE
  48 #undef BPF_MAP_TYPE
  49 };
  50
  51 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
  52 {
  53         struct bpf_map *map;
  54
  55         if (attr->map_type >= ARRAY_SIZE(bpf_map_types) ||
  56             !bpf_map_types[attr->map_type])
  57                 return ERR_PTR(-EINVAL);
  58
  59         map = bpf_map_types[attr->map_type]->map_alloc(attr);
  60         if (IS_ERR(map))
  61                 return map;
  62         map->ops = bpf_map_types[attr->map_type];
  63         map->map_type = attr->map_type;
  64         return map;
  65 }
  66
  67 void *bpf_map_area_alloc(size_t size)
  68 {
  69         /* We definitely need __GFP_NORETRY, so OOM killer doesn't
  70          * trigger under memory pressure as we really just want to
  71          * fail instead.
  72          */
  73         const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO;
  74         void *area;
  75
  76         if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
  77                 area = kmalloc(size, GFP_USER | flags);
  78                 if (area != NULL)
  79                         return area;
  80         }
  81
  82         return __vmalloc(size, GFP_KERNEL | flags, PAGE_KERNEL);
  83 }
  84
  85 void bpf_map_area_free(void *area)
  86 {
  87         kvfree(area);
  88 }
  89
  90 int bpf_map_precharge_memlock(u32 pages)
  91 {
  92         struct user_struct *user = get_current_user();
  93         unsigned long memlock_limit, cur;
  94
  95         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  96         cur = atomic_long_read(&user->locked_vm);
  97         free_uid(user);
  98         if (cur + pages > memlock_limit)
  99                 return -EPERM;
 100         return 0;
 101 }
 102
 103 static int bpf_map_charge_memlock(struct bpf_map *map)
 104 {
 105         struct user_struct *user = get_current_user();
 106         unsigned long memlock_limit;
 107
 108         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 109
 110         atomic_long_add(map->pages, &user->locked_vm);
 111
 112         if (atomic_long_read(&user->locked_vm) > memlock_limit) {
 113                 atomic_long_sub(map->pages, &user->locked_vm);
 114                 free_uid(user);
 115                 return -EPERM;
 116         }
 117         map->user = user;
 118         return 0;
 119 }
 120
 121 static void bpf_map_uncharge_memlock(struct bpf_map *map)
 122 {
 123         struct user_struct *user = map->user;
 124
 125         atomic_long_sub(map->pages, &user->locked_vm);
 126         free_uid(user);
 127 }
 128
 129 static int bpf_map_alloc_id(struct bpf_map *map)
 130 {
 131         int id;
 132
 133         spin_lock_bh(&map_idr_lock);
 134         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 135         if (id > 0)
 136                 map->id = id;
 137         spin_unlock_bh(&map_idr_lock);
 138
 139         if (WARN_ON_ONCE(!id))
 140                 return -ENOSPC;
 141
 142         return id > 0 ? 0 : id;
 143 }
 144
 145 static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 146 {
 147         if (do_idr_lock)
 148                 spin_lock_bh(&map_idr_lock);
 149         else
 150                 __acquire(&map_idr_lock);
 151
 152         idr_remove(&map_idr, map->id);
 153
 154         if (do_idr_lock)
 155                 spin_unlock_bh(&map_idr_lock);
 156         else
 157                 __release(&map_idr_lock);
 158 }
 159
 160 /* called from workqueue */
 161 static void bpf_map_free_deferred(struct work_struct *work)
 162 {
 163         struct bpf_map *map = container_of(work, struct bpf_map, work);
 164
 165         bpf_map_uncharge_memlock(map);
 166         /* implementation dependent freeing */
 167         map->ops->map_free(map);
 168 }
 169
 170 static void bpf_map_put_uref(struct bpf_map *map)
 171 {
 172         if (atomic_dec_and_test(&map->usercnt)) {
 173                 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
 174                         bpf_fd_array_map_clear(map);
 175         }
 176 }
 177
 178 /* decrement map refcnt and schedule it for freeing via workqueue
 179  * (unrelying map implementation ops->map_free() might sleep)
 180  */
 181 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 182 {
 183         if (atomic_dec_and_test(&map->refcnt)) {
 184                 /* bpf_map_free_id() must be called first */
 185                 bpf_map_free_id(map, do_idr_lock);
 186                 INIT_WORK(&map->work, bpf_map_free_deferred);
 187                 schedule_work(&map->work);
 188         }
 189 }
 190
 191 void bpf_map_put(struct bpf_map *map)
 192 {
 193         __bpf_map_put(map, true);
 194 }
 195
 196 void bpf_map_put_with_uref(struct bpf_map *map)
 197 {
 198         bpf_map_put_uref(map);
 199         bpf_map_put(map);
 200 }
 201
 202 static int bpf_map_release(struct inode *inode, struct file *filp)
 203 {
 204         struct bpf_map *map = filp->private_data;
 205
 206         if (map->ops->map_release)
 207                 map->ops->map_release(map, filp);
 208
 209         bpf_map_put_with_uref(map);
 210         return 0;
 211 }
 212
 213 #ifdef CONFIG_PROC_FS
 214 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 215 {
 216         const struct bpf_map *map = filp->private_data;
 217         const struct bpf_array *array;
 218         u32 owner_prog_type = 0;
 219         u32 owner_jited = 0;
 220
 221         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 222                 array = container_of(map, struct bpf_array, map);
 223                 owner_prog_type = array->owner_prog_type;
 224                 owner_jited = array->owner_jited;
 225         }
 226
 227         seq_printf(m,
 228                    "map_type:\t%u\n"
 229                    "key_size:\t%u\n"
 230                    "value_size:\t%u\n"
 231                    "max_entries:\t%u\n"
 232                    "map_flags:\t%#x\n"
 233                    "memlock:\t%llu\n",
 234                    map->map_type,
 235                    map->key_size,
 236                    map->value_size,
 237                    map->max_entries,
 238                    map->map_flags,
 239                    map->pages * 1ULL << PAGE_SHIFT);
 240
 241         if (owner_prog_type) {
 242                 seq_printf(m, "owner_prog_type:\t%u\n",
 243                            owner_prog_type);
 244                 seq_printf(m, "owner_jited:\t%u\n",
 245                            owner_jited);
 246         }
 247 }
 248 #endif
 249
 250 static const struct file_operations bpf_map_fops = {
 251 #ifdef CONFIG_PROC_FS
 252         .show_fdinfo    = bpf_map_show_fdinfo,
 253 #endif
 254         .release        = bpf_map_release,
 255 };
 256
 257 int bpf_map_new_fd(struct bpf_map *map)
 258 {
 259         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 260                                 O_RDWR | O_CLOEXEC);
 261 }
 262
 263 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 264 #define CHECK_ATTR(CMD) \
 265         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 266                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 267                    sizeof(*attr) - \
 268                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 269                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 270
 271 #define BPF_MAP_CREATE_LAST_FIELD inner_map_fd
 272 /* called via syscall */
 273 static int map_create(union bpf_attr *attr)
 274 {
 275         struct bpf_map *map;
 276         int err;
 277
 278         err = CHECK_ATTR(BPF_MAP_CREATE);
 279         if (err)
 280                 return -EINVAL;
 281
 282         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 283         map = find_and_alloc_map(attr);
 284         if (IS_ERR(map))
 285                 return PTR_ERR(map);
 286
 287         atomic_set(&map->refcnt, 1);
 288         atomic_set(&map->usercnt, 1);
 289
 290         err = bpf_map_charge_memlock(map);
 291         if (err)
 292                 goto free_map_nouncharge;
 293
 294         err = bpf_map_alloc_id(map);
 295         if (err)
 296                 goto free_map;
 297
 298         err = bpf_map_new_fd(map);
 299         if (err < 0) {
 300                 /* failed to allocate fd.
 301                  * bpf_map_put() is needed because the above
 302                  * bpf_map_alloc_id() has published the map
 303                  * to the userspace and the userspace may
 304                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 305                  */
 306                 bpf_map_put(map);
 307                 return err;
 308         }
 309
 310         trace_bpf_map_create(map, err);
 311         return err;
 312
 313 free_map:
 314         bpf_map_uncharge_memlock(map);
 315 free_map_nouncharge:
 316         map->ops->map_free(map);
 317         return err;
 318 }
 319
 320 /* if error is returned, fd is released.
 321  * On success caller should complete fd access with matching fdput()
 322  */
 323 struct bpf_map *__bpf_map_get(struct fd f)
 324 {
 325         if (!f.file)
 326                 return ERR_PTR(-EBADF);
 327         if (f.file->f_op != &bpf_map_fops) {
 328                 fdput(f);
 329                 return ERR_PTR(-EINVAL);
 330         }
 331
 332         return f.file->private_data;
 333 }
 334
 335 /* prog's and map's refcnt limit */
 336 #define BPF_MAX_REFCNT 32768
 337
 338 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
 339 {
 340         if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
 341                 atomic_dec(&map->refcnt);
 342                 return ERR_PTR(-EBUSY);
 343         }
 344         if (uref)
 345                 atomic_inc(&map->usercnt);
 346         return map;
 347 }
 348
 349 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 350 {
 351         struct fd f = fdget(ufd);
 352         struct bpf_map *map;
 353
 354         map = __bpf_map_get(f);
 355         if (IS_ERR(map))
 356                 return map;
 357
 358         map = bpf_map_inc(map, true);
 359         fdput(f);
 360
 361         return map;
 362 }
 363
 364 /* map_idr_lock should have been held */
 365 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
 366                                             bool uref)
 367 {
 368         int refold;
 369
 370         refold = __atomic_add_unless(&map->refcnt, 1, 0);
 371
 372         if (refold >= BPF_MAX_REFCNT) {
 373                 __bpf_map_put(map, false);
 374                 return ERR_PTR(-EBUSY);
 375         }
 376
 377         if (!refold)
 378                 return ERR_PTR(-ENOENT);
 379
 380         if (uref)
 381                 atomic_inc(&map->usercnt);
 382
 383         return map;
 384 }
 385
 386 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 387 {
 388         return -ENOTSUPP;
 389 }
 390
 391 /* last field in 'union bpf_attr' used by this command */
 392 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
 393
 394 static int map_lookup_elem(union bpf_attr *attr)
 395 {
 396         void __user *ukey = u64_to_user_ptr(attr->key);
 397         void __user *uvalue = u64_to_user_ptr(attr->value);
 398         int ufd = attr->map_fd;
 399         struct bpf_map *map;
 400         void *key, *value, *ptr;
 401         u32 value_size;
 402         struct fd f;
 403         int err;
 404
 405         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
 406                 return -EINVAL;
 407
 408         f = fdget(ufd);
 409         map = __bpf_map_get(f);
 410         if (IS_ERR(map))
 411                 return PTR_ERR(map);
 412
 413         key = memdup_user(ukey, map->key_size);
 414         if (IS_ERR(key)) {
 415                 err = PTR_ERR(key);
 416                 goto err_put;
 417         }
 418
 419         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 420             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 421             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
 422                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 423         else if (IS_FD_MAP(map))
 424                 value_size = sizeof(u32);
 425         else
 426                 value_size = map->value_size;
 427
 428         err = -ENOMEM;
 429         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 430         if (!value)
 431                 goto free_key;
 432
 433         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 434             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 435                 err = bpf_percpu_hash_copy(map, key, value);
 436         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 437                 err = bpf_percpu_array_copy(map, key, value);
 438         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 439                 err = bpf_stackmap_copy(map, key, value);
 440         } else if (IS_FD_ARRAY(map)) {
 441                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 442         } else if (IS_FD_HASH(map)) {
 443                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 444         } else {
 445                 rcu_read_lock();
 446                 ptr = map->ops->map_lookup_elem(map, key);
 447                 if (ptr)
 448                         memcpy(value, ptr, value_size);
 449                 rcu_read_unlock();
 450                 err = ptr ? 0 : -ENOENT;
 451         }
 452
 453         if (err)
 454                 goto free_value;
 455
 456         err = -EFAULT;
 457         if (copy_to_user(uvalue, value, value_size) != 0)
 458                 goto free_value;
 459
 460         trace_bpf_map_lookup_elem(map, ufd, key, value);
 461         err = 0;
 462
 463 free_value:
 464         kfree(value);
 465 free_key:
 466         kfree(key);
 467 err_put:
 468         fdput(f);
 469         return err;
 470 }
 471
 472 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
 473
 474 static int map_update_elem(union bpf_attr *attr)
 475 {
 476         void __user *ukey = u64_to_user_ptr(attr->key);
 477         void __user *uvalue = u64_to_user_ptr(attr->value);
 478         int ufd = attr->map_fd;
 479         struct bpf_map *map;
 480         void *key, *value;
 481         u32 value_size;
 482         struct fd f;
 483         int err;
 484
 485         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
 486                 return -EINVAL;
 487
 488         f = fdget(ufd);
 489         map = __bpf_map_get(f);
 490         if (IS_ERR(map))
 491                 return PTR_ERR(map);
 492
 493         key = memdup_user(ukey, map->key_size);
 494         if (IS_ERR(key)) {
 495                 err = PTR_ERR(key);
 496                 goto err_put;
 497         }
 498
 499         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 500             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 501             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
 502                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 503         else
 504                 value_size = map->value_size;
 505
 506         err = -ENOMEM;
 507         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 508         if (!value)
 509                 goto free_key;
 510
 511         err = -EFAULT;
 512         if (copy_from_user(value, uvalue, value_size) != 0)
 513                 goto free_value;
 514
 515         /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
 516          * inside bpf map update or delete otherwise deadlocks are possible
 517          */
 518         preempt_disable();
 519         __this_cpu_inc(bpf_prog_active);
 520         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 521             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 522                 err = bpf_percpu_hash_update(map, key, value, attr->flags);
 523         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 524                 err = bpf_percpu_array_update(map, key, value, attr->flags);
 525         } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
 526                    map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
 527                    map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY ||
 528                    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) {
 529                 rcu_read_lock();
 530                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 531                                                    attr->flags);
 532                 rcu_read_unlock();
 533         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 534                 rcu_read_lock();
 535                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 536                                                   attr->flags);
 537                 rcu_read_unlock();
 538         } else {
 539                 rcu_read_lock();
 540                 err = map->ops->map_update_elem(map, key, value, attr->flags);
 541                 rcu_read_unlock();
 542         }
 543         __this_cpu_dec(bpf_prog_active);
 544         preempt_enable();
 545
 546         if (!err)
 547                 trace_bpf_map_update_elem(map, ufd, key, value);
 548 free_value:
 549         kfree(value);
 550 free_key:
 551         kfree(key);
 552 err_put:
 553         fdput(f);
 554         return err;
 555 }
 556
 557 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
 558
 559 static int map_delete_elem(union bpf_attr *attr)
 560 {
 561         void __user *ukey = u64_to_user_ptr(attr->key);
 562         int ufd = attr->map_fd;
 563         struct bpf_map *map;
 564         struct fd f;
 565         void *key;
 566         int err;
 567
 568         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
 569                 return -EINVAL;
 570
 571         f = fdget(ufd);
 572         map = __bpf_map_get(f);
 573         if (IS_ERR(map))
 574                 return PTR_ERR(map);
 575
 576         key = memdup_user(ukey, map->key_size);
 577         if (IS_ERR(key)) {
 578                 err = PTR_ERR(key);
 579                 goto err_put;
 580         }
 581
 582         preempt_disable();
 583         __this_cpu_inc(bpf_prog_active);
 584         rcu_read_lock();
 585         err = map->ops->map_delete_elem(map, key);
 586         rcu_read_unlock();
 587         __this_cpu_dec(bpf_prog_active);
 588         preempt_enable();
 589
 590         if (!err)
 591                 trace_bpf_map_delete_elem(map, ufd, key);
 592         kfree(key);
 593 err_put:
 594         fdput(f);
 595         return err;
 596 }
 597
 598 /* last field in 'union bpf_attr' used by this command */
 599 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
 600
 601 static int map_get_next_key(union bpf_attr *attr)
 602 {
 603         void __user *ukey = u64_to_user_ptr(attr->key);
 604         void __user *unext_key = u64_to_user_ptr(attr->next_key);
 605         int ufd = attr->map_fd;
 606         struct bpf_map *map;
 607         void *key, *next_key;
 608         struct fd f;
 609         int err;
 610
 611         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
 612                 return -EINVAL;
 613
 614         f = fdget(ufd);
 615         map = __bpf_map_get(f);
 616         if (IS_ERR(map))
 617                 return PTR_ERR(map);
 618
 619         if (ukey) {
 620                 key = memdup_user(ukey, map->key_size);
 621                 if (IS_ERR(key)) {
 622                         err = PTR_ERR(key);
 623                         goto err_put;
 624                 }
 625         } else {
 626                 key = NULL;
 627         }
 628
 629         err = -ENOMEM;
 630         next_key = kmalloc(map->key_size, GFP_USER);
 631         if (!next_key)
 632                 goto free_key;
 633
 634         rcu_read_lock();
 635         err = map->ops->map_get_next_key(map, key, next_key);
 636         rcu_read_unlock();
 637         if (err)
 638                 goto free_next_key;
 639
 640         err = -EFAULT;
 641         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
 642                 goto free_next_key;
 643
 644         trace_bpf_map_next_key(map, ufd, key, next_key);
 645         err = 0;
 646
 647 free_next_key:
 648         kfree(next_key);
 649 free_key:
 650         kfree(key);
 651 err_put:
 652         fdput(f);
 653         return err;
 654 }
 655
 656 static const struct bpf_verifier_ops * const bpf_prog_types[] = {
 657 #define BPF_PROG_TYPE(_id, _ops) \
 658         [_id] = &_ops,
 659 #define BPF_MAP_TYPE(_id, _ops)
 660 #include <linux/bpf_types.h>
 661 #undef BPF_PROG_TYPE
 662 #undef BPF_MAP_TYPE
 663 };
 664
 665 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
 666 {
 667         if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type])
 668                 return -EINVAL;
 669
 670         prog->aux->ops = bpf_prog_types[type];
 671         prog->type = type;
 672         return 0;
 673 }
 674
 675 /* drop refcnt on maps used by eBPF program and free auxilary data */
 676 static void free_used_maps(struct bpf_prog_aux *aux)
 677 {
 678         int i;
 679
 680         for (i = 0; i < aux->used_map_cnt; i++)
 681                 bpf_map_put(aux->used_maps[i]);
 682
 683         kfree(aux->used_maps);
 684 }
 685
 686 int __bpf_prog_charge(struct user_struct *user, u32 pages)
 687 {
 688         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 689         unsigned long user_bufs;
 690
 691         if (user) {
 692                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
 693                 if (user_bufs > memlock_limit) {
 694                         atomic_long_sub(pages, &user->locked_vm);
 695                         return -EPERM;
 696                 }
 697         }
 698
 699         return 0;
 700 }
 701
 702 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
 703 {
 704         if (user)
 705                 atomic_long_sub(pages, &user->locked_vm);
 706 }
 707
 708 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
 709 {
 710         struct user_struct *user = get_current_user();
 711         int ret;
 712
 713         ret = __bpf_prog_charge(user, prog->pages);
 714         if (ret) {
 715                 free_uid(user);
 716                 return ret;
 717         }
 718
 719         prog->aux->user = user;
 720         return 0;
 721 }
 722
 723 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
 724 {
 725         struct user_struct *user = prog->aux->user;
 726
 727         __bpf_prog_uncharge(user, prog->pages);
 728         free_uid(user);
 729 }
 730
 731 static int bpf_prog_alloc_id(struct bpf_prog *prog)
 732 {
 733         int id;
 734
 735         spin_lock_bh(&prog_idr_lock);
 736         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
 737         if (id > 0)
 738                 prog->aux->id = id;
 739         spin_unlock_bh(&prog_idr_lock);
 740
 741         /* id is in [1, INT_MAX) */
 742         if (WARN_ON_ONCE(!id))
 743                 return -ENOSPC;
 744
 745         return id > 0 ? 0 : id;
 746 }
 747
 748 static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
 749 {
 750         /* cBPF to eBPF migrations are currently not in the idr store. */
 751         if (!prog->aux->id)
 752                 return;
 753
 754         if (do_idr_lock)
 755                 spin_lock_bh(&prog_idr_lock);
 756         else
 757                 __acquire(&prog_idr_lock);
 758
 759         idr_remove(&prog_idr, prog->aux->id);
 760
 761         if (do_idr_lock)
 762                 spin_unlock_bh(&prog_idr_lock);
 763         else
 764                 __release(&prog_idr_lock);
 765 }
 766
 767 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
 768 {
 769         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
 770
 771         free_used_maps(aux);
 772         bpf_prog_uncharge_memlock(aux->prog);
 773         bpf_prog_free(aux->prog);
 774 }
 775
 776 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
 777 {
 778         if (atomic_dec_and_test(&prog->aux->refcnt)) {
 779                 trace_bpf_prog_put_rcu(prog);
 780                 /* bpf_prog_free_id() must be called first */
 781                 bpf_prog_free_id(prog, do_idr_lock);
 782                 bpf_prog_kallsyms_del(prog);
 783                 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
 784         }
 785 }
 786
 787 void bpf_prog_put(struct bpf_prog *prog)
 788 {
 789         __bpf_prog_put(prog, true);
 790 }
 791 EXPORT_SYMBOL_GPL(bpf_prog_put);
 792
 793 static int bpf_prog_release(struct inode *inode, struct file *filp)
 794 {
 795         struct bpf_prog *prog = filp->private_data;
 796
 797         bpf_prog_put(prog);
 798         return 0;
 799 }
 800
 801 #ifdef CONFIG_PROC_FS
 802 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
 803 {
 804         const struct bpf_prog *prog = filp->private_data;
 805         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
 806
 807         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
 808         seq_printf(m,
 809                    "prog_type:\t%u\n"
 810                    "prog_jited:\t%u\n"
 811                    "prog_tag:\t%s\n"
 812                    "memlock:\t%llu\n",
 813                    prog->type,
 814                    prog->jited,
 815                    prog_tag,
 816                    prog->pages * 1ULL << PAGE_SHIFT);
 817 }
 818 #endif
 819
 820 static const struct file_operations bpf_prog_fops = {
 821 #ifdef CONFIG_PROC_FS
 822         .show_fdinfo    = bpf_prog_show_fdinfo,
 823 #endif
 824         .release        = bpf_prog_release,
 825 };
 826
 827 int bpf_prog_new_fd(struct bpf_prog *prog)
 828 {
 829         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
 830                                 O_RDWR | O_CLOEXEC);
 831 }
 832
 833 static struct bpf_prog *____bpf_prog_get(struct fd f)
 834 {
 835         if (!f.file)
 836                 return ERR_PTR(-EBADF);
 837         if (f.file->f_op != &bpf_prog_fops) {
 838                 fdput(f);
 839                 return ERR_PTR(-EINVAL);
 840         }
 841
 842         return f.file->private_data;
 843 }
 844
 845 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
 846 {
 847         if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
 848                 atomic_sub(i, &prog->aux->refcnt);
 849                 return ERR_PTR(-EBUSY);
 850         }
 851         return prog;
 852 }
 853 EXPORT_SYMBOL_GPL(bpf_prog_add);
 854
 855 void bpf_prog_sub(struct bpf_prog *prog, int i)
 856 {
 857         /* Only to be used for undoing previous bpf_prog_add() in some
 858          * error path. We still know that another entity in our call
 859          * path holds a reference to the program, thus atomic_sub() can
 860          * be safely used in such cases!
 861          */
 862         WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
 863 }
 864 EXPORT_SYMBOL_GPL(bpf_prog_sub);
 865
 866 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
 867 {
 868         return bpf_prog_add(prog, 1);
 869 }
 870 EXPORT_SYMBOL_GPL(bpf_prog_inc);
 871
 872 /* prog_idr_lock should have been held */
 873 static struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
 874 {
 875         int refold;
 876
 877         refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0);
 878
 879         if (refold >= BPF_MAX_REFCNT) {
 880                 __bpf_prog_put(prog, false);
 881                 return ERR_PTR(-EBUSY);
 882         }
 883
 884         if (!refold)
 885                 return ERR_PTR(-ENOENT);
 886
 887         return prog;
 888 }
 889
 890 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
 891 {
 892         struct fd f = fdget(ufd);
 893         struct bpf_prog *prog;
 894
 895         prog = ____bpf_prog_get(f);
 896         if (IS_ERR(prog))
 897                 return prog;
 898         if (type && prog->type != *type) {
 899                 prog = ERR_PTR(-EINVAL);
 900                 goto out;
 901         }
 902
 903         prog = bpf_prog_inc(prog);
 904 out:
 905         fdput(f);
 906         return prog;
 907 }
 908
 909 struct bpf_prog *bpf_prog_get(u32 ufd)
 910 {
 911         return __bpf_prog_get(ufd, NULL);
 912 }
 913
 914 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
 915 {
 916         struct bpf_prog *prog = __bpf_prog_get(ufd, &type);
 917
 918         if (!IS_ERR(prog))
 919                 trace_bpf_prog_get_type(prog);
 920         return prog;
 921 }
 922 EXPORT_SYMBOL_GPL(bpf_prog_get_type);
 923
 924 /* last field in 'union bpf_attr' used by this command */
 925 #define BPF_PROG_LOAD_LAST_FIELD prog_flags
 926
 927 static int bpf_prog_load(union bpf_attr *attr)
 928 {
 929         enum bpf_prog_type type = attr->prog_type;
 930         struct bpf_prog *prog;
 931         int err;
 932         char license[128];
 933         bool is_gpl;
 934
 935         if (CHECK_ATTR(BPF_PROG_LOAD))
 936                 return -EINVAL;
 937
 938         if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT)
 939                 return -EINVAL;
 940
 941         /* copy eBPF program license from user space */
 942         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
 943                               sizeof(license) - 1) < 0)
 944                 return -EFAULT;
 945         license[sizeof(license) - 1] = 0;
 946
 947         /* eBPF programs must be GPL compatible to use GPL-ed functions */
 948         is_gpl = license_is_gpl_compatible(license);
 949
 950         if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS)
 951                 return -E2BIG;
 952
 953         if (type == BPF_PROG_TYPE_KPROBE &&
 954             attr->kern_version != LINUX_VERSION_CODE)
 955                 return -EINVAL;
 956
 957         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
 958             type != BPF_PROG_TYPE_CGROUP_SKB &&
 959             !capable(CAP_SYS_ADMIN))
 960                 return -EPERM;
 961
 962         /* plain bpf_prog allocation */
 963         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
 964         if (!prog)
 965                 return -ENOMEM;
 966
 967         err = bpf_prog_charge_memlock(prog);
 968         if (err)
 969                 goto free_prog_nouncharge;
 970
 971         prog->len = attr->insn_cnt;
 972
 973         err = -EFAULT;
 974         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
 975                            bpf_prog_insn_size(prog)) != 0)
 976                 goto free_prog;
 977
 978         prog->orig_prog = NULL;
 979         prog->jited = 0;
 980
 981         atomic_set(&prog->aux->refcnt, 1);
 982         prog->gpl_compatible = is_gpl ? 1 : 0;
 983
 984         /* find program type: socket_filter vs tracing_filter */
 985         err = find_prog_type(type, prog);
 986         if (err < 0)
 987                 goto free_prog;
 988
 989         /* run eBPF verifier */
 990         err = bpf_check(&prog, attr);
 991         if (err < 0)
 992                 goto free_used_maps;
 993
 994         /* eBPF program is ready to be JITed */
 995         prog = bpf_prog_select_runtime(prog, &err);
 996         if (err < 0)
 997                 goto free_used_maps;
 998
 999         err = bpf_prog_alloc_id(prog);
1000         if (err)
1001                 goto free_used_maps;
1002
1003         err = bpf_prog_new_fd(prog);
1004         if (err < 0) {
1005                 /* failed to allocate fd.
1006                  * bpf_prog_put() is needed because the above
1007                  * bpf_prog_alloc_id() has published the prog
1008                  * to the userspace and the userspace may
1009                  * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID.
1010                  */
1011                 bpf_prog_put(prog);
1012                 return err;
1013         }
1014
1015         bpf_prog_kallsyms_add(prog);
1016         trace_bpf_prog_load(prog, err);
1017         return err;
1018
1019 free_used_maps:
1020         free_used_maps(prog->aux);
1021 free_prog:
1022         bpf_prog_uncharge_memlock(prog);
1023 free_prog_nouncharge:
1024         bpf_prog_free(prog);
1025         return err;
1026 }
1027
1028 #define BPF_OBJ_LAST_FIELD bpf_fd
1029
1030 static int bpf_obj_pin(const union bpf_attr *attr)
1031 {
1032         if (CHECK_ATTR(BPF_OBJ))
1033                 return -EINVAL;
1034
1035         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
1036 }
1037
1038 static int bpf_obj_get(const union bpf_attr *attr)
1039 {
1040         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
1041                 return -EINVAL;
1042
1043         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
1044 }
1045
1046 #ifdef CONFIG_CGROUP_BPF
1047
1048 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
1049
1050 static int bpf_prog_attach(const union bpf_attr *attr)
1051 {
1052         enum bpf_prog_type ptype;
1053         struct bpf_prog *prog;
1054         struct cgroup *cgrp;
1055         int ret;
1056
1057         if (!capable(CAP_NET_ADMIN))
1058                 return -EPERM;
1059
1060         if (CHECK_ATTR(BPF_PROG_ATTACH))
1061                 return -EINVAL;
1062
1063         if (attr->attach_flags & ~BPF_F_ALLOW_OVERRIDE)
1064                 return -EINVAL;
1065
1066         switch (attr->attach_type) {
1067         case BPF_CGROUP_INET_INGRESS:
1068         case BPF_CGROUP_INET_EGRESS:
1069                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1070                 break;
1071         case BPF_CGROUP_INET_SOCK_CREATE:
1072                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1073                 break;
1074         case BPF_CGROUP_SOCK_OPS:
1075                 ptype = BPF_PROG_TYPE_SOCK_OPS;
1076                 break;
1077         default:
1078                 return -EINVAL;
1079         }
1080
1081         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1082         if (IS_ERR(prog))
1083                 return PTR_ERR(prog);
1084
1085         cgrp = cgroup_get_from_fd(attr->target_fd);
1086         if (IS_ERR(cgrp)) {
1087                 bpf_prog_put(prog);
1088                 return PTR_ERR(cgrp);
1089         }
1090
1091         ret = cgroup_bpf_update(cgrp, prog, attr->attach_type,
1092                                 attr->attach_flags & BPF_F_ALLOW_OVERRIDE);
1093         if (ret)
1094                 bpf_prog_put(prog);
1095         cgroup_put(cgrp);
1096
1097         return ret;
1098 }
1099
1100 #define BPF_PROG_DETACH_LAST_FIELD attach_type
1101
1102 static int bpf_prog_detach(const union bpf_attr *attr)
1103 {
1104         struct cgroup *cgrp;
1105         int ret;
1106
1107         if (!capable(CAP_NET_ADMIN))
1108                 return -EPERM;
1109
1110         if (CHECK_ATTR(BPF_PROG_DETACH))
1111                 return -EINVAL;
1112
1113         switch (attr->attach_type) {
1114         case BPF_CGROUP_INET_INGRESS:
1115         case BPF_CGROUP_INET_EGRESS:
1116         case BPF_CGROUP_INET_SOCK_CREATE:
1117         case BPF_CGROUP_SOCK_OPS:
1118                 cgrp = cgroup_get_from_fd(attr->target_fd);
1119                 if (IS_ERR(cgrp))
1120                         return PTR_ERR(cgrp);
1121
1122                 ret = cgroup_bpf_update(cgrp, NULL, attr->attach_type, false);
1123                 cgroup_put(cgrp);
1124                 break;
1125
1126         default:
1127                 return -EINVAL;
1128         }
1129
1130         return ret;
1131 }
1132
1133 #endif /* CONFIG_CGROUP_BPF */
1134
1135 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
1136
1137 static int bpf_prog_test_run(const union bpf_attr *attr,
1138                              union bpf_attr __user *uattr)
1139 {
1140         struct bpf_prog *prog;
1141         int ret = -ENOTSUPP;
1142
1143         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
1144                 return -EINVAL;
1145
1146         prog = bpf_prog_get(attr->test.prog_fd);
1147         if (IS_ERR(prog))
1148                 return PTR_ERR(prog);
1149
1150         if (prog->aux->ops->test_run)
1151                 ret = prog->aux->ops->test_run(prog, attr, uattr);
1152
1153         bpf_prog_put(prog);
1154         return ret;
1155 }
1156
1157 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
1158
1159 static int bpf_obj_get_next_id(const union bpf_attr *attr,
1160                                union bpf_attr __user *uattr,
1161                                struct idr *idr,
1162                                spinlock_t *lock)
1163 {
1164         u32 next_id = attr->start_id;
1165         int err = 0;
1166
1167         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
1168                 return -EINVAL;
1169
1170         if (!capable(CAP_SYS_ADMIN))
1171                 return -EPERM;
1172
1173         next_id++;
1174         spin_lock_bh(lock);
1175         if (!idr_get_next(idr, &next_id))
1176                 err = -ENOENT;
1177         spin_unlock_bh(lock);
1178
1179         if (!err)
1180                 err = put_user(next_id, &uattr->next_id);
1181
1182         return err;
1183 }
1184
1185 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
1186
1187 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
1188 {
1189         struct bpf_prog *prog;
1190         u32 id = attr->prog_id;
1191         int fd;
1192
1193         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
1194                 return -EINVAL;
1195
1196         if (!capable(CAP_SYS_ADMIN))
1197                 return -EPERM;
1198
1199         spin_lock_bh(&prog_idr_lock);
1200         prog = idr_find(&prog_idr, id);
1201         if (prog)
1202                 prog = bpf_prog_inc_not_zero(prog);
1203         else
1204                 prog = ERR_PTR(-ENOENT);
1205         spin_unlock_bh(&prog_idr_lock);
1206
1207         if (IS_ERR(prog))
1208                 return PTR_ERR(prog);
1209
1210         fd = bpf_prog_new_fd(prog);
1211         if (fd < 0)
1212                 bpf_prog_put(prog);
1213
1214         return fd;
1215 }
1216
1217 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id
1218
1219 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
1220 {
1221         struct bpf_map *map;
1222         u32 id = attr->map_id;
1223         int fd;
1224
1225         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID))
1226                 return -EINVAL;
1227
1228         if (!capable(CAP_SYS_ADMIN))
1229                 return -EPERM;
1230
1231         spin_lock_bh(&map_idr_lock);
1232         map = idr_find(&map_idr, id);
1233         if (map)
1234                 map = bpf_map_inc_not_zero(map, true);
1235         else
1236                 map = ERR_PTR(-ENOENT);
1237         spin_unlock_bh(&map_idr_lock);
1238
1239         if (IS_ERR(map))
1240                 return PTR_ERR(map);
1241
1242         fd = bpf_map_new_fd(map);
1243         if (fd < 0)
1244                 bpf_map_put(map);
1245
1246         return fd;
1247 }
1248
1249 static int check_uarg_tail_zero(void __user *uaddr,
1250                                 size_t expected_size,
1251                                 size_t actual_size)
1252 {
1253         unsigned char __user *addr;
1254         unsigned char __user *end;
1255         unsigned char val;
1256         int err;
1257
1258         if (actual_size <= expected_size)
1259                 return 0;
1260
1261         addr = uaddr + expected_size;
1262         end  = uaddr + actual_size;
1263
1264         for (; addr < end; addr++) {
1265                 err = get_user(val, addr);
1266                 if (err)
1267                         return err;
1268                 if (val)
1269                         return -E2BIG;
1270         }
1271
1272         return 0;
1273 }
1274
1275 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
1276                                    const union bpf_attr *attr,
1277                                    union bpf_attr __user *uattr)
1278 {
1279         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
1280         struct bpf_prog_info info = {};
1281         u32 info_len = attr->info.info_len;
1282         char __user *uinsns;
1283         u32 ulen;
1284         int err;
1285
1286         err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
1287         if (err)
1288                 return err;
1289         info_len = min_t(u32, sizeof(info), info_len);
1290
1291         if (copy_from_user(&info, uinfo, info_len))
1292                 return -EFAULT;
1293
1294         info.type = prog->type;
1295         info.id = prog->aux->id;
1296
1297         memcpy(info.tag, prog->tag, sizeof(prog->tag));
1298
1299         if (!capable(CAP_SYS_ADMIN)) {
1300                 info.jited_prog_len = 0;
1301                 info.xlated_prog_len = 0;
1302                 goto done;
1303         }
1304
1305         ulen = info.jited_prog_len;
1306         info.jited_prog_len = prog->jited_len;
1307         if (info.jited_prog_len && ulen) {
1308                 uinsns = u64_to_user_ptr(info.jited_prog_insns);
1309                 ulen = min_t(u32, info.jited_prog_len, ulen);
1310                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
1311                         return -EFAULT;
1312         }
1313
1314         ulen = info.xlated_prog_len;
1315         info.xlated_prog_len = bpf_prog_insn_size(prog);
1316         if (info.xlated_prog_len && ulen) {
1317                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
1318                 ulen = min_t(u32, info.xlated_prog_len, ulen);
1319                 if (copy_to_user(uinsns, prog->insnsi, ulen))
1320                         return -EFAULT;
1321         }
1322
1323 done:
1324         if (copy_to_user(uinfo, &info, info_len) ||
1325             put_user(info_len, &uattr->info.info_len))
1326                 return -EFAULT;
1327
1328         return 0;
1329 }
1330
1331 static int bpf_map_get_info_by_fd(struct bpf_map *map,
1332                                   const union bpf_attr *attr,
1333                                   union bpf_attr __user *uattr)
1334 {
1335         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
1336         struct bpf_map_info info = {};
1337         u32 info_len = attr->info.info_len;
1338         int err;
1339
1340         err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
1341         if (err)
1342                 return err;
1343         info_len = min_t(u32, sizeof(info), info_len);
1344
1345         info.type = map->map_type;
1346         info.id = map->id;
1347         info.key_size = map->key_size;
1348         info.value_size = map->value_size;
1349         info.max_entries = map->max_entries;
1350         info.map_flags = map->map_flags;
1351
1352         if (copy_to_user(uinfo, &info, info_len) ||
1353             put_user(info_len, &uattr->info.info_len))
1354                 return -EFAULT;
1355
1356         return 0;
1357 }
1358
1359 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
1360
1361 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
1362                                   union bpf_attr __user *uattr)
1363 {
1364         int ufd = attr->info.bpf_fd;
1365         struct fd f;
1366         int err;
1367
1368         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
1369                 return -EINVAL;
1370
1371         f = fdget(ufd);
1372         if (!f.file)
1373                 return -EBADFD;
1374
1375         if (f.file->f_op == &bpf_prog_fops)
1376                 err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
1377                                               uattr);
1378         else if (f.file->f_op == &bpf_map_fops)
1379                 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
1380                                              uattr);
1381         else
1382                 err = -EINVAL;
1383
1384         fdput(f);
1385         return err;
1386 }
1387
1388 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
1389 {
1390         union bpf_attr attr = {};
1391         int err;
1392
1393         if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
1394                 return -EPERM;
1395
1396         if (!access_ok(VERIFY_READ, uattr, 1))
1397                 return -EFAULT;
1398
1399         if (size > PAGE_SIZE)   /* silly large */
1400                 return -E2BIG;
1401
1402         /* If we're handed a bigger struct than we know of,
1403          * ensure all the unknown bits are 0 - i.e. new
1404          * user-space does not rely on any kernel feature
1405          * extensions we dont know about yet.
1406          */
1407         err = check_uarg_tail_zero(uattr, sizeof(attr), size);
1408         if (err)
1409                 return err;
1410         size = min_t(u32, size, sizeof(attr));
1411
1412         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
1413         if (copy_from_user(&attr, uattr, size) != 0)
1414                 return -EFAULT;
1415
1416         switch (cmd) {
1417         case BPF_MAP_CREATE:
1418                 err = map_create(&attr);
1419                 break;
1420         case BPF_MAP_LOOKUP_ELEM:
1421                 err = map_lookup_elem(&attr);
1422                 break;
1423         case BPF_MAP_UPDATE_ELEM:
1424                 err = map_update_elem(&attr);
1425                 break;
1426         case BPF_MAP_DELETE_ELEM:
1427                 err = map_delete_elem(&attr);
1428                 break;
1429         case BPF_MAP_GET_NEXT_KEY:
1430                 err = map_get_next_key(&attr);
1431                 break;
1432         case BPF_PROG_LOAD:
1433                 err = bpf_prog_load(&attr);
1434                 break;
1435         case BPF_OBJ_PIN:
1436                 err = bpf_obj_pin(&attr);
1437                 break;
1438         case BPF_OBJ_GET:
1439                 err = bpf_obj_get(&attr);
1440                 break;
1441 #ifdef CONFIG_CGROUP_BPF
1442         case BPF_PROG_ATTACH:
1443                 err = bpf_prog_attach(&attr);
1444                 break;
1445         case BPF_PROG_DETACH:
1446                 err = bpf_prog_detach(&attr);
1447                 break;
1448 #endif
1449         case BPF_PROG_TEST_RUN:
1450                 err = bpf_prog_test_run(&attr, uattr);
1451                 break;
1452         case BPF_PROG_GET_NEXT_ID:
1453                 err = bpf_obj_get_next_id(&attr, uattr,
1454                                           &prog_idr, &prog_idr_lock);
1455                 break;
1456         case BPF_MAP_GET_NEXT_ID:
1457                 err = bpf_obj_get_next_id(&attr, uattr,
1458                                           &map_idr, &map_idr_lock);
1459                 break;
1460         case BPF_PROG_GET_FD_BY_ID:
1461                 err = bpf_prog_get_fd_by_id(&attr);
1462                 break;
1463         case BPF_MAP_GET_FD_BY_ID:
1464                 err = bpf_map_get_fd_by_id(&attr);
1465                 break;
1466         case BPF_OBJ_GET_INFO_BY_FD:
1467                 err = bpf_obj_get_info_by_fd(&attr, uattr);
1468                 break;
1469         default:
1470                 err = -EINVAL;
1471                 break;
1472         }
1473
1474         return err;
1475 }