kernel/bpf/syscall.c

   1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful, but
   8  * WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  10  * General Public License for more details.
  11  */
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_trace.h>
  14 #include <linux/syscalls.h>
  15 #include <linux/slab.h>
  16 #include <linux/sched/signal.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/mmzone.h>
  19 #include <linux/anon_inodes.h>
  20 #include <linux/file.h>
  21 #include <linux/license.h>
  22 #include <linux/filter.h>
  23 #include <linux/version.h>
  24 #include <linux/kernel.h>
  25 #include <linux/idr.h>
  26 #include <linux/cred.h>
  27 #include <linux/timekeeping.h>
  28 #include <linux/ctype.h>
  29
  30 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
  31                            (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  32                            (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  33                            (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  34 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  35 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
  36
  37 DEFINE_PER_CPU(int, bpf_prog_active);
  38 static DEFINE_IDR(prog_idr);
  39 static DEFINE_SPINLOCK(prog_idr_lock);
  40 static DEFINE_IDR(map_idr);
  41 static DEFINE_SPINLOCK(map_idr_lock);
  42
  43 int sysctl_unprivileged_bpf_disabled __read_mostly;
  44
  45 static const struct bpf_map_ops * const bpf_map_types[] = {
  46 #define BPF_PROG_TYPE(_id, _ops)
  47 #define BPF_MAP_TYPE(_id, _ops) \
  48         [_id] = &_ops,
  49 #include <linux/bpf_types.h>
  50 #undef BPF_PROG_TYPE
  51 #undef BPF_MAP_TYPE
  52 };
  53
  54 /*
  55  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  56  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  57  * we don't know about yet.
  58  *
  59  * There is a ToCToU between this function call and the following
  60  * copy_from_user() call. However, this is not a concern since this function is
  61  * meant to be a future-proofing of bits.
  62  */
  63 static int check_uarg_tail_zero(void __user *uaddr,
  64                                 size_t expected_size,
  65                                 size_t actual_size)
  66 {
  67         unsigned char __user *addr;
  68         unsigned char __user *end;
  69         unsigned char val;
  70         int err;
  71
  72         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  73                 return -E2BIG;
  74
  75         if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size)))
  76                 return -EFAULT;
  77
  78         if (actual_size <= expected_size)
  79                 return 0;
  80
  81         addr = uaddr + expected_size;
  82         end  = uaddr + actual_size;
  83
  84         for (; addr < end; addr++) {
  85                 err = get_user(val, addr);
  86                 if (err)
  87                         return err;
  88                 if (val)
  89                         return -E2BIG;
  90         }
  91
  92         return 0;
  93 }
  94
  95 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
  96 {
  97         struct bpf_map *map;
  98
  99         if (attr->map_type >= ARRAY_SIZE(bpf_map_types) ||
 100             !bpf_map_types[attr->map_type])
 101                 return ERR_PTR(-EINVAL);
 102
 103         map = bpf_map_types[attr->map_type]->map_alloc(attr);
 104         if (IS_ERR(map))
 105                 return map;
 106         map->ops = bpf_map_types[attr->map_type];
 107         map->map_type = attr->map_type;
 108         return map;
 109 }
 110
 111 void *bpf_map_area_alloc(size_t size, int numa_node)
 112 {
 113         /* We definitely need __GFP_NORETRY, so OOM killer doesn't
 114          * trigger under memory pressure as we really just want to
 115          * fail instead.
 116          */
 117         const gfp_t flags = __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO;
 118         void *area;
 119
 120         if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 121                 area = kmalloc_node(size, GFP_USER | flags, numa_node);
 122                 if (area != NULL)
 123                         return area;
 124         }
 125
 126         return __vmalloc_node_flags_caller(size, numa_node, GFP_KERNEL | flags,
 127                                            __builtin_return_address(0));
 128 }
 129
 130 void bpf_map_area_free(void *area)
 131 {
 132         kvfree(area);
 133 }
 134
 135 int bpf_map_precharge_memlock(u32 pages)
 136 {
 137         struct user_struct *user = get_current_user();
 138         unsigned long memlock_limit, cur;
 139
 140         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 141         cur = atomic_long_read(&user->locked_vm);
 142         free_uid(user);
 143         if (cur + pages > memlock_limit)
 144                 return -EPERM;
 145         return 0;
 146 }
 147
 148 static int bpf_map_charge_memlock(struct bpf_map *map)
 149 {
 150         struct user_struct *user = get_current_user();
 151         unsigned long memlock_limit;
 152
 153         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 154
 155         atomic_long_add(map->pages, &user->locked_vm);
 156
 157         if (atomic_long_read(&user->locked_vm) > memlock_limit) {
 158                 atomic_long_sub(map->pages, &user->locked_vm);
 159                 free_uid(user);
 160                 return -EPERM;
 161         }
 162         map->user = user;
 163         return 0;
 164 }
 165
 166 static void bpf_map_uncharge_memlock(struct bpf_map *map)
 167 {
 168         struct user_struct *user = map->user;
 169
 170         atomic_long_sub(map->pages, &user->locked_vm);
 171         free_uid(user);
 172 }
 173
 174 static int bpf_map_alloc_id(struct bpf_map *map)
 175 {
 176         int id;
 177
 178         spin_lock_bh(&map_idr_lock);
 179         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 180         if (id > 0)
 181                 map->id = id;
 182         spin_unlock_bh(&map_idr_lock);
 183
 184         if (WARN_ON_ONCE(!id))
 185                 return -ENOSPC;
 186
 187         return id > 0 ? 0 : id;
 188 }
 189
 190 static void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 191 {
 192         unsigned long flags;
 193
 194         if (do_idr_lock)
 195                 spin_lock_irqsave(&map_idr_lock, flags);
 196         else
 197                 __acquire(&map_idr_lock);
 198
 199         idr_remove(&map_idr, map->id);
 200
 201         if (do_idr_lock)
 202                 spin_unlock_irqrestore(&map_idr_lock, flags);
 203         else
 204                 __release(&map_idr_lock);
 205 }
 206
 207 /* called from workqueue */
 208 static void bpf_map_free_deferred(struct work_struct *work)
 209 {
 210         struct bpf_map *map = container_of(work, struct bpf_map, work);
 211
 212         bpf_map_uncharge_memlock(map);
 213         /* implementation dependent freeing */
 214         map->ops->map_free(map);
 215 }
 216
 217 static void bpf_map_put_uref(struct bpf_map *map)
 218 {
 219         if (atomic_dec_and_test(&map->usercnt)) {
 220                 if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
 221                         bpf_fd_array_map_clear(map);
 222         }
 223 }
 224
 225 /* decrement map refcnt and schedule it for freeing via workqueue
 226  * (unrelying map implementation ops->map_free() might sleep)
 227  */
 228 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 229 {
 230         if (atomic_dec_and_test(&map->refcnt)) {
 231                 /* bpf_map_free_id() must be called first */
 232                 bpf_map_free_id(map, do_idr_lock);
 233                 INIT_WORK(&map->work, bpf_map_free_deferred);
 234                 schedule_work(&map->work);
 235         }
 236 }
 237
 238 void bpf_map_put(struct bpf_map *map)
 239 {
 240         __bpf_map_put(map, true);
 241 }
 242
 243 void bpf_map_put_with_uref(struct bpf_map *map)
 244 {
 245         bpf_map_put_uref(map);
 246         bpf_map_put(map);
 247 }
 248
 249 static int bpf_map_release(struct inode *inode, struct file *filp)
 250 {
 251         struct bpf_map *map = filp->private_data;
 252
 253         if (map->ops->map_release)
 254                 map->ops->map_release(map, filp);
 255
 256         bpf_map_put_with_uref(map);
 257         return 0;
 258 }
 259
 260 #ifdef CONFIG_PROC_FS
 261 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 262 {
 263         const struct bpf_map *map = filp->private_data;
 264         const struct bpf_array *array;
 265         u32 owner_prog_type = 0;
 266         u32 owner_jited = 0;
 267
 268         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 269                 array = container_of(map, struct bpf_array, map);
 270                 owner_prog_type = array->owner_prog_type;
 271                 owner_jited = array->owner_jited;
 272         }
 273
 274         seq_printf(m,
 275                    "map_type:\t%u\n"
 276                    "key_size:\t%u\n"
 277                    "value_size:\t%u\n"
 278                    "max_entries:\t%u\n"
 279                    "map_flags:\t%#x\n"
 280                    "memlock:\t%llu\n",
 281                    map->map_type,
 282                    map->key_size,
 283                    map->value_size,
 284                    map->max_entries,
 285                    map->map_flags,
 286                    map->pages * 1ULL << PAGE_SHIFT);
 287
 288         if (owner_prog_type) {
 289                 seq_printf(m, "owner_prog_type:\t%u\n",
 290                            owner_prog_type);
 291                 seq_printf(m, "owner_jited:\t%u\n",
 292                            owner_jited);
 293         }
 294 }
 295 #endif
 296
 297 static const struct file_operations bpf_map_fops = {
 298 #ifdef CONFIG_PROC_FS
 299         .show_fdinfo    = bpf_map_show_fdinfo,
 300 #endif
 301         .release        = bpf_map_release,
 302 };
 303
 304 int bpf_map_new_fd(struct bpf_map *map)
 305 {
 306         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 307                                 O_RDWR | O_CLOEXEC);
 308 }
 309
 310 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 311 #define CHECK_ATTR(CMD) \
 312         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 313                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 314                    sizeof(*attr) - \
 315                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 316                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 317
 318 /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes.
 319  * Return 0 on success and < 0 on error.
 320  */
 321 static int bpf_obj_name_cpy(char *dst, const char *src)
 322 {
 323         const char *end = src + BPF_OBJ_NAME_LEN;
 324
 325         memset(dst, 0, BPF_OBJ_NAME_LEN);
 326
 327         /* Copy all isalnum() and '_' char */
 328         while (src < end && *src) {
 329                 if (!isalnum(*src) && *src != '_')
 330                         return -EINVAL;
 331                 *dst++ = *src++;
 332         }
 333
 334         /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */
 335         if (src == end)
 336                 return -EINVAL;
 337
 338         return 0;
 339 }
 340
 341 #define BPF_MAP_CREATE_LAST_FIELD map_name
 342 /* called via syscall */
 343 static int map_create(union bpf_attr *attr)
 344 {
 345         int numa_node = bpf_map_attr_numa_node(attr);
 346         struct bpf_map *map;
 347         int err;
 348
 349         err = CHECK_ATTR(BPF_MAP_CREATE);
 350         if (err)
 351                 return -EINVAL;
 352
 353         if (numa_node != NUMA_NO_NODE &&
 354             ((unsigned int)numa_node >= nr_node_ids ||
 355              !node_online(numa_node)))
 356                 return -EINVAL;
 357
 358         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 359         map = find_and_alloc_map(attr);
 360         if (IS_ERR(map))
 361                 return PTR_ERR(map);
 362
 363         err = bpf_obj_name_cpy(map->name, attr->map_name);
 364         if (err)
 365                 goto free_map_nouncharge;
 366
 367         atomic_set(&map->refcnt, 1);
 368         atomic_set(&map->usercnt, 1);
 369
 370         err = bpf_map_charge_memlock(map);
 371         if (err)
 372                 goto free_map_nouncharge;
 373
 374         err = bpf_map_alloc_id(map);
 375         if (err)
 376                 goto free_map;
 377
 378         err = bpf_map_new_fd(map);
 379         if (err < 0) {
 380                 /* failed to allocate fd.
 381                  * bpf_map_put() is needed because the above
 382                  * bpf_map_alloc_id() has published the map
 383                  * to the userspace and the userspace may
 384                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 385                  */
 386                 bpf_map_put(map);
 387                 return err;
 388         }
 389
 390         trace_bpf_map_create(map, err);
 391         return err;
 392
 393 free_map:
 394         bpf_map_uncharge_memlock(map);
 395 free_map_nouncharge:
 396         map->ops->map_free(map);
 397         return err;
 398 }
 399
 400 /* if error is returned, fd is released.
 401  * On success caller should complete fd access with matching fdput()
 402  */
 403 struct bpf_map *__bpf_map_get(struct fd f)
 404 {
 405         if (!f.file)
 406                 return ERR_PTR(-EBADF);
 407         if (f.file->f_op != &bpf_map_fops) {
 408                 fdput(f);
 409                 return ERR_PTR(-EINVAL);
 410         }
 411
 412         return f.file->private_data;
 413 }
 414
 415 /* prog's and map's refcnt limit */
 416 #define BPF_MAX_REFCNT 32768
 417
 418 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
 419 {
 420         if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
 421                 atomic_dec(&map->refcnt);
 422                 return ERR_PTR(-EBUSY);
 423         }
 424         if (uref)
 425                 atomic_inc(&map->usercnt);
 426         return map;
 427 }
 428
 429 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 430 {
 431         struct fd f = fdget(ufd);
 432         struct bpf_map *map;
 433
 434         map = __bpf_map_get(f);
 435         if (IS_ERR(map))
 436                 return map;
 437
 438         map = bpf_map_inc(map, true);
 439         fdput(f);
 440
 441         return map;
 442 }
 443
 444 /* map_idr_lock should have been held */
 445 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
 446                                             bool uref)
 447 {
 448         int refold;
 449
 450         refold = __atomic_add_unless(&map->refcnt, 1, 0);
 451
 452         if (refold >= BPF_MAX_REFCNT) {
 453                 __bpf_map_put(map, false);
 454                 return ERR_PTR(-EBUSY);
 455         }
 456
 457         if (!refold)
 458                 return ERR_PTR(-ENOENT);
 459
 460         if (uref)
 461                 atomic_inc(&map->usercnt);
 462
 463         return map;
 464 }
 465
 466 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 467 {
 468         return -ENOTSUPP;
 469 }
 470
 471 /* last field in 'union bpf_attr' used by this command */
 472 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
 473
 474 static int map_lookup_elem(union bpf_attr *attr)
 475 {
 476         void __user *ukey = u64_to_user_ptr(attr->key);
 477         void __user *uvalue = u64_to_user_ptr(attr->value);
 478         int ufd = attr->map_fd;
 479         struct bpf_map *map;
 480         void *key, *value, *ptr;
 481         u32 value_size;
 482         struct fd f;
 483         int err;
 484
 485         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
 486                 return -EINVAL;
 487
 488         f = fdget(ufd);
 489         map = __bpf_map_get(f);
 490         if (IS_ERR(map))
 491                 return PTR_ERR(map);
 492
 493         key = memdup_user(ukey, map->key_size);
 494         if (IS_ERR(key)) {
 495                 err = PTR_ERR(key);
 496                 goto err_put;
 497         }
 498
 499         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 500             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 501             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
 502                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 503         else if (IS_FD_MAP(map))
 504                 value_size = sizeof(u32);
 505         else
 506                 value_size = map->value_size;
 507
 508         err = -ENOMEM;
 509         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 510         if (!value)
 511                 goto free_key;
 512
 513         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 514             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 515                 err = bpf_percpu_hash_copy(map, key, value);
 516         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 517                 err = bpf_percpu_array_copy(map, key, value);
 518         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 519                 err = bpf_stackmap_copy(map, key, value);
 520         } else if (IS_FD_ARRAY(map)) {
 521                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 522         } else if (IS_FD_HASH(map)) {
 523                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 524         } else {
 525                 rcu_read_lock();
 526                 ptr = map->ops->map_lookup_elem(map, key);
 527                 if (ptr)
 528                         memcpy(value, ptr, value_size);
 529                 rcu_read_unlock();
 530                 err = ptr ? 0 : -ENOENT;
 531         }
 532
 533         if (err)
 534                 goto free_value;
 535
 536         err = -EFAULT;
 537         if (copy_to_user(uvalue, value, value_size) != 0)
 538                 goto free_value;
 539
 540         trace_bpf_map_lookup_elem(map, ufd, key, value);
 541         err = 0;
 542
 543 free_value:
 544         kfree(value);
 545 free_key:
 546         kfree(key);
 547 err_put:
 548         fdput(f);
 549         return err;
 550 }
 551
 552 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
 553
 554 static int map_update_elem(union bpf_attr *attr)
 555 {
 556         void __user *ukey = u64_to_user_ptr(attr->key);
 557         void __user *uvalue = u64_to_user_ptr(attr->value);
 558         int ufd = attr->map_fd;
 559         struct bpf_map *map;
 560         void *key, *value;
 561         u32 value_size;
 562         struct fd f;
 563         int err;
 564
 565         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
 566                 return -EINVAL;
 567
 568         f = fdget(ufd);
 569         map = __bpf_map_get(f);
 570         if (IS_ERR(map))
 571                 return PTR_ERR(map);
 572
 573         key = memdup_user(ukey, map->key_size);
 574         if (IS_ERR(key)) {
 575                 err = PTR_ERR(key);
 576                 goto err_put;
 577         }
 578
 579         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 580             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 581             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY)
 582                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 583         else
 584                 value_size = map->value_size;
 585
 586         err = -ENOMEM;
 587         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 588         if (!value)
 589                 goto free_key;
 590
 591         err = -EFAULT;
 592         if (copy_from_user(value, uvalue, value_size) != 0)
 593                 goto free_value;
 594
 595         /* Need to create a kthread, thus must support schedule */
 596         if (map->map_type == BPF_MAP_TYPE_CPUMAP) {
 597                 err = map->ops->map_update_elem(map, key, value, attr->flags);
 598                 goto out;
 599         }
 600
 601         /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
 602          * inside bpf map update or delete otherwise deadlocks are possible
 603          */
 604         preempt_disable();
 605         __this_cpu_inc(bpf_prog_active);
 606         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 607             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 608                 err = bpf_percpu_hash_update(map, key, value, attr->flags);
 609         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 610                 err = bpf_percpu_array_update(map, key, value, attr->flags);
 611         } else if (map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
 612                    map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
 613                    map->map_type == BPF_MAP_TYPE_CGROUP_ARRAY ||
 614                    map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) {
 615                 rcu_read_lock();
 616                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 617                                                    attr->flags);
 618                 rcu_read_unlock();
 619         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 620                 rcu_read_lock();
 621                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 622                                                   attr->flags);
 623                 rcu_read_unlock();
 624         } else {
 625                 rcu_read_lock();
 626                 err = map->ops->map_update_elem(map, key, value, attr->flags);
 627                 rcu_read_unlock();
 628         }
 629         __this_cpu_dec(bpf_prog_active);
 630         preempt_enable();
 631 out:
 632         if (!err)
 633                 trace_bpf_map_update_elem(map, ufd, key, value);
 634 free_value:
 635         kfree(value);
 636 free_key:
 637         kfree(key);
 638 err_put:
 639         fdput(f);
 640         return err;
 641 }
 642
 643 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
 644
 645 static int map_delete_elem(union bpf_attr *attr)
 646 {
 647         void __user *ukey = u64_to_user_ptr(attr->key);
 648         int ufd = attr->map_fd;
 649         struct bpf_map *map;
 650         struct fd f;
 651         void *key;
 652         int err;
 653
 654         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
 655                 return -EINVAL;
 656
 657         f = fdget(ufd);
 658         map = __bpf_map_get(f);
 659         if (IS_ERR(map))
 660                 return PTR_ERR(map);
 661
 662         key = memdup_user(ukey, map->key_size);
 663         if (IS_ERR(key)) {
 664                 err = PTR_ERR(key);
 665                 goto err_put;
 666         }
 667
 668         preempt_disable();
 669         __this_cpu_inc(bpf_prog_active);
 670         rcu_read_lock();
 671         err = map->ops->map_delete_elem(map, key);
 672         rcu_read_unlock();
 673         __this_cpu_dec(bpf_prog_active);
 674         preempt_enable();
 675
 676         if (!err)
 677                 trace_bpf_map_delete_elem(map, ufd, key);
 678         kfree(key);
 679 err_put:
 680         fdput(f);
 681         return err;
 682 }
 683
 684 /* last field in 'union bpf_attr' used by this command */
 685 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
 686
 687 static int map_get_next_key(union bpf_attr *attr)
 688 {
 689         void __user *ukey = u64_to_user_ptr(attr->key);
 690         void __user *unext_key = u64_to_user_ptr(attr->next_key);
 691         int ufd = attr->map_fd;
 692         struct bpf_map *map;
 693         void *key, *next_key;
 694         struct fd f;
 695         int err;
 696
 697         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
 698                 return -EINVAL;
 699
 700         f = fdget(ufd);
 701         map = __bpf_map_get(f);
 702         if (IS_ERR(map))
 703                 return PTR_ERR(map);
 704
 705         if (ukey) {
 706                 key = memdup_user(ukey, map->key_size);
 707                 if (IS_ERR(key)) {
 708                         err = PTR_ERR(key);
 709                         goto err_put;
 710                 }
 711         } else {
 712                 key = NULL;
 713         }
 714
 715         err = -ENOMEM;
 716         next_key = kmalloc(map->key_size, GFP_USER);
 717         if (!next_key)
 718                 goto free_key;
 719
 720         rcu_read_lock();
 721         err = map->ops->map_get_next_key(map, key, next_key);
 722         rcu_read_unlock();
 723         if (err)
 724                 goto free_next_key;
 725
 726         err = -EFAULT;
 727         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
 728                 goto free_next_key;
 729
 730         trace_bpf_map_next_key(map, ufd, key, next_key);
 731         err = 0;
 732
 733 free_next_key:
 734         kfree(next_key);
 735 free_key:
 736         kfree(key);
 737 err_put:
 738         fdput(f);
 739         return err;
 740 }
 741
 742 static const struct bpf_prog_ops * const bpf_prog_types[] = {
 743 #define BPF_PROG_TYPE(_id, _name) \
 744         [_id] = & _name ## _prog_ops,
 745 #define BPF_MAP_TYPE(_id, _ops)
 746 #include <linux/bpf_types.h>
 747 #undef BPF_PROG_TYPE
 748 #undef BPF_MAP_TYPE
 749 };
 750
 751 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
 752 #define BPF_PROG_TYPE(_id, _name) \
 753         [_id] = & _name ## _verifier_ops,
 754 #define BPF_MAP_TYPE(_id, _ops)
 755 #include <linux/bpf_types.h>
 756 #undef BPF_PROG_TYPE
 757 #undef BPF_MAP_TYPE
 758 };
 759
 760 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
 761 {
 762         if (type >= ARRAY_SIZE(bpf_prog_types) || !bpf_prog_types[type])
 763                 return -EINVAL;
 764
 765         prog->aux->ops = bpf_prog_types[type];
 766         prog->aux->vops = bpf_verifier_ops[type];
 767         prog->type = type;
 768         return 0;
 769 }
 770
 771 /* drop refcnt on maps used by eBPF program and free auxilary data */
 772 static void free_used_maps(struct bpf_prog_aux *aux)
 773 {
 774         int i;
 775
 776         for (i = 0; i < aux->used_map_cnt; i++)
 777                 bpf_map_put(aux->used_maps[i]);
 778
 779         kfree(aux->used_maps);
 780 }
 781
 782 int __bpf_prog_charge(struct user_struct *user, u32 pages)
 783 {
 784         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 785         unsigned long user_bufs;
 786
 787         if (user) {
 788                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
 789                 if (user_bufs > memlock_limit) {
 790                         atomic_long_sub(pages, &user->locked_vm);
 791                         return -EPERM;
 792                 }
 793         }
 794
 795         return 0;
 796 }
 797
 798 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
 799 {
 800         if (user)
 801                 atomic_long_sub(pages, &user->locked_vm);
 802 }
 803
 804 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
 805 {
 806         struct user_struct *user = get_current_user();
 807         int ret;
 808
 809         ret = __bpf_prog_charge(user, prog->pages);
 810         if (ret) {
 811                 free_uid(user);
 812                 return ret;
 813         }
 814
 815         prog->aux->user = user;
 816         return 0;
 817 }
 818
 819 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
 820 {
 821         struct user_struct *user = prog->aux->user;
 822
 823         __bpf_prog_uncharge(user, prog->pages);
 824         free_uid(user);
 825 }
 826
 827 static int bpf_prog_alloc_id(struct bpf_prog *prog)
 828 {
 829         int id;
 830
 831         spin_lock_bh(&prog_idr_lock);
 832         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
 833         if (id > 0)
 834                 prog->aux->id = id;
 835         spin_unlock_bh(&prog_idr_lock);
 836
 837         /* id is in [1, INT_MAX) */
 838         if (WARN_ON_ONCE(!id))
 839                 return -ENOSPC;
 840
 841         return id > 0 ? 0 : id;
 842 }
 843
 844 static void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
 845 {
 846         /* cBPF to eBPF migrations are currently not in the idr store. */
 847         if (!prog->aux->id)
 848                 return;
 849
 850         if (do_idr_lock)
 851                 spin_lock_bh(&prog_idr_lock);
 852         else
 853                 __acquire(&prog_idr_lock);
 854
 855         idr_remove(&prog_idr, prog->aux->id);
 856
 857         if (do_idr_lock)
 858                 spin_unlock_bh(&prog_idr_lock);
 859         else
 860                 __release(&prog_idr_lock);
 861 }
 862
 863 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
 864 {
 865         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
 866
 867         free_used_maps(aux);
 868         bpf_prog_uncharge_memlock(aux->prog);
 869         bpf_prog_free(aux->prog);
 870 }
 871
 872 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
 873 {
 874         if (atomic_dec_and_test(&prog->aux->refcnt)) {
 875                 trace_bpf_prog_put_rcu(prog);
 876                 /* bpf_prog_free_id() must be called first */
 877                 bpf_prog_free_id(prog, do_idr_lock);
 878                 bpf_prog_kallsyms_del(prog);
 879                 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
 880         }
 881 }
 882
 883 void bpf_prog_put(struct bpf_prog *prog)
 884 {
 885         __bpf_prog_put(prog, true);
 886 }
 887 EXPORT_SYMBOL_GPL(bpf_prog_put);
 888
 889 static int bpf_prog_release(struct inode *inode, struct file *filp)
 890 {
 891         struct bpf_prog *prog = filp->private_data;
 892
 893         bpf_prog_put(prog);
 894         return 0;
 895 }
 896
 897 #ifdef CONFIG_PROC_FS
 898 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
 899 {
 900         const struct bpf_prog *prog = filp->private_data;
 901         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
 902
 903         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
 904         seq_printf(m,
 905                    "prog_type:\t%u\n"
 906                    "prog_jited:\t%u\n"
 907                    "prog_tag:\t%s\n"
 908                    "memlock:\t%llu\n",
 909                    prog->type,
 910                    prog->jited,
 911                    prog_tag,
 912                    prog->pages * 1ULL << PAGE_SHIFT);
 913 }
 914 #endif
 915
 916 static const struct file_operations bpf_prog_fops = {
 917 #ifdef CONFIG_PROC_FS
 918         .show_fdinfo    = bpf_prog_show_fdinfo,
 919 #endif
 920         .release        = bpf_prog_release,
 921 };
 922
 923 int bpf_prog_new_fd(struct bpf_prog *prog)
 924 {
 925         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
 926                                 O_RDWR | O_CLOEXEC);
 927 }
 928
 929 static struct bpf_prog *____bpf_prog_get(struct fd f)
 930 {
 931         if (!f.file)
 932                 return ERR_PTR(-EBADF);
 933         if (f.file->f_op != &bpf_prog_fops) {
 934                 fdput(f);
 935                 return ERR_PTR(-EINVAL);
 936         }
 937
 938         return f.file->private_data;
 939 }
 940
 941 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
 942 {
 943         if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
 944                 atomic_sub(i, &prog->aux->refcnt);
 945                 return ERR_PTR(-EBUSY);
 946         }
 947         return prog;
 948 }
 949 EXPORT_SYMBOL_GPL(bpf_prog_add);
 950
 951 void bpf_prog_sub(struct bpf_prog *prog, int i)
 952 {
 953         /* Only to be used for undoing previous bpf_prog_add() in some
 954          * error path. We still know that another entity in our call
 955          * path holds a reference to the program, thus atomic_sub() can
 956          * be safely used in such cases!
 957          */
 958         WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
 959 }
 960 EXPORT_SYMBOL_GPL(bpf_prog_sub);
 961
 962 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
 963 {
 964         return bpf_prog_add(prog, 1);
 965 }
 966 EXPORT_SYMBOL_GPL(bpf_prog_inc);
 967
 968 /* prog_idr_lock should have been held */
 969 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
 970 {
 971         int refold;
 972
 973         refold = __atomic_add_unless(&prog->aux->refcnt, 1, 0);
 974
 975         if (refold >= BPF_MAX_REFCNT) {
 976                 __bpf_prog_put(prog, false);
 977                 return ERR_PTR(-EBUSY);
 978         }
 979
 980         if (!refold)
 981                 return ERR_PTR(-ENOENT);
 982
 983         return prog;
 984 }
 985 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
 986
 987 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *type)
 988 {
 989         struct fd f = fdget(ufd);
 990         struct bpf_prog *prog;
 991
 992         prog = ____bpf_prog_get(f);
 993         if (IS_ERR(prog))
 994                 return prog;
 995         if (type && prog->type != *type) {
 996                 prog = ERR_PTR(-EINVAL);
 997                 goto out;
 998         }
 999
1000         prog = bpf_prog_inc(prog);
1001 out:
1002         fdput(f);
1003         return prog;
1004 }
1005
1006 struct bpf_prog *bpf_prog_get(u32 ufd)
1007 {
1008         return __bpf_prog_get(ufd, NULL);
1009 }
1010
1011 struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type)
1012 {
1013         struct bpf_prog *prog = __bpf_prog_get(ufd, &type);
1014
1015         if (!IS_ERR(prog))
1016                 trace_bpf_prog_get_type(prog);
1017         return prog;
1018 }
1019 EXPORT_SYMBOL_GPL(bpf_prog_get_type);
1020
1021 /* last field in 'union bpf_attr' used by this command */
1022 #define BPF_PROG_LOAD_LAST_FIELD prog_name
1023
1024 static int bpf_prog_load(union bpf_attr *attr)
1025 {
1026         enum bpf_prog_type type = attr->prog_type;
1027         struct bpf_prog *prog;
1028         int err;
1029         char license[128];
1030         bool is_gpl;
1031
1032         if (CHECK_ATTR(BPF_PROG_LOAD))
1033                 return -EINVAL;
1034
1035         if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT)
1036                 return -EINVAL;
1037
1038         /* copy eBPF program license from user space */
1039         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
1040                               sizeof(license) - 1) < 0)
1041                 return -EFAULT;
1042         license[sizeof(license) - 1] = 0;
1043
1044         /* eBPF programs must be GPL compatible to use GPL-ed functions */
1045         is_gpl = license_is_gpl_compatible(license);
1046
1047         if (attr->insn_cnt == 0 || attr->insn_cnt > BPF_MAXINSNS)
1048                 return -E2BIG;
1049
1050         if (type == BPF_PROG_TYPE_KPROBE &&
1051             attr->kern_version != LINUX_VERSION_CODE)
1052                 return -EINVAL;
1053
1054         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
1055             type != BPF_PROG_TYPE_CGROUP_SKB &&
1056             !capable(CAP_SYS_ADMIN))
1057                 return -EPERM;
1058
1059         /* plain bpf_prog allocation */
1060         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
1061         if (!prog)
1062                 return -ENOMEM;
1063
1064         err = bpf_prog_charge_memlock(prog);
1065         if (err)
1066                 goto free_prog_nouncharge;
1067
1068         prog->len = attr->insn_cnt;
1069
1070         err = -EFAULT;
1071         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
1072                            bpf_prog_insn_size(prog)) != 0)
1073                 goto free_prog;
1074
1075         prog->orig_prog = NULL;
1076         prog->jited = 0;
1077
1078         atomic_set(&prog->aux->refcnt, 1);
1079         prog->gpl_compatible = is_gpl ? 1 : 0;
1080
1081         /* find program type: socket_filter vs tracing_filter */
1082         err = find_prog_type(type, prog);
1083         if (err < 0)
1084                 goto free_prog;
1085
1086         prog->aux->load_time = ktime_get_boot_ns();
1087         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);
1088         if (err)
1089                 goto free_prog;
1090
1091         /* run eBPF verifier */
1092         err = bpf_check(&prog, attr);
1093         if (err < 0)
1094                 goto free_used_maps;
1095
1096         /* eBPF program is ready to be JITed */
1097         prog = bpf_prog_select_runtime(prog, &err);
1098         if (err < 0)
1099                 goto free_used_maps;
1100
1101         err = bpf_prog_alloc_id(prog);
1102         if (err)
1103                 goto free_used_maps;
1104
1105         err = bpf_prog_new_fd(prog);
1106         if (err < 0) {
1107                 /* failed to allocate fd.
1108                  * bpf_prog_put() is needed because the above
1109                  * bpf_prog_alloc_id() has published the prog
1110                  * to the userspace and the userspace may
1111                  * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID.
1112                  */
1113                 bpf_prog_put(prog);
1114                 return err;
1115         }
1116
1117         bpf_prog_kallsyms_add(prog);
1118         trace_bpf_prog_load(prog, err);
1119         return err;
1120
1121 free_used_maps:
1122         free_used_maps(prog->aux);
1123 free_prog:
1124         bpf_prog_uncharge_memlock(prog);
1125 free_prog_nouncharge:
1126         bpf_prog_free(prog);
1127         return err;
1128 }
1129
1130 #define BPF_OBJ_LAST_FIELD bpf_fd
1131
1132 static int bpf_obj_pin(const union bpf_attr *attr)
1133 {
1134         if (CHECK_ATTR(BPF_OBJ))
1135                 return -EINVAL;
1136
1137         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
1138 }
1139
1140 static int bpf_obj_get(const union bpf_attr *attr)
1141 {
1142         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
1143                 return -EINVAL;
1144
1145         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname));
1146 }
1147
1148 #ifdef CONFIG_CGROUP_BPF
1149
1150 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
1151
1152 static int sockmap_get_from_fd(const union bpf_attr *attr, bool attach)
1153 {
1154         struct bpf_prog *prog = NULL;
1155         int ufd = attr->target_fd;
1156         struct bpf_map *map;
1157         struct fd f;
1158         int err;
1159
1160         f = fdget(ufd);
1161         map = __bpf_map_get(f);
1162         if (IS_ERR(map))
1163                 return PTR_ERR(map);
1164
1165         if (attach) {
1166                 prog = bpf_prog_get_type(attr->attach_bpf_fd,
1167                                          BPF_PROG_TYPE_SK_SKB);
1168                 if (IS_ERR(prog)) {
1169                         fdput(f);
1170                         return PTR_ERR(prog);
1171                 }
1172         }
1173
1174         err = sock_map_prog(map, prog, attr->attach_type);
1175         if (err) {
1176                 fdput(f);
1177                 if (prog)
1178                         bpf_prog_put(prog);
1179                 return err;
1180         }
1181
1182         fdput(f);
1183         return 0;
1184 }
1185
1186 #define BPF_F_ATTACH_MASK \
1187         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
1188
1189 static int bpf_prog_attach(const union bpf_attr *attr)
1190 {
1191         enum bpf_prog_type ptype;
1192         struct bpf_prog *prog;
1193         struct cgroup *cgrp;
1194         int ret;
1195
1196         if (!capable(CAP_NET_ADMIN))
1197                 return -EPERM;
1198
1199         if (CHECK_ATTR(BPF_PROG_ATTACH))
1200                 return -EINVAL;
1201
1202         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
1203                 return -EINVAL;
1204
1205         switch (attr->attach_type) {
1206         case BPF_CGROUP_INET_INGRESS:
1207         case BPF_CGROUP_INET_EGRESS:
1208                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1209                 break;
1210         case BPF_CGROUP_INET_SOCK_CREATE:
1211                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1212                 break;
1213         case BPF_CGROUP_SOCK_OPS:
1214                 ptype = BPF_PROG_TYPE_SOCK_OPS;
1215                 break;
1216         case BPF_SK_SKB_STREAM_PARSER:
1217         case BPF_SK_SKB_STREAM_VERDICT:
1218                 return sockmap_get_from_fd(attr, true);
1219         default:
1220                 return -EINVAL;
1221         }
1222
1223         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1224         if (IS_ERR(prog))
1225                 return PTR_ERR(prog);
1226
1227         cgrp = cgroup_get_from_fd(attr->target_fd);
1228         if (IS_ERR(cgrp)) {
1229                 bpf_prog_put(prog);
1230                 return PTR_ERR(cgrp);
1231         }
1232
1233         ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type,
1234                                 attr->attach_flags);
1235         if (ret)
1236                 bpf_prog_put(prog);
1237         cgroup_put(cgrp);
1238
1239         return ret;
1240 }
1241
1242 #define BPF_PROG_DETACH_LAST_FIELD attach_type
1243
1244 static int bpf_prog_detach(const union bpf_attr *attr)
1245 {
1246         enum bpf_prog_type ptype;
1247         struct bpf_prog *prog;
1248         struct cgroup *cgrp;
1249         int ret;
1250
1251         if (!capable(CAP_NET_ADMIN))
1252                 return -EPERM;
1253
1254         if (CHECK_ATTR(BPF_PROG_DETACH))
1255                 return -EINVAL;
1256
1257         switch (attr->attach_type) {
1258         case BPF_CGROUP_INET_INGRESS:
1259         case BPF_CGROUP_INET_EGRESS:
1260                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1261                 break;
1262         case BPF_CGROUP_INET_SOCK_CREATE:
1263                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1264                 break;
1265         case BPF_CGROUP_SOCK_OPS:
1266                 ptype = BPF_PROG_TYPE_SOCK_OPS;
1267                 break;
1268         case BPF_SK_SKB_STREAM_PARSER:
1269         case BPF_SK_SKB_STREAM_VERDICT:
1270                 return sockmap_get_from_fd(attr, false);
1271         default:
1272                 return -EINVAL;
1273         }
1274
1275         cgrp = cgroup_get_from_fd(attr->target_fd);
1276         if (IS_ERR(cgrp))
1277                 return PTR_ERR(cgrp);
1278
1279         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1280         if (IS_ERR(prog))
1281                 prog = NULL;
1282
1283         ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0);
1284         if (prog)
1285                 bpf_prog_put(prog);
1286         cgroup_put(cgrp);
1287         return ret;
1288 }
1289
1290 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
1291
1292 static int bpf_prog_query(const union bpf_attr *attr,
1293                           union bpf_attr __user *uattr)
1294 {
1295         struct cgroup *cgrp;
1296         int ret;
1297
1298         if (!capable(CAP_NET_ADMIN))
1299                 return -EPERM;
1300         if (CHECK_ATTR(BPF_PROG_QUERY))
1301                 return -EINVAL;
1302         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
1303                 return -EINVAL;
1304
1305         switch (attr->query.attach_type) {
1306         case BPF_CGROUP_INET_INGRESS:
1307         case BPF_CGROUP_INET_EGRESS:
1308         case BPF_CGROUP_INET_SOCK_CREATE:
1309         case BPF_CGROUP_SOCK_OPS:
1310                 break;
1311         default:
1312                 return -EINVAL;
1313         }
1314         cgrp = cgroup_get_from_fd(attr->query.target_fd);
1315         if (IS_ERR(cgrp))
1316                 return PTR_ERR(cgrp);
1317         ret = cgroup_bpf_query(cgrp, attr, uattr);
1318         cgroup_put(cgrp);
1319         return ret;
1320 }
1321 #endif /* CONFIG_CGROUP_BPF */
1322
1323 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
1324
1325 static int bpf_prog_test_run(const union bpf_attr *attr,
1326                              union bpf_attr __user *uattr)
1327 {
1328         struct bpf_prog *prog;
1329         int ret = -ENOTSUPP;
1330
1331         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
1332                 return -EINVAL;
1333
1334         prog = bpf_prog_get(attr->test.prog_fd);
1335         if (IS_ERR(prog))
1336                 return PTR_ERR(prog);
1337
1338         if (prog->aux->ops->test_run)
1339                 ret = prog->aux->ops->test_run(prog, attr, uattr);
1340
1341         bpf_prog_put(prog);
1342         return ret;
1343 }
1344
1345 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
1346
1347 static int bpf_obj_get_next_id(const union bpf_attr *attr,
1348                                union bpf_attr __user *uattr,
1349                                struct idr *idr,
1350                                spinlock_t *lock)
1351 {
1352         u32 next_id = attr->start_id;
1353         int err = 0;
1354
1355         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
1356                 return -EINVAL;
1357
1358         if (!capable(CAP_SYS_ADMIN))
1359                 return -EPERM;
1360
1361         next_id++;
1362         spin_lock_bh(lock);
1363         if (!idr_get_next(idr, &next_id))
1364                 err = -ENOENT;
1365         spin_unlock_bh(lock);
1366
1367         if (!err)
1368                 err = put_user(next_id, &uattr->next_id);
1369
1370         return err;
1371 }
1372
1373 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
1374
1375 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
1376 {
1377         struct bpf_prog *prog;
1378         u32 id = attr->prog_id;
1379         int fd;
1380
1381         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
1382                 return -EINVAL;
1383
1384         if (!capable(CAP_SYS_ADMIN))
1385                 return -EPERM;
1386
1387         spin_lock_bh(&prog_idr_lock);
1388         prog = idr_find(&prog_idr, id);
1389         if (prog)
1390                 prog = bpf_prog_inc_not_zero(prog);
1391         else
1392                 prog = ERR_PTR(-ENOENT);
1393         spin_unlock_bh(&prog_idr_lock);
1394
1395         if (IS_ERR(prog))
1396                 return PTR_ERR(prog);
1397
1398         fd = bpf_prog_new_fd(prog);
1399         if (fd < 0)
1400                 bpf_prog_put(prog);
1401
1402         return fd;
1403 }
1404
1405 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD map_id
1406
1407 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
1408 {
1409         struct bpf_map *map;
1410         u32 id = attr->map_id;
1411         int fd;
1412
1413         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID))
1414                 return -EINVAL;
1415
1416         if (!capable(CAP_SYS_ADMIN))
1417                 return -EPERM;
1418
1419         spin_lock_bh(&map_idr_lock);
1420         map = idr_find(&map_idr, id);
1421         if (map)
1422                 map = bpf_map_inc_not_zero(map, true);
1423         else
1424                 map = ERR_PTR(-ENOENT);
1425         spin_unlock_bh(&map_idr_lock);
1426
1427         if (IS_ERR(map))
1428                 return PTR_ERR(map);
1429
1430         fd = bpf_map_new_fd(map);
1431         if (fd < 0)
1432                 bpf_map_put(map);
1433
1434         return fd;
1435 }
1436
1437 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
1438                                    const union bpf_attr *attr,
1439                                    union bpf_attr __user *uattr)
1440 {
1441         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
1442         struct bpf_prog_info info = {};
1443         u32 info_len = attr->info.info_len;
1444         char __user *uinsns;
1445         u32 ulen;
1446         int err;
1447
1448         err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
1449         if (err)
1450                 return err;
1451         info_len = min_t(u32, sizeof(info), info_len);
1452
1453         if (copy_from_user(&info, uinfo, info_len))
1454                 return -EFAULT;
1455
1456         info.type = prog->type;
1457         info.id = prog->aux->id;
1458         info.load_time = prog->aux->load_time;
1459         info.created_by_uid = from_kuid_munged(current_user_ns(),
1460                                                prog->aux->user->uid);
1461
1462         memcpy(info.tag, prog->tag, sizeof(prog->tag));
1463         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
1464
1465         ulen = info.nr_map_ids;
1466         info.nr_map_ids = prog->aux->used_map_cnt;
1467         ulen = min_t(u32, info.nr_map_ids, ulen);
1468         if (ulen) {
1469                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
1470                 u32 i;
1471
1472                 for (i = 0; i < ulen; i++)
1473                         if (put_user(prog->aux->used_maps[i]->id,
1474                                      &user_map_ids[i]))
1475                                 return -EFAULT;
1476         }
1477
1478         if (!capable(CAP_SYS_ADMIN)) {
1479                 info.jited_prog_len = 0;
1480                 info.xlated_prog_len = 0;
1481                 goto done;
1482         }
1483
1484         ulen = info.jited_prog_len;
1485         info.jited_prog_len = prog->jited_len;
1486         if (info.jited_prog_len && ulen) {
1487                 uinsns = u64_to_user_ptr(info.jited_prog_insns);
1488                 ulen = min_t(u32, info.jited_prog_len, ulen);
1489                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
1490                         return -EFAULT;
1491         }
1492
1493         ulen = info.xlated_prog_len;
1494         info.xlated_prog_len = bpf_prog_insn_size(prog);
1495         if (info.xlated_prog_len && ulen) {
1496                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
1497                 ulen = min_t(u32, info.xlated_prog_len, ulen);
1498                 if (copy_to_user(uinsns, prog->insnsi, ulen))
1499                         return -EFAULT;
1500         }
1501
1502 done:
1503         if (copy_to_user(uinfo, &info, info_len) ||
1504             put_user(info_len, &uattr->info.info_len))
1505                 return -EFAULT;
1506
1507         return 0;
1508 }
1509
1510 static int bpf_map_get_info_by_fd(struct bpf_map *map,
1511                                   const union bpf_attr *attr,
1512                                   union bpf_attr __user *uattr)
1513 {
1514         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
1515         struct bpf_map_info info = {};
1516         u32 info_len = attr->info.info_len;
1517         int err;
1518
1519         err = check_uarg_tail_zero(uinfo, sizeof(info), info_len);
1520         if (err)
1521                 return err;
1522         info_len = min_t(u32, sizeof(info), info_len);
1523
1524         info.type = map->map_type;
1525         info.id = map->id;
1526         info.key_size = map->key_size;
1527         info.value_size = map->value_size;
1528         info.max_entries = map->max_entries;
1529         info.map_flags = map->map_flags;
1530         memcpy(info.name, map->name, sizeof(map->name));
1531
1532         if (copy_to_user(uinfo, &info, info_len) ||
1533             put_user(info_len, &uattr->info.info_len))
1534                 return -EFAULT;
1535
1536         return 0;
1537 }
1538
1539 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
1540
1541 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
1542                                   union bpf_attr __user *uattr)
1543 {
1544         int ufd = attr->info.bpf_fd;
1545         struct fd f;
1546         int err;
1547
1548         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
1549                 return -EINVAL;
1550
1551         f = fdget(ufd);
1552         if (!f.file)
1553                 return -EBADFD;
1554
1555         if (f.file->f_op == &bpf_prog_fops)
1556                 err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
1557                                               uattr);
1558         else if (f.file->f_op == &bpf_map_fops)
1559                 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
1560                                              uattr);
1561         else
1562                 err = -EINVAL;
1563
1564         fdput(f);
1565         return err;
1566 }
1567
1568 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
1569 {
1570         union bpf_attr attr = {};
1571         int err;
1572
1573         if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
1574                 return -EPERM;
1575
1576         err = check_uarg_tail_zero(uattr, sizeof(attr), size);
1577         if (err)
1578                 return err;
1579         size = min_t(u32, size, sizeof(attr));
1580
1581         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
1582         if (copy_from_user(&attr, uattr, size) != 0)
1583                 return -EFAULT;
1584
1585         switch (cmd) {
1586         case BPF_MAP_CREATE:
1587                 err = map_create(&attr);
1588                 break;
1589         case BPF_MAP_LOOKUP_ELEM:
1590                 err = map_lookup_elem(&attr);
1591                 break;
1592         case BPF_MAP_UPDATE_ELEM:
1593                 err = map_update_elem(&attr);
1594                 break;
1595         case BPF_MAP_DELETE_ELEM:
1596                 err = map_delete_elem(&attr);
1597                 break;
1598         case BPF_MAP_GET_NEXT_KEY:
1599                 err = map_get_next_key(&attr);
1600                 break;
1601         case BPF_PROG_LOAD:
1602                 err = bpf_prog_load(&attr);
1603                 break;
1604         case BPF_OBJ_PIN:
1605                 err = bpf_obj_pin(&attr);
1606                 break;
1607         case BPF_OBJ_GET:
1608                 err = bpf_obj_get(&attr);
1609                 break;
1610 #ifdef CONFIG_CGROUP_BPF
1611         case BPF_PROG_ATTACH:
1612                 err = bpf_prog_attach(&attr);
1613                 break;
1614         case BPF_PROG_DETACH:
1615                 err = bpf_prog_detach(&attr);
1616                 break;
1617         case BPF_PROG_QUERY:
1618                 err = bpf_prog_query(&attr, uattr);
1619                 break;
1620 #endif
1621         case BPF_PROG_TEST_RUN:
1622                 err = bpf_prog_test_run(&attr, uattr);
1623                 break;
1624         case BPF_PROG_GET_NEXT_ID:
1625                 err = bpf_obj_get_next_id(&attr, uattr,
1626                                           &prog_idr, &prog_idr_lock);
1627                 break;
1628         case BPF_MAP_GET_NEXT_ID:
1629                 err = bpf_obj_get_next_id(&attr, uattr,
1630                                           &map_idr, &map_idr_lock);
1631                 break;
1632         case BPF_PROG_GET_FD_BY_ID:
1633                 err = bpf_prog_get_fd_by_id(&attr);
1634                 break;
1635         case BPF_MAP_GET_FD_BY_ID:
1636                 err = bpf_map_get_fd_by_id(&attr);
1637                 break;
1638         case BPF_OBJ_GET_INFO_BY_FD:
1639                 err = bpf_obj_get_info_by_fd(&attr, uattr);
1640                 break;
1641         default:
1642                 err = -EINVAL;
1643                 break;
1644         }
1645
1646         return err;
1647 }