1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
12 #include <linux/bpf.h>
13 #include <linux/bpf_trace.h>
14 #include <linux/bpf_lirc.h>
15 #include <linux/btf.h>
16 #include <linux/syscalls.h>
17 #include <linux/slab.h>
18 #include <linux/sched/signal.h>
19 #include <linux/vmalloc.h>
20 #include <linux/mmzone.h>
21 #include <linux/anon_inodes.h>
22 #include <linux/fdtable.h>
23 #include <linux/file.h>
25 #include <linux/license.h>
26 #include <linux/filter.h>
27 #include <linux/version.h>
28 #include <linux/kernel.h>
29 #include <linux/idr.h>
30 #include <linux/cred.h>
31 #include <linux/timekeeping.h>
32 #include <linux/ctype.h>
33 #include <linux/nospec.h>
35 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
36 (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
37 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
38 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
39 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
40 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
42 #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
44 DEFINE_PER_CPU(int, bpf_prog_active
);
45 static DEFINE_IDR(prog_idr
);
46 static DEFINE_SPINLOCK(prog_idr_lock
);
47 static DEFINE_IDR(map_idr
);
48 static DEFINE_SPINLOCK(map_idr_lock
);
50 int sysctl_unprivileged_bpf_disabled __read_mostly
;
52 static const struct bpf_map_ops
* const bpf_map_types
[] = {
53 #define BPF_PROG_TYPE(_id, _ops)
54 #define BPF_MAP_TYPE(_id, _ops) \
56 #include <linux/bpf_types.h>
62 * If we're handed a bigger struct than we know of, ensure all the unknown bits
63 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
64 * we don't know about yet.
66 * There is a ToCToU between this function call and the following
67 * copy_from_user() call. However, this is not a concern since this function is
68 * meant to be a future-proofing of bits.
70 int bpf_check_uarg_tail_zero(void __user
*uaddr
,
74 unsigned char __user
*addr
;
75 unsigned char __user
*end
;
79 if (unlikely(actual_size
> PAGE_SIZE
)) /* silly large */
82 if (unlikely(!access_ok(uaddr
, actual_size
)))
85 if (actual_size
<= expected_size
)
88 addr
= uaddr
+ expected_size
;
89 end
= uaddr
+ actual_size
;
91 for (; addr
< end
; addr
++) {
92 err
= get_user(val
, addr
);
102 const struct bpf_map_ops bpf_map_offload_ops
= {
103 .map_alloc
= bpf_map_offload_map_alloc
,
104 .map_free
= bpf_map_offload_map_free
,
105 .map_check_btf
= map_check_no_btf
,
108 static struct bpf_map
*find_and_alloc_map(union bpf_attr
*attr
)
110 const struct bpf_map_ops
*ops
;
111 u32 type
= attr
->map_type
;
115 if (type
>= ARRAY_SIZE(bpf_map_types
))
116 return ERR_PTR(-EINVAL
);
117 type
= array_index_nospec(type
, ARRAY_SIZE(bpf_map_types
));
118 ops
= bpf_map_types
[type
];
120 return ERR_PTR(-EINVAL
);
122 if (ops
->map_alloc_check
) {
123 err
= ops
->map_alloc_check(attr
);
127 if (attr
->map_ifindex
)
128 ops
= &bpf_map_offload_ops
;
129 map
= ops
->map_alloc(attr
);
133 map
->map_type
= type
;
137 void *bpf_map_area_alloc(size_t size
, int numa_node
)
139 /* We definitely need __GFP_NORETRY, so OOM killer doesn't
140 * trigger under memory pressure as we really just want to
143 const gfp_t flags
= __GFP_NOWARN
| __GFP_NORETRY
| __GFP_ZERO
;
146 if (size
<= (PAGE_SIZE
<< PAGE_ALLOC_COSTLY_ORDER
)) {
147 area
= kmalloc_node(size
, GFP_USER
| flags
, numa_node
);
152 return __vmalloc_node_flags_caller(size
, numa_node
, GFP_KERNEL
| flags
,
153 __builtin_return_address(0));
156 void bpf_map_area_free(void *area
)
161 void bpf_map_init_from_attr(struct bpf_map
*map
, union bpf_attr
*attr
)
163 map
->map_type
= attr
->map_type
;
164 map
->key_size
= attr
->key_size
;
165 map
->value_size
= attr
->value_size
;
166 map
->max_entries
= attr
->max_entries
;
167 map
->map_flags
= attr
->map_flags
;
168 map
->numa_node
= bpf_map_attr_numa_node(attr
);
171 int bpf_map_precharge_memlock(u32 pages
)
173 struct user_struct
*user
= get_current_user();
174 unsigned long memlock_limit
, cur
;
176 memlock_limit
= rlimit(RLIMIT_MEMLOCK
) >> PAGE_SHIFT
;
177 cur
= atomic_long_read(&user
->locked_vm
);
179 if (cur
+ pages
> memlock_limit
)
184 static int bpf_charge_memlock(struct user_struct
*user
, u32 pages
)
186 unsigned long memlock_limit
= rlimit(RLIMIT_MEMLOCK
) >> PAGE_SHIFT
;
188 if (atomic_long_add_return(pages
, &user
->locked_vm
) > memlock_limit
) {
189 atomic_long_sub(pages
, &user
->locked_vm
);
195 static void bpf_uncharge_memlock(struct user_struct
*user
, u32 pages
)
197 atomic_long_sub(pages
, &user
->locked_vm
);
200 static int bpf_map_init_memlock(struct bpf_map
*map
)
202 struct user_struct
*user
= get_current_user();
205 ret
= bpf_charge_memlock(user
, map
->pages
);
214 static void bpf_map_release_memlock(struct bpf_map
*map
)
216 struct user_struct
*user
= map
->user
;
217 bpf_uncharge_memlock(user
, map
->pages
);
221 int bpf_map_charge_memlock(struct bpf_map
*map
, u32 pages
)
225 ret
= bpf_charge_memlock(map
->user
, pages
);
232 void bpf_map_uncharge_memlock(struct bpf_map
*map
, u32 pages
)
234 bpf_uncharge_memlock(map
->user
, pages
);
238 static int bpf_map_alloc_id(struct bpf_map
*map
)
242 idr_preload(GFP_KERNEL
);
243 spin_lock_bh(&map_idr_lock
);
244 id
= idr_alloc_cyclic(&map_idr
, map
, 1, INT_MAX
, GFP_ATOMIC
);
247 spin_unlock_bh(&map_idr_lock
);
250 if (WARN_ON_ONCE(!id
))
253 return id
> 0 ? 0 : id
;
256 void bpf_map_free_id(struct bpf_map
*map
, bool do_idr_lock
)
260 /* Offloaded maps are removed from the IDR store when their device
261 * disappears - even if someone holds an fd to them they are unusable,
262 * the memory is gone, all ops will fail; they are simply waiting for
263 * refcnt to drop to be freed.
269 spin_lock_irqsave(&map_idr_lock
, flags
);
271 __acquire(&map_idr_lock
);
273 idr_remove(&map_idr
, map
->id
);
277 spin_unlock_irqrestore(&map_idr_lock
, flags
);
279 __release(&map_idr_lock
);
282 /* called from workqueue */
283 static void bpf_map_free_deferred(struct work_struct
*work
)
285 struct bpf_map
*map
= container_of(work
, struct bpf_map
, work
);
287 bpf_map_release_memlock(map
);
288 security_bpf_map_free(map
);
289 /* implementation dependent freeing */
290 map
->ops
->map_free(map
);
293 static void bpf_map_put_uref(struct bpf_map
*map
)
295 if (atomic_dec_and_test(&map
->usercnt
)) {
296 if (map
->ops
->map_release_uref
)
297 map
->ops
->map_release_uref(map
);
301 /* decrement map refcnt and schedule it for freeing via workqueue
302 * (unrelying map implementation ops->map_free() might sleep)
304 static void __bpf_map_put(struct bpf_map
*map
, bool do_idr_lock
)
306 if (atomic_dec_and_test(&map
->refcnt
)) {
307 /* bpf_map_free_id() must be called first */
308 bpf_map_free_id(map
, do_idr_lock
);
310 INIT_WORK(&map
->work
, bpf_map_free_deferred
);
311 schedule_work(&map
->work
);
315 void bpf_map_put(struct bpf_map
*map
)
317 __bpf_map_put(map
, true);
319 EXPORT_SYMBOL_GPL(bpf_map_put
);
321 void bpf_map_put_with_uref(struct bpf_map
*map
)
323 bpf_map_put_uref(map
);
327 static int bpf_map_release(struct inode
*inode
, struct file
*filp
)
329 struct bpf_map
*map
= filp
->private_data
;
331 if (map
->ops
->map_release
)
332 map
->ops
->map_release(map
, filp
);
334 bpf_map_put_with_uref(map
);
338 #ifdef CONFIG_PROC_FS
339 static void bpf_map_show_fdinfo(struct seq_file
*m
, struct file
*filp
)
341 const struct bpf_map
*map
= filp
->private_data
;
342 const struct bpf_array
*array
;
343 u32 owner_prog_type
= 0;
346 if (map
->map_type
== BPF_MAP_TYPE_PROG_ARRAY
) {
347 array
= container_of(map
, struct bpf_array
, map
);
348 owner_prog_type
= array
->owner_prog_type
;
349 owner_jited
= array
->owner_jited
;
365 map
->pages
* 1ULL << PAGE_SHIFT
,
368 if (owner_prog_type
) {
369 seq_printf(m
, "owner_prog_type:\t%u\n",
371 seq_printf(m
, "owner_jited:\t%u\n",
377 static ssize_t
bpf_dummy_read(struct file
*filp
, char __user
*buf
, size_t siz
,
380 /* We need this handler such that alloc_file() enables
381 * f_mode with FMODE_CAN_READ.
386 static ssize_t
bpf_dummy_write(struct file
*filp
, const char __user
*buf
,
387 size_t siz
, loff_t
*ppos
)
389 /* We need this handler such that alloc_file() enables
390 * f_mode with FMODE_CAN_WRITE.
395 const struct file_operations bpf_map_fops
= {
396 #ifdef CONFIG_PROC_FS
397 .show_fdinfo
= bpf_map_show_fdinfo
,
399 .release
= bpf_map_release
,
400 .read
= bpf_dummy_read
,
401 .write
= bpf_dummy_write
,
404 int bpf_map_new_fd(struct bpf_map
*map
, int flags
)
408 ret
= security_bpf_map(map
, OPEN_FMODE(flags
));
412 return anon_inode_getfd("bpf-map", &bpf_map_fops
, map
,
416 int bpf_get_file_flag(int flags
)
418 if ((flags
& BPF_F_RDONLY
) && (flags
& BPF_F_WRONLY
))
420 if (flags
& BPF_F_RDONLY
)
422 if (flags
& BPF_F_WRONLY
)
427 /* helper macro to check that unused fields 'union bpf_attr' are zero */
428 #define CHECK_ATTR(CMD) \
429 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
430 sizeof(attr->CMD##_LAST_FIELD), 0, \
432 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
433 sizeof(attr->CMD##_LAST_FIELD)) != NULL
435 /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes.
436 * Return 0 on success and < 0 on error.
438 static int bpf_obj_name_cpy(char *dst
, const char *src
)
440 const char *end
= src
+ BPF_OBJ_NAME_LEN
;
442 memset(dst
, 0, BPF_OBJ_NAME_LEN
);
444 /* Copy all isalnum() and '_' char */
445 while (src
< end
&& *src
) {
446 if (!isalnum(*src
) && *src
!= '_')
451 /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */
458 int map_check_no_btf(const struct bpf_map
*map
,
459 const struct btf
*btf
,
460 const struct btf_type
*key_type
,
461 const struct btf_type
*value_type
)
466 static int map_check_btf(const struct bpf_map
*map
, const struct btf
*btf
,
467 u32 btf_key_id
, u32 btf_value_id
)
469 const struct btf_type
*key_type
, *value_type
;
470 u32 key_size
, value_size
;
473 key_type
= btf_type_id_size(btf
, &btf_key_id
, &key_size
);
474 if (!key_type
|| key_size
!= map
->key_size
)
477 value_type
= btf_type_id_size(btf
, &btf_value_id
, &value_size
);
478 if (!value_type
|| value_size
!= map
->value_size
)
481 if (map
->ops
->map_check_btf
)
482 ret
= map
->ops
->map_check_btf(map
, btf
, key_type
, value_type
);
487 #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id
488 /* called via syscall */
489 static int map_create(union bpf_attr
*attr
)
491 int numa_node
= bpf_map_attr_numa_node(attr
);
496 err
= CHECK_ATTR(BPF_MAP_CREATE
);
500 f_flags
= bpf_get_file_flag(attr
->map_flags
);
504 if (numa_node
!= NUMA_NO_NODE
&&
505 ((unsigned int)numa_node
>= nr_node_ids
||
506 !node_online(numa_node
)))
509 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
510 map
= find_and_alloc_map(attr
);
514 err
= bpf_obj_name_cpy(map
->name
, attr
->map_name
);
516 goto free_map_nouncharge
;
518 atomic_set(&map
->refcnt
, 1);
519 atomic_set(&map
->usercnt
, 1);
521 if (attr
->btf_key_type_id
|| attr
->btf_value_type_id
) {
524 if (!attr
->btf_key_type_id
|| !attr
->btf_value_type_id
) {
526 goto free_map_nouncharge
;
529 btf
= btf_get_by_fd(attr
->btf_fd
);
532 goto free_map_nouncharge
;
535 err
= map_check_btf(map
, btf
, attr
->btf_key_type_id
,
536 attr
->btf_value_type_id
);
539 goto free_map_nouncharge
;
543 map
->btf_key_type_id
= attr
->btf_key_type_id
;
544 map
->btf_value_type_id
= attr
->btf_value_type_id
;
547 err
= security_bpf_map_alloc(map
);
549 goto free_map_nouncharge
;
551 err
= bpf_map_init_memlock(map
);
555 err
= bpf_map_alloc_id(map
);
559 err
= bpf_map_new_fd(map
, f_flags
);
561 /* failed to allocate fd.
562 * bpf_map_put() is needed because the above
563 * bpf_map_alloc_id() has published the map
564 * to the userspace and the userspace may
565 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
574 bpf_map_release_memlock(map
);
576 security_bpf_map_free(map
);
579 map
->ops
->map_free(map
);
583 /* if error is returned, fd is released.
584 * On success caller should complete fd access with matching fdput()
586 struct bpf_map
*__bpf_map_get(struct fd f
)
589 return ERR_PTR(-EBADF
);
590 if (f
.file
->f_op
!= &bpf_map_fops
) {
592 return ERR_PTR(-EINVAL
);
595 return f
.file
->private_data
;
598 /* prog's and map's refcnt limit */
599 #define BPF_MAX_REFCNT 32768
601 struct bpf_map
*bpf_map_inc(struct bpf_map
*map
, bool uref
)
603 if (atomic_inc_return(&map
->refcnt
) > BPF_MAX_REFCNT
) {
604 atomic_dec(&map
->refcnt
);
605 return ERR_PTR(-EBUSY
);
608 atomic_inc(&map
->usercnt
);
611 EXPORT_SYMBOL_GPL(bpf_map_inc
);
613 struct bpf_map
*bpf_map_get_with_uref(u32 ufd
)
615 struct fd f
= fdget(ufd
);
618 map
= __bpf_map_get(f
);
622 map
= bpf_map_inc(map
, true);
628 /* map_idr_lock should have been held */
629 static struct bpf_map
*bpf_map_inc_not_zero(struct bpf_map
*map
,
634 refold
= atomic_fetch_add_unless(&map
->refcnt
, 1, 0);
636 if (refold
>= BPF_MAX_REFCNT
) {
637 __bpf_map_put(map
, false);
638 return ERR_PTR(-EBUSY
);
642 return ERR_PTR(-ENOENT
);
645 atomic_inc(&map
->usercnt
);
650 int __weak
bpf_stackmap_copy(struct bpf_map
*map
, void *key
, void *value
)
655 static void *__bpf_copy_key(void __user
*ukey
, u64 key_size
)
658 return memdup_user(ukey
, key_size
);
661 return ERR_PTR(-EINVAL
);
666 /* last field in 'union bpf_attr' used by this command */
667 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
669 static int map_lookup_elem(union bpf_attr
*attr
)
671 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
672 void __user
*uvalue
= u64_to_user_ptr(attr
->value
);
673 int ufd
= attr
->map_fd
;
675 void *key
, *value
, *ptr
;
680 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM
))
684 map
= __bpf_map_get(f
);
688 if (!(f
.file
->f_mode
& FMODE_CAN_READ
)) {
693 key
= __bpf_copy_key(ukey
, map
->key_size
);
699 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
700 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
||
701 map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
||
702 map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
)
703 value_size
= round_up(map
->value_size
, 8) * num_possible_cpus();
704 else if (IS_FD_MAP(map
))
705 value_size
= sizeof(u32
);
707 value_size
= map
->value_size
;
710 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
714 if (bpf_map_is_dev_bound(map
)) {
715 err
= bpf_map_offload_lookup_elem(map
, key
, value
);
716 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
717 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
) {
718 err
= bpf_percpu_hash_copy(map
, key
, value
);
719 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
) {
720 err
= bpf_percpu_array_copy(map
, key
, value
);
721 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
) {
722 err
= bpf_percpu_cgroup_storage_copy(map
, key
, value
);
723 } else if (map
->map_type
== BPF_MAP_TYPE_STACK_TRACE
) {
724 err
= bpf_stackmap_copy(map
, key
, value
);
725 } else if (IS_FD_ARRAY(map
)) {
726 err
= bpf_fd_array_map_lookup_elem(map
, key
, value
);
727 } else if (IS_FD_HASH(map
)) {
728 err
= bpf_fd_htab_map_lookup_elem(map
, key
, value
);
729 } else if (map
->map_type
== BPF_MAP_TYPE_REUSEPORT_SOCKARRAY
) {
730 err
= bpf_fd_reuseport_array_lookup_elem(map
, key
, value
);
731 } else if (map
->map_type
== BPF_MAP_TYPE_QUEUE
||
732 map
->map_type
== BPF_MAP_TYPE_STACK
) {
733 err
= map
->ops
->map_peek_elem(map
, value
);
736 ptr
= map
->ops
->map_lookup_elem(map
, key
);
743 memcpy(value
, ptr
, value_size
);
752 if (copy_to_user(uvalue
, value
, value_size
) != 0)
766 static void maybe_wait_bpf_programs(struct bpf_map
*map
)
768 /* Wait for any running BPF programs to complete so that
769 * userspace, when we return to it, knows that all programs
770 * that could be running use the new map value.
772 if (map
->map_type
== BPF_MAP_TYPE_HASH_OF_MAPS
||
773 map
->map_type
== BPF_MAP_TYPE_ARRAY_OF_MAPS
)
777 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
779 static int map_update_elem(union bpf_attr
*attr
)
781 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
782 void __user
*uvalue
= u64_to_user_ptr(attr
->value
);
783 int ufd
= attr
->map_fd
;
790 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM
))
794 map
= __bpf_map_get(f
);
798 if (!(f
.file
->f_mode
& FMODE_CAN_WRITE
)) {
803 key
= __bpf_copy_key(ukey
, map
->key_size
);
809 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
810 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
||
811 map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
||
812 map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
)
813 value_size
= round_up(map
->value_size
, 8) * num_possible_cpus();
815 value_size
= map
->value_size
;
818 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
823 if (copy_from_user(value
, uvalue
, value_size
) != 0)
826 /* Need to create a kthread, thus must support schedule */
827 if (bpf_map_is_dev_bound(map
)) {
828 err
= bpf_map_offload_update_elem(map
, key
, value
, attr
->flags
);
830 } else if (map
->map_type
== BPF_MAP_TYPE_CPUMAP
||
831 map
->map_type
== BPF_MAP_TYPE_SOCKHASH
||
832 map
->map_type
== BPF_MAP_TYPE_SOCKMAP
) {
833 err
= map
->ops
->map_update_elem(map
, key
, value
, attr
->flags
);
837 /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
838 * inside bpf map update or delete otherwise deadlocks are possible
841 __this_cpu_inc(bpf_prog_active
);
842 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
843 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
) {
844 err
= bpf_percpu_hash_update(map
, key
, value
, attr
->flags
);
845 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
) {
846 err
= bpf_percpu_array_update(map
, key
, value
, attr
->flags
);
847 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
) {
848 err
= bpf_percpu_cgroup_storage_update(map
, key
, value
,
850 } else if (IS_FD_ARRAY(map
)) {
852 err
= bpf_fd_array_map_update_elem(map
, f
.file
, key
, value
,
855 } else if (map
->map_type
== BPF_MAP_TYPE_HASH_OF_MAPS
) {
857 err
= bpf_fd_htab_map_update_elem(map
, f
.file
, key
, value
,
860 } else if (map
->map_type
== BPF_MAP_TYPE_REUSEPORT_SOCKARRAY
) {
861 /* rcu_read_lock() is not needed */
862 err
= bpf_fd_reuseport_array_update_elem(map
, key
, value
,
864 } else if (map
->map_type
== BPF_MAP_TYPE_QUEUE
||
865 map
->map_type
== BPF_MAP_TYPE_STACK
) {
866 err
= map
->ops
->map_push_elem(map
, value
, attr
->flags
);
869 err
= map
->ops
->map_update_elem(map
, key
, value
, attr
->flags
);
872 __this_cpu_dec(bpf_prog_active
);
874 maybe_wait_bpf_programs(map
);
885 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
887 static int map_delete_elem(union bpf_attr
*attr
)
889 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
890 int ufd
= attr
->map_fd
;
896 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM
))
900 map
= __bpf_map_get(f
);
904 if (!(f
.file
->f_mode
& FMODE_CAN_WRITE
)) {
909 key
= __bpf_copy_key(ukey
, map
->key_size
);
915 if (bpf_map_is_dev_bound(map
)) {
916 err
= bpf_map_offload_delete_elem(map
, key
);
921 __this_cpu_inc(bpf_prog_active
);
923 err
= map
->ops
->map_delete_elem(map
, key
);
925 __this_cpu_dec(bpf_prog_active
);
927 maybe_wait_bpf_programs(map
);
935 /* last field in 'union bpf_attr' used by this command */
936 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
938 static int map_get_next_key(union bpf_attr
*attr
)
940 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
941 void __user
*unext_key
= u64_to_user_ptr(attr
->next_key
);
942 int ufd
= attr
->map_fd
;
944 void *key
, *next_key
;
948 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY
))
952 map
= __bpf_map_get(f
);
956 if (!(f
.file
->f_mode
& FMODE_CAN_READ
)) {
962 key
= __bpf_copy_key(ukey
, map
->key_size
);
972 next_key
= kmalloc(map
->key_size
, GFP_USER
);
976 if (bpf_map_is_dev_bound(map
)) {
977 err
= bpf_map_offload_get_next_key(map
, key
, next_key
);
982 err
= map
->ops
->map_get_next_key(map
, key
, next_key
);
989 if (copy_to_user(unext_key
, next_key
, map
->key_size
) != 0)
1003 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
1005 static int map_lookup_and_delete_elem(union bpf_attr
*attr
)
1007 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
1008 void __user
*uvalue
= u64_to_user_ptr(attr
->value
);
1009 int ufd
= attr
->map_fd
;
1010 struct bpf_map
*map
;
1016 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM
))
1020 map
= __bpf_map_get(f
);
1022 return PTR_ERR(map
);
1024 if (!(f
.file
->f_mode
& FMODE_CAN_WRITE
)) {
1029 key
= __bpf_copy_key(ukey
, map
->key_size
);
1035 value_size
= map
->value_size
;
1038 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
1042 if (map
->map_type
== BPF_MAP_TYPE_QUEUE
||
1043 map
->map_type
== BPF_MAP_TYPE_STACK
) {
1044 err
= map
->ops
->map_pop_elem(map
, value
);
1052 if (copy_to_user(uvalue
, value
, value_size
) != 0)
1066 static const struct bpf_prog_ops
* const bpf_prog_types
[] = {
1067 #define BPF_PROG_TYPE(_id, _name) \
1068 [_id] = & _name ## _prog_ops,
1069 #define BPF_MAP_TYPE(_id, _ops)
1070 #include <linux/bpf_types.h>
1071 #undef BPF_PROG_TYPE
1075 static int find_prog_type(enum bpf_prog_type type
, struct bpf_prog
*prog
)
1077 const struct bpf_prog_ops
*ops
;
1079 if (type
>= ARRAY_SIZE(bpf_prog_types
))
1081 type
= array_index_nospec(type
, ARRAY_SIZE(bpf_prog_types
));
1082 ops
= bpf_prog_types
[type
];
1086 if (!bpf_prog_is_dev_bound(prog
->aux
))
1087 prog
->aux
->ops
= ops
;
1089 prog
->aux
->ops
= &bpf_offload_prog_ops
;
1094 /* drop refcnt on maps used by eBPF program and free auxilary data */
1095 static void free_used_maps(struct bpf_prog_aux
*aux
)
1097 enum bpf_cgroup_storage_type stype
;
1100 for_each_cgroup_storage_type(stype
) {
1101 if (!aux
->cgroup_storage
[stype
])
1103 bpf_cgroup_storage_release(aux
->prog
,
1104 aux
->cgroup_storage
[stype
]);
1107 for (i
= 0; i
< aux
->used_map_cnt
; i
++)
1108 bpf_map_put(aux
->used_maps
[i
]);
1110 kfree(aux
->used_maps
);
1113 int __bpf_prog_charge(struct user_struct
*user
, u32 pages
)
1115 unsigned long memlock_limit
= rlimit(RLIMIT_MEMLOCK
) >> PAGE_SHIFT
;
1116 unsigned long user_bufs
;
1119 user_bufs
= atomic_long_add_return(pages
, &user
->locked_vm
);
1120 if (user_bufs
> memlock_limit
) {
1121 atomic_long_sub(pages
, &user
->locked_vm
);
1129 void __bpf_prog_uncharge(struct user_struct
*user
, u32 pages
)
1132 atomic_long_sub(pages
, &user
->locked_vm
);
1135 static int bpf_prog_charge_memlock(struct bpf_prog
*prog
)
1137 struct user_struct
*user
= get_current_user();
1140 ret
= __bpf_prog_charge(user
, prog
->pages
);
1146 prog
->aux
->user
= user
;
1150 static void bpf_prog_uncharge_memlock(struct bpf_prog
*prog
)
1152 struct user_struct
*user
= prog
->aux
->user
;
1154 __bpf_prog_uncharge(user
, prog
->pages
);
1158 static int bpf_prog_alloc_id(struct bpf_prog
*prog
)
1162 idr_preload(GFP_KERNEL
);
1163 spin_lock_bh(&prog_idr_lock
);
1164 id
= idr_alloc_cyclic(&prog_idr
, prog
, 1, INT_MAX
, GFP_ATOMIC
);
1167 spin_unlock_bh(&prog_idr_lock
);
1170 /* id is in [1, INT_MAX) */
1171 if (WARN_ON_ONCE(!id
))
1174 return id
> 0 ? 0 : id
;
1177 void bpf_prog_free_id(struct bpf_prog
*prog
, bool do_idr_lock
)
1179 /* cBPF to eBPF migrations are currently not in the idr store.
1180 * Offloaded programs are removed from the store when their device
1181 * disappears - even if someone grabs an fd to them they are unusable,
1182 * simply waiting for refcnt to drop to be freed.
1188 spin_lock_bh(&prog_idr_lock
);
1190 __acquire(&prog_idr_lock
);
1192 idr_remove(&prog_idr
, prog
->aux
->id
);
1196 spin_unlock_bh(&prog_idr_lock
);
1198 __release(&prog_idr_lock
);
1201 static void __bpf_prog_put_rcu(struct rcu_head
*rcu
)
1203 struct bpf_prog_aux
*aux
= container_of(rcu
, struct bpf_prog_aux
, rcu
);
1205 free_used_maps(aux
);
1206 bpf_prog_uncharge_memlock(aux
->prog
);
1207 security_bpf_prog_free(aux
);
1208 bpf_prog_free(aux
->prog
);
1211 static void __bpf_prog_put(struct bpf_prog
*prog
, bool do_idr_lock
)
1213 if (atomic_dec_and_test(&prog
->aux
->refcnt
)) {
1214 /* bpf_prog_free_id() must be called first */
1215 bpf_prog_free_id(prog
, do_idr_lock
);
1216 bpf_prog_kallsyms_del_all(prog
);
1217 btf_put(prog
->aux
->btf
);
1218 kvfree(prog
->aux
->func_info
);
1219 bpf_prog_free_linfo(prog
);
1221 call_rcu(&prog
->aux
->rcu
, __bpf_prog_put_rcu
);
1225 void bpf_prog_put(struct bpf_prog
*prog
)
1227 __bpf_prog_put(prog
, true);
1229 EXPORT_SYMBOL_GPL(bpf_prog_put
);
1231 static int bpf_prog_release(struct inode
*inode
, struct file
*filp
)
1233 struct bpf_prog
*prog
= filp
->private_data
;
1239 #ifdef CONFIG_PROC_FS
1240 static void bpf_prog_show_fdinfo(struct seq_file
*m
, struct file
*filp
)
1242 const struct bpf_prog
*prog
= filp
->private_data
;
1243 char prog_tag
[sizeof(prog
->tag
) * 2 + 1] = { };
1245 bin2hex(prog_tag
, prog
->tag
, sizeof(prog
->tag
));
1255 prog
->pages
* 1ULL << PAGE_SHIFT
,
1260 const struct file_operations bpf_prog_fops
= {
1261 #ifdef CONFIG_PROC_FS
1262 .show_fdinfo
= bpf_prog_show_fdinfo
,
1264 .release
= bpf_prog_release
,
1265 .read
= bpf_dummy_read
,
1266 .write
= bpf_dummy_write
,
1269 int bpf_prog_new_fd(struct bpf_prog
*prog
)
1273 ret
= security_bpf_prog(prog
);
1277 return anon_inode_getfd("bpf-prog", &bpf_prog_fops
, prog
,
1278 O_RDWR
| O_CLOEXEC
);
1281 static struct bpf_prog
*____bpf_prog_get(struct fd f
)
1284 return ERR_PTR(-EBADF
);
1285 if (f
.file
->f_op
!= &bpf_prog_fops
) {
1287 return ERR_PTR(-EINVAL
);
1290 return f
.file
->private_data
;
1293 struct bpf_prog
*bpf_prog_add(struct bpf_prog
*prog
, int i
)
1295 if (atomic_add_return(i
, &prog
->aux
->refcnt
) > BPF_MAX_REFCNT
) {
1296 atomic_sub(i
, &prog
->aux
->refcnt
);
1297 return ERR_PTR(-EBUSY
);
1301 EXPORT_SYMBOL_GPL(bpf_prog_add
);
1303 void bpf_prog_sub(struct bpf_prog
*prog
, int i
)
1305 /* Only to be used for undoing previous bpf_prog_add() in some
1306 * error path. We still know that another entity in our call
1307 * path holds a reference to the program, thus atomic_sub() can
1308 * be safely used in such cases!
1310 WARN_ON(atomic_sub_return(i
, &prog
->aux
->refcnt
) == 0);
1312 EXPORT_SYMBOL_GPL(bpf_prog_sub
);
1314 struct bpf_prog
*bpf_prog_inc(struct bpf_prog
*prog
)
1316 return bpf_prog_add(prog
, 1);
1318 EXPORT_SYMBOL_GPL(bpf_prog_inc
);
1320 /* prog_idr_lock should have been held */
1321 struct bpf_prog
*bpf_prog_inc_not_zero(struct bpf_prog
*prog
)
1325 refold
= atomic_fetch_add_unless(&prog
->aux
->refcnt
, 1, 0);
1327 if (refold
>= BPF_MAX_REFCNT
) {
1328 __bpf_prog_put(prog
, false);
1329 return ERR_PTR(-EBUSY
);
1333 return ERR_PTR(-ENOENT
);
1337 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero
);
1339 bool bpf_prog_get_ok(struct bpf_prog
*prog
,
1340 enum bpf_prog_type
*attach_type
, bool attach_drv
)
1342 /* not an attachment, just a refcount inc, always allow */
1346 if (prog
->type
!= *attach_type
)
1348 if (bpf_prog_is_dev_bound(prog
->aux
) && !attach_drv
)
1354 static struct bpf_prog
*__bpf_prog_get(u32 ufd
, enum bpf_prog_type
*attach_type
,
1357 struct fd f
= fdget(ufd
);
1358 struct bpf_prog
*prog
;
1360 prog
= ____bpf_prog_get(f
);
1363 if (!bpf_prog_get_ok(prog
, attach_type
, attach_drv
)) {
1364 prog
= ERR_PTR(-EINVAL
);
1368 prog
= bpf_prog_inc(prog
);
1374 struct bpf_prog
*bpf_prog_get(u32 ufd
)
1376 return __bpf_prog_get(ufd
, NULL
, false);
1379 struct bpf_prog
*bpf_prog_get_type_dev(u32 ufd
, enum bpf_prog_type type
,
1382 return __bpf_prog_get(ufd
, &type
, attach_drv
);
1384 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev
);
1386 /* Initially all BPF programs could be loaded w/o specifying
1387 * expected_attach_type. Later for some of them specifying expected_attach_type
1388 * at load time became required so that program could be validated properly.
1389 * Programs of types that are allowed to be loaded both w/ and w/o (for
1390 * backward compatibility) expected_attach_type, should have the default attach
1391 * type assigned to expected_attach_type for the latter case, so that it can be
1392 * validated later at attach time.
1394 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1395 * prog type requires it but has some attach types that have to be backward
1398 static void bpf_prog_load_fixup_attach_type(union bpf_attr
*attr
)
1400 switch (attr
->prog_type
) {
1401 case BPF_PROG_TYPE_CGROUP_SOCK
:
1402 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
1403 * exist so checking for non-zero is the way to go here.
1405 if (!attr
->expected_attach_type
)
1406 attr
->expected_attach_type
=
1407 BPF_CGROUP_INET_SOCK_CREATE
;
1413 bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type
,
1414 enum bpf_attach_type expected_attach_type
)
1416 switch (prog_type
) {
1417 case BPF_PROG_TYPE_CGROUP_SOCK
:
1418 switch (expected_attach_type
) {
1419 case BPF_CGROUP_INET_SOCK_CREATE
:
1420 case BPF_CGROUP_INET4_POST_BIND
:
1421 case BPF_CGROUP_INET6_POST_BIND
:
1426 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
1427 switch (expected_attach_type
) {
1428 case BPF_CGROUP_INET4_BIND
:
1429 case BPF_CGROUP_INET6_BIND
:
1430 case BPF_CGROUP_INET4_CONNECT
:
1431 case BPF_CGROUP_INET6_CONNECT
:
1432 case BPF_CGROUP_UDP4_SENDMSG
:
1433 case BPF_CGROUP_UDP6_SENDMSG
:
1443 /* last field in 'union bpf_attr' used by this command */
1444 #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
1446 static int bpf_prog_load(union bpf_attr
*attr
, union bpf_attr __user
*uattr
)
1448 enum bpf_prog_type type
= attr
->prog_type
;
1449 struct bpf_prog
*prog
;
1454 if (CHECK_ATTR(BPF_PROG_LOAD
))
1457 if (attr
->prog_flags
& ~(BPF_F_STRICT_ALIGNMENT
| BPF_F_ANY_ALIGNMENT
))
1460 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
) &&
1461 (attr
->prog_flags
& BPF_F_ANY_ALIGNMENT
) &&
1462 !capable(CAP_SYS_ADMIN
))
1465 /* copy eBPF program license from user space */
1466 if (strncpy_from_user(license
, u64_to_user_ptr(attr
->license
),
1467 sizeof(license
) - 1) < 0)
1469 license
[sizeof(license
) - 1] = 0;
1471 /* eBPF programs must be GPL compatible to use GPL-ed functions */
1472 is_gpl
= license_is_gpl_compatible(license
);
1474 if (attr
->insn_cnt
== 0 || attr
->insn_cnt
> BPF_MAXINSNS
)
1476 if (type
!= BPF_PROG_TYPE_SOCKET_FILTER
&&
1477 type
!= BPF_PROG_TYPE_CGROUP_SKB
&&
1478 !capable(CAP_SYS_ADMIN
))
1481 bpf_prog_load_fixup_attach_type(attr
);
1482 if (bpf_prog_load_check_attach_type(type
, attr
->expected_attach_type
))
1485 /* plain bpf_prog allocation */
1486 prog
= bpf_prog_alloc(bpf_prog_size(attr
->insn_cnt
), GFP_USER
);
1490 prog
->expected_attach_type
= attr
->expected_attach_type
;
1492 prog
->aux
->offload_requested
= !!attr
->prog_ifindex
;
1494 err
= security_bpf_prog_alloc(prog
->aux
);
1496 goto free_prog_nouncharge
;
1498 err
= bpf_prog_charge_memlock(prog
);
1502 prog
->len
= attr
->insn_cnt
;
1505 if (copy_from_user(prog
->insns
, u64_to_user_ptr(attr
->insns
),
1506 bpf_prog_insn_size(prog
)) != 0)
1509 prog
->orig_prog
= NULL
;
1512 atomic_set(&prog
->aux
->refcnt
, 1);
1513 prog
->gpl_compatible
= is_gpl
? 1 : 0;
1515 if (bpf_prog_is_dev_bound(prog
->aux
)) {
1516 err
= bpf_prog_offload_init(prog
, attr
);
1521 /* find program type: socket_filter vs tracing_filter */
1522 err
= find_prog_type(type
, prog
);
1526 prog
->aux
->load_time
= ktime_get_boot_ns();
1527 err
= bpf_obj_name_cpy(prog
->aux
->name
, attr
->prog_name
);
1531 /* run eBPF verifier */
1532 err
= bpf_check(&prog
, attr
, uattr
);
1534 goto free_used_maps
;
1536 prog
= bpf_prog_select_runtime(prog
, &err
);
1538 goto free_used_maps
;
1540 err
= bpf_prog_alloc_id(prog
);
1542 goto free_used_maps
;
1544 err
= bpf_prog_new_fd(prog
);
1546 /* failed to allocate fd.
1547 * bpf_prog_put() is needed because the above
1548 * bpf_prog_alloc_id() has published the prog
1549 * to the userspace and the userspace may
1550 * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID.
1556 bpf_prog_kallsyms_add(prog
);
1560 bpf_prog_free_linfo(prog
);
1561 kvfree(prog
->aux
->func_info
);
1562 btf_put(prog
->aux
->btf
);
1563 bpf_prog_kallsyms_del_subprogs(prog
);
1564 free_used_maps(prog
->aux
);
1566 bpf_prog_uncharge_memlock(prog
);
1568 security_bpf_prog_free(prog
->aux
);
1569 free_prog_nouncharge
:
1570 bpf_prog_free(prog
);
1574 #define BPF_OBJ_LAST_FIELD file_flags
1576 static int bpf_obj_pin(const union bpf_attr
*attr
)
1578 if (CHECK_ATTR(BPF_OBJ
) || attr
->file_flags
!= 0)
1581 return bpf_obj_pin_user(attr
->bpf_fd
, u64_to_user_ptr(attr
->pathname
));
1584 static int bpf_obj_get(const union bpf_attr
*attr
)
1586 if (CHECK_ATTR(BPF_OBJ
) || attr
->bpf_fd
!= 0 ||
1587 attr
->file_flags
& ~BPF_OBJ_FLAG_MASK
)
1590 return bpf_obj_get_user(u64_to_user_ptr(attr
->pathname
),
1594 struct bpf_raw_tracepoint
{
1595 struct bpf_raw_event_map
*btp
;
1596 struct bpf_prog
*prog
;
1599 static int bpf_raw_tracepoint_release(struct inode
*inode
, struct file
*filp
)
1601 struct bpf_raw_tracepoint
*raw_tp
= filp
->private_data
;
1604 bpf_probe_unregister(raw_tp
->btp
, raw_tp
->prog
);
1605 bpf_prog_put(raw_tp
->prog
);
1607 bpf_put_raw_tracepoint(raw_tp
->btp
);
1612 static const struct file_operations bpf_raw_tp_fops
= {
1613 .release
= bpf_raw_tracepoint_release
,
1614 .read
= bpf_dummy_read
,
1615 .write
= bpf_dummy_write
,
1618 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
1620 static int bpf_raw_tracepoint_open(const union bpf_attr
*attr
)
1622 struct bpf_raw_tracepoint
*raw_tp
;
1623 struct bpf_raw_event_map
*btp
;
1624 struct bpf_prog
*prog
;
1628 if (strncpy_from_user(tp_name
, u64_to_user_ptr(attr
->raw_tracepoint
.name
),
1629 sizeof(tp_name
) - 1) < 0)
1631 tp_name
[sizeof(tp_name
) - 1] = 0;
1633 btp
= bpf_get_raw_tracepoint(tp_name
);
1637 raw_tp
= kzalloc(sizeof(*raw_tp
), GFP_USER
);
1644 prog
= bpf_prog_get_type(attr
->raw_tracepoint
.prog_fd
,
1645 BPF_PROG_TYPE_RAW_TRACEPOINT
);
1647 err
= PTR_ERR(prog
);
1651 err
= bpf_probe_register(raw_tp
->btp
, prog
);
1655 raw_tp
->prog
= prog
;
1656 tp_fd
= anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops
, raw_tp
,
1659 bpf_probe_unregister(raw_tp
->btp
, prog
);
1670 bpf_put_raw_tracepoint(btp
);
1674 static int bpf_prog_attach_check_attach_type(const struct bpf_prog
*prog
,
1675 enum bpf_attach_type attach_type
)
1677 switch (prog
->type
) {
1678 case BPF_PROG_TYPE_CGROUP_SOCK
:
1679 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
1680 return attach_type
== prog
->expected_attach_type
? 0 : -EINVAL
;
1686 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
1688 #define BPF_F_ATTACH_MASK \
1689 (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
1691 static int bpf_prog_attach(const union bpf_attr
*attr
)
1693 enum bpf_prog_type ptype
;
1694 struct bpf_prog
*prog
;
1697 if (!capable(CAP_NET_ADMIN
))
1700 if (CHECK_ATTR(BPF_PROG_ATTACH
))
1703 if (attr
->attach_flags
& ~BPF_F_ATTACH_MASK
)
1706 switch (attr
->attach_type
) {
1707 case BPF_CGROUP_INET_INGRESS
:
1708 case BPF_CGROUP_INET_EGRESS
:
1709 ptype
= BPF_PROG_TYPE_CGROUP_SKB
;
1711 case BPF_CGROUP_INET_SOCK_CREATE
:
1712 case BPF_CGROUP_INET4_POST_BIND
:
1713 case BPF_CGROUP_INET6_POST_BIND
:
1714 ptype
= BPF_PROG_TYPE_CGROUP_SOCK
;
1716 case BPF_CGROUP_INET4_BIND
:
1717 case BPF_CGROUP_INET6_BIND
:
1718 case BPF_CGROUP_INET4_CONNECT
:
1719 case BPF_CGROUP_INET6_CONNECT
:
1720 case BPF_CGROUP_UDP4_SENDMSG
:
1721 case BPF_CGROUP_UDP6_SENDMSG
:
1722 ptype
= BPF_PROG_TYPE_CGROUP_SOCK_ADDR
;
1724 case BPF_CGROUP_SOCK_OPS
:
1725 ptype
= BPF_PROG_TYPE_SOCK_OPS
;
1727 case BPF_CGROUP_DEVICE
:
1728 ptype
= BPF_PROG_TYPE_CGROUP_DEVICE
;
1730 case BPF_SK_MSG_VERDICT
:
1731 ptype
= BPF_PROG_TYPE_SK_MSG
;
1733 case BPF_SK_SKB_STREAM_PARSER
:
1734 case BPF_SK_SKB_STREAM_VERDICT
:
1735 ptype
= BPF_PROG_TYPE_SK_SKB
;
1737 case BPF_LIRC_MODE2
:
1738 ptype
= BPF_PROG_TYPE_LIRC_MODE2
;
1740 case BPF_FLOW_DISSECTOR
:
1741 ptype
= BPF_PROG_TYPE_FLOW_DISSECTOR
;
1747 prog
= bpf_prog_get_type(attr
->attach_bpf_fd
, ptype
);
1749 return PTR_ERR(prog
);
1751 if (bpf_prog_attach_check_attach_type(prog
, attr
->attach_type
)) {
1757 case BPF_PROG_TYPE_SK_SKB
:
1758 case BPF_PROG_TYPE_SK_MSG
:
1759 ret
= sock_map_get_from_fd(attr
, prog
);
1761 case BPF_PROG_TYPE_LIRC_MODE2
:
1762 ret
= lirc_prog_attach(attr
, prog
);
1764 case BPF_PROG_TYPE_FLOW_DISSECTOR
:
1765 ret
= skb_flow_dissector_bpf_prog_attach(attr
, prog
);
1768 ret
= cgroup_bpf_prog_attach(attr
, ptype
, prog
);
1776 #define BPF_PROG_DETACH_LAST_FIELD attach_type
1778 static int bpf_prog_detach(const union bpf_attr
*attr
)
1780 enum bpf_prog_type ptype
;
1782 if (!capable(CAP_NET_ADMIN
))
1785 if (CHECK_ATTR(BPF_PROG_DETACH
))
1788 switch (attr
->attach_type
) {
1789 case BPF_CGROUP_INET_INGRESS
:
1790 case BPF_CGROUP_INET_EGRESS
:
1791 ptype
= BPF_PROG_TYPE_CGROUP_SKB
;
1793 case BPF_CGROUP_INET_SOCK_CREATE
:
1794 case BPF_CGROUP_INET4_POST_BIND
:
1795 case BPF_CGROUP_INET6_POST_BIND
:
1796 ptype
= BPF_PROG_TYPE_CGROUP_SOCK
;
1798 case BPF_CGROUP_INET4_BIND
:
1799 case BPF_CGROUP_INET6_BIND
:
1800 case BPF_CGROUP_INET4_CONNECT
:
1801 case BPF_CGROUP_INET6_CONNECT
:
1802 case BPF_CGROUP_UDP4_SENDMSG
:
1803 case BPF_CGROUP_UDP6_SENDMSG
:
1804 ptype
= BPF_PROG_TYPE_CGROUP_SOCK_ADDR
;
1806 case BPF_CGROUP_SOCK_OPS
:
1807 ptype
= BPF_PROG_TYPE_SOCK_OPS
;
1809 case BPF_CGROUP_DEVICE
:
1810 ptype
= BPF_PROG_TYPE_CGROUP_DEVICE
;
1812 case BPF_SK_MSG_VERDICT
:
1813 return sock_map_get_from_fd(attr
, NULL
);
1814 case BPF_SK_SKB_STREAM_PARSER
:
1815 case BPF_SK_SKB_STREAM_VERDICT
:
1816 return sock_map_get_from_fd(attr
, NULL
);
1817 case BPF_LIRC_MODE2
:
1818 return lirc_prog_detach(attr
);
1819 case BPF_FLOW_DISSECTOR
:
1820 return skb_flow_dissector_bpf_prog_detach(attr
);
1825 return cgroup_bpf_prog_detach(attr
, ptype
);
1828 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
1830 static int bpf_prog_query(const union bpf_attr
*attr
,
1831 union bpf_attr __user
*uattr
)
1833 if (!capable(CAP_NET_ADMIN
))
1835 if (CHECK_ATTR(BPF_PROG_QUERY
))
1837 if (attr
->query
.query_flags
& ~BPF_F_QUERY_EFFECTIVE
)
1840 switch (attr
->query
.attach_type
) {
1841 case BPF_CGROUP_INET_INGRESS
:
1842 case BPF_CGROUP_INET_EGRESS
:
1843 case BPF_CGROUP_INET_SOCK_CREATE
:
1844 case BPF_CGROUP_INET4_BIND
:
1845 case BPF_CGROUP_INET6_BIND
:
1846 case BPF_CGROUP_INET4_POST_BIND
:
1847 case BPF_CGROUP_INET6_POST_BIND
:
1848 case BPF_CGROUP_INET4_CONNECT
:
1849 case BPF_CGROUP_INET6_CONNECT
:
1850 case BPF_CGROUP_UDP4_SENDMSG
:
1851 case BPF_CGROUP_UDP6_SENDMSG
:
1852 case BPF_CGROUP_SOCK_OPS
:
1853 case BPF_CGROUP_DEVICE
:
1855 case BPF_LIRC_MODE2
:
1856 return lirc_prog_query(attr
, uattr
);
1861 return cgroup_bpf_prog_query(attr
, uattr
);
1864 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
1866 static int bpf_prog_test_run(const union bpf_attr
*attr
,
1867 union bpf_attr __user
*uattr
)
1869 struct bpf_prog
*prog
;
1870 int ret
= -ENOTSUPP
;
1872 if (!capable(CAP_SYS_ADMIN
))
1874 if (CHECK_ATTR(BPF_PROG_TEST_RUN
))
1877 prog
= bpf_prog_get(attr
->test
.prog_fd
);
1879 return PTR_ERR(prog
);
1881 if (prog
->aux
->ops
->test_run
)
1882 ret
= prog
->aux
->ops
->test_run(prog
, attr
, uattr
);
1888 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
1890 static int bpf_obj_get_next_id(const union bpf_attr
*attr
,
1891 union bpf_attr __user
*uattr
,
1895 u32 next_id
= attr
->start_id
;
1898 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID
) || next_id
>= INT_MAX
)
1901 if (!capable(CAP_SYS_ADMIN
))
1906 if (!idr_get_next(idr
, &next_id
))
1908 spin_unlock_bh(lock
);
1911 err
= put_user(next_id
, &uattr
->next_id
);
1916 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
1918 static int bpf_prog_get_fd_by_id(const union bpf_attr
*attr
)
1920 struct bpf_prog
*prog
;
1921 u32 id
= attr
->prog_id
;
1924 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID
))
1927 if (!capable(CAP_SYS_ADMIN
))
1930 spin_lock_bh(&prog_idr_lock
);
1931 prog
= idr_find(&prog_idr
, id
);
1933 prog
= bpf_prog_inc_not_zero(prog
);
1935 prog
= ERR_PTR(-ENOENT
);
1936 spin_unlock_bh(&prog_idr_lock
);
1939 return PTR_ERR(prog
);
1941 fd
= bpf_prog_new_fd(prog
);
1948 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
1950 static int bpf_map_get_fd_by_id(const union bpf_attr
*attr
)
1952 struct bpf_map
*map
;
1953 u32 id
= attr
->map_id
;
1957 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID
) ||
1958 attr
->open_flags
& ~BPF_OBJ_FLAG_MASK
)
1961 if (!capable(CAP_SYS_ADMIN
))
1964 f_flags
= bpf_get_file_flag(attr
->open_flags
);
1968 spin_lock_bh(&map_idr_lock
);
1969 map
= idr_find(&map_idr
, id
);
1971 map
= bpf_map_inc_not_zero(map
, true);
1973 map
= ERR_PTR(-ENOENT
);
1974 spin_unlock_bh(&map_idr_lock
);
1977 return PTR_ERR(map
);
1979 fd
= bpf_map_new_fd(map
, f_flags
);
1986 static const struct bpf_map
*bpf_map_from_imm(const struct bpf_prog
*prog
,
1991 for (i
= 0; i
< prog
->aux
->used_map_cnt
; i
++)
1992 if (prog
->aux
->used_maps
[i
] == (void *)addr
)
1993 return prog
->aux
->used_maps
[i
];
1997 static struct bpf_insn
*bpf_insn_prepare_dump(const struct bpf_prog
*prog
)
1999 const struct bpf_map
*map
;
2000 struct bpf_insn
*insns
;
2004 insns
= kmemdup(prog
->insnsi
, bpf_prog_insn_size(prog
),
2009 for (i
= 0; i
< prog
->len
; i
++) {
2010 if (insns
[i
].code
== (BPF_JMP
| BPF_TAIL_CALL
)) {
2011 insns
[i
].code
= BPF_JMP
| BPF_CALL
;
2012 insns
[i
].imm
= BPF_FUNC_tail_call
;
2015 if (insns
[i
].code
== (BPF_JMP
| BPF_CALL
) ||
2016 insns
[i
].code
== (BPF_JMP
| BPF_CALL_ARGS
)) {
2017 if (insns
[i
].code
== (BPF_JMP
| BPF_CALL_ARGS
))
2018 insns
[i
].code
= BPF_JMP
| BPF_CALL
;
2019 if (!bpf_dump_raw_ok())
2024 if (insns
[i
].code
!= (BPF_LD
| BPF_IMM
| BPF_DW
))
2027 imm
= ((u64
)insns
[i
+ 1].imm
<< 32) | (u32
)insns
[i
].imm
;
2028 map
= bpf_map_from_imm(prog
, imm
);
2030 insns
[i
].src_reg
= BPF_PSEUDO_MAP_FD
;
2031 insns
[i
].imm
= map
->id
;
2032 insns
[i
+ 1].imm
= 0;
2040 static int set_info_rec_size(struct bpf_prog_info
*info
)
2043 * Ensure info.*_rec_size is the same as kernel expected size
2047 * Only allow zero *_rec_size if both _rec_size and _cnt are
2048 * zero. In this case, the kernel will set the expected
2049 * _rec_size back to the info.
2052 if ((info
->nr_func_info
|| info
->func_info_rec_size
) &&
2053 info
->func_info_rec_size
!= sizeof(struct bpf_func_info
))
2056 if ((info
->nr_line_info
|| info
->line_info_rec_size
) &&
2057 info
->line_info_rec_size
!= sizeof(struct bpf_line_info
))
2060 if ((info
->nr_jited_line_info
|| info
->jited_line_info_rec_size
) &&
2061 info
->jited_line_info_rec_size
!= sizeof(__u64
))
2064 info
->func_info_rec_size
= sizeof(struct bpf_func_info
);
2065 info
->line_info_rec_size
= sizeof(struct bpf_line_info
);
2066 info
->jited_line_info_rec_size
= sizeof(__u64
);
2071 static int bpf_prog_get_info_by_fd(struct bpf_prog
*prog
,
2072 const union bpf_attr
*attr
,
2073 union bpf_attr __user
*uattr
)
2075 struct bpf_prog_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
2076 struct bpf_prog_info info
= {};
2077 u32 info_len
= attr
->info
.info_len
;
2078 char __user
*uinsns
;
2082 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(info
), info_len
);
2085 info_len
= min_t(u32
, sizeof(info
), info_len
);
2087 if (copy_from_user(&info
, uinfo
, info_len
))
2090 info
.type
= prog
->type
;
2091 info
.id
= prog
->aux
->id
;
2092 info
.load_time
= prog
->aux
->load_time
;
2093 info
.created_by_uid
= from_kuid_munged(current_user_ns(),
2094 prog
->aux
->user
->uid
);
2095 info
.gpl_compatible
= prog
->gpl_compatible
;
2097 memcpy(info
.tag
, prog
->tag
, sizeof(prog
->tag
));
2098 memcpy(info
.name
, prog
->aux
->name
, sizeof(prog
->aux
->name
));
2100 ulen
= info
.nr_map_ids
;
2101 info
.nr_map_ids
= prog
->aux
->used_map_cnt
;
2102 ulen
= min_t(u32
, info
.nr_map_ids
, ulen
);
2104 u32 __user
*user_map_ids
= u64_to_user_ptr(info
.map_ids
);
2107 for (i
= 0; i
< ulen
; i
++)
2108 if (put_user(prog
->aux
->used_maps
[i
]->id
,
2113 err
= set_info_rec_size(&info
);
2117 if (!capable(CAP_SYS_ADMIN
)) {
2118 info
.jited_prog_len
= 0;
2119 info
.xlated_prog_len
= 0;
2120 info
.nr_jited_ksyms
= 0;
2121 info
.nr_jited_func_lens
= 0;
2122 info
.nr_func_info
= 0;
2123 info
.nr_line_info
= 0;
2124 info
.nr_jited_line_info
= 0;
2128 ulen
= info
.xlated_prog_len
;
2129 info
.xlated_prog_len
= bpf_prog_insn_size(prog
);
2130 if (info
.xlated_prog_len
&& ulen
) {
2131 struct bpf_insn
*insns_sanitized
;
2134 if (prog
->blinded
&& !bpf_dump_raw_ok()) {
2135 info
.xlated_prog_insns
= 0;
2138 insns_sanitized
= bpf_insn_prepare_dump(prog
);
2139 if (!insns_sanitized
)
2141 uinsns
= u64_to_user_ptr(info
.xlated_prog_insns
);
2142 ulen
= min_t(u32
, info
.xlated_prog_len
, ulen
);
2143 fault
= copy_to_user(uinsns
, insns_sanitized
, ulen
);
2144 kfree(insns_sanitized
);
2149 if (bpf_prog_is_dev_bound(prog
->aux
)) {
2150 err
= bpf_prog_offload_info_fill(&info
, prog
);
2156 /* NOTE: the following code is supposed to be skipped for offload.
2157 * bpf_prog_offload_info_fill() is the place to fill similar fields
2160 ulen
= info
.jited_prog_len
;
2161 if (prog
->aux
->func_cnt
) {
2164 info
.jited_prog_len
= 0;
2165 for (i
= 0; i
< prog
->aux
->func_cnt
; i
++)
2166 info
.jited_prog_len
+= prog
->aux
->func
[i
]->jited_len
;
2168 info
.jited_prog_len
= prog
->jited_len
;
2171 if (info
.jited_prog_len
&& ulen
) {
2172 if (bpf_dump_raw_ok()) {
2173 uinsns
= u64_to_user_ptr(info
.jited_prog_insns
);
2174 ulen
= min_t(u32
, info
.jited_prog_len
, ulen
);
2176 /* for multi-function programs, copy the JITed
2177 * instructions for all the functions
2179 if (prog
->aux
->func_cnt
) {
2184 for (i
= 0; i
< prog
->aux
->func_cnt
; i
++) {
2185 len
= prog
->aux
->func
[i
]->jited_len
;
2186 len
= min_t(u32
, len
, free
);
2187 img
= (u8
*) prog
->aux
->func
[i
]->bpf_func
;
2188 if (copy_to_user(uinsns
, img
, len
))
2196 if (copy_to_user(uinsns
, prog
->bpf_func
, ulen
))
2200 info
.jited_prog_insns
= 0;
2204 ulen
= info
.nr_jited_ksyms
;
2205 info
.nr_jited_ksyms
= prog
->aux
->func_cnt
? : 1;
2207 if (bpf_dump_raw_ok()) {
2208 unsigned long ksym_addr
;
2209 u64 __user
*user_ksyms
;
2212 /* copy the address of the kernel symbol
2213 * corresponding to each function
2215 ulen
= min_t(u32
, info
.nr_jited_ksyms
, ulen
);
2216 user_ksyms
= u64_to_user_ptr(info
.jited_ksyms
);
2217 if (prog
->aux
->func_cnt
) {
2218 for (i
= 0; i
< ulen
; i
++) {
2219 ksym_addr
= (unsigned long)
2220 prog
->aux
->func
[i
]->bpf_func
;
2221 if (put_user((u64
) ksym_addr
,
2226 ksym_addr
= (unsigned long) prog
->bpf_func
;
2227 if (put_user((u64
) ksym_addr
, &user_ksyms
[0]))
2231 info
.jited_ksyms
= 0;
2235 ulen
= info
.nr_jited_func_lens
;
2236 info
.nr_jited_func_lens
= prog
->aux
->func_cnt
? : 1;
2238 if (bpf_dump_raw_ok()) {
2239 u32 __user
*user_lens
;
2242 /* copy the JITed image lengths for each function */
2243 ulen
= min_t(u32
, info
.nr_jited_func_lens
, ulen
);
2244 user_lens
= u64_to_user_ptr(info
.jited_func_lens
);
2245 if (prog
->aux
->func_cnt
) {
2246 for (i
= 0; i
< ulen
; i
++) {
2248 prog
->aux
->func
[i
]->jited_len
;
2249 if (put_user(func_len
, &user_lens
[i
]))
2253 func_len
= prog
->jited_len
;
2254 if (put_user(func_len
, &user_lens
[0]))
2258 info
.jited_func_lens
= 0;
2263 info
.btf_id
= btf_id(prog
->aux
->btf
);
2265 ulen
= info
.nr_func_info
;
2266 info
.nr_func_info
= prog
->aux
->func_info_cnt
;
2267 if (info
.nr_func_info
&& ulen
) {
2268 char __user
*user_finfo
;
2270 user_finfo
= u64_to_user_ptr(info
.func_info
);
2271 ulen
= min_t(u32
, info
.nr_func_info
, ulen
);
2272 if (copy_to_user(user_finfo
, prog
->aux
->func_info
,
2273 info
.func_info_rec_size
* ulen
))
2277 ulen
= info
.nr_line_info
;
2278 info
.nr_line_info
= prog
->aux
->nr_linfo
;
2279 if (info
.nr_line_info
&& ulen
) {
2280 __u8 __user
*user_linfo
;
2282 user_linfo
= u64_to_user_ptr(info
.line_info
);
2283 ulen
= min_t(u32
, info
.nr_line_info
, ulen
);
2284 if (copy_to_user(user_linfo
, prog
->aux
->linfo
,
2285 info
.line_info_rec_size
* ulen
))
2289 ulen
= info
.nr_jited_line_info
;
2290 if (prog
->aux
->jited_linfo
)
2291 info
.nr_jited_line_info
= prog
->aux
->nr_linfo
;
2293 info
.nr_jited_line_info
= 0;
2294 if (info
.nr_jited_line_info
&& ulen
) {
2295 if (bpf_dump_raw_ok()) {
2296 __u64 __user
*user_linfo
;
2299 user_linfo
= u64_to_user_ptr(info
.jited_line_info
);
2300 ulen
= min_t(u32
, info
.nr_jited_line_info
, ulen
);
2301 for (i
= 0; i
< ulen
; i
++) {
2302 if (put_user((__u64
)(long)prog
->aux
->jited_linfo
[i
],
2307 info
.jited_line_info
= 0;
2311 ulen
= info
.nr_prog_tags
;
2312 info
.nr_prog_tags
= prog
->aux
->func_cnt
? : 1;
2314 __u8
__user (*user_prog_tags
)[BPF_TAG_SIZE
];
2317 user_prog_tags
= u64_to_user_ptr(info
.prog_tags
);
2318 ulen
= min_t(u32
, info
.nr_prog_tags
, ulen
);
2319 if (prog
->aux
->func_cnt
) {
2320 for (i
= 0; i
< ulen
; i
++) {
2321 if (copy_to_user(user_prog_tags
[i
],
2322 prog
->aux
->func
[i
]->tag
,
2327 if (copy_to_user(user_prog_tags
[0],
2328 prog
->tag
, BPF_TAG_SIZE
))
2334 if (copy_to_user(uinfo
, &info
, info_len
) ||
2335 put_user(info_len
, &uattr
->info
.info_len
))
2341 static int bpf_map_get_info_by_fd(struct bpf_map
*map
,
2342 const union bpf_attr
*attr
,
2343 union bpf_attr __user
*uattr
)
2345 struct bpf_map_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
2346 struct bpf_map_info info
= {};
2347 u32 info_len
= attr
->info
.info_len
;
2350 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(info
), info_len
);
2353 info_len
= min_t(u32
, sizeof(info
), info_len
);
2355 info
.type
= map
->map_type
;
2357 info
.key_size
= map
->key_size
;
2358 info
.value_size
= map
->value_size
;
2359 info
.max_entries
= map
->max_entries
;
2360 info
.map_flags
= map
->map_flags
;
2361 memcpy(info
.name
, map
->name
, sizeof(map
->name
));
2364 info
.btf_id
= btf_id(map
->btf
);
2365 info
.btf_key_type_id
= map
->btf_key_type_id
;
2366 info
.btf_value_type_id
= map
->btf_value_type_id
;
2369 if (bpf_map_is_dev_bound(map
)) {
2370 err
= bpf_map_offload_info_fill(&info
, map
);
2375 if (copy_to_user(uinfo
, &info
, info_len
) ||
2376 put_user(info_len
, &uattr
->info
.info_len
))
2382 static int bpf_btf_get_info_by_fd(struct btf
*btf
,
2383 const union bpf_attr
*attr
,
2384 union bpf_attr __user
*uattr
)
2386 struct bpf_btf_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
2387 u32 info_len
= attr
->info
.info_len
;
2390 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(*uinfo
), info_len
);
2394 return btf_get_info_by_fd(btf
, attr
, uattr
);
2397 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
2399 static int bpf_obj_get_info_by_fd(const union bpf_attr
*attr
,
2400 union bpf_attr __user
*uattr
)
2402 int ufd
= attr
->info
.bpf_fd
;
2406 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD
))
2413 if (f
.file
->f_op
== &bpf_prog_fops
)
2414 err
= bpf_prog_get_info_by_fd(f
.file
->private_data
, attr
,
2416 else if (f
.file
->f_op
== &bpf_map_fops
)
2417 err
= bpf_map_get_info_by_fd(f
.file
->private_data
, attr
,
2419 else if (f
.file
->f_op
== &btf_fops
)
2420 err
= bpf_btf_get_info_by_fd(f
.file
->private_data
, attr
, uattr
);
2428 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
2430 static int bpf_btf_load(const union bpf_attr
*attr
)
2432 if (CHECK_ATTR(BPF_BTF_LOAD
))
2435 if (!capable(CAP_SYS_ADMIN
))
2438 return btf_new_fd(attr
);
2441 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
2443 static int bpf_btf_get_fd_by_id(const union bpf_attr
*attr
)
2445 if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID
))
2448 if (!capable(CAP_SYS_ADMIN
))
2451 return btf_get_fd_by_id(attr
->btf_id
);
2454 static int bpf_task_fd_query_copy(const union bpf_attr
*attr
,
2455 union bpf_attr __user
*uattr
,
2456 u32 prog_id
, u32 fd_type
,
2457 const char *buf
, u64 probe_offset
,
2460 char __user
*ubuf
= u64_to_user_ptr(attr
->task_fd_query
.buf
);
2461 u32 len
= buf
? strlen(buf
) : 0, input_len
;
2464 if (put_user(len
, &uattr
->task_fd_query
.buf_len
))
2466 input_len
= attr
->task_fd_query
.buf_len
;
2467 if (input_len
&& ubuf
) {
2469 /* nothing to copy, just make ubuf NULL terminated */
2472 if (put_user(zero
, ubuf
))
2474 } else if (input_len
>= len
+ 1) {
2475 /* ubuf can hold the string with NULL terminator */
2476 if (copy_to_user(ubuf
, buf
, len
+ 1))
2479 /* ubuf cannot hold the string with NULL terminator,
2480 * do a partial copy with NULL terminator.
2485 if (copy_to_user(ubuf
, buf
, input_len
- 1))
2487 if (put_user(zero
, ubuf
+ input_len
- 1))
2492 if (put_user(prog_id
, &uattr
->task_fd_query
.prog_id
) ||
2493 put_user(fd_type
, &uattr
->task_fd_query
.fd_type
) ||
2494 put_user(probe_offset
, &uattr
->task_fd_query
.probe_offset
) ||
2495 put_user(probe_addr
, &uattr
->task_fd_query
.probe_addr
))
2501 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
2503 static int bpf_task_fd_query(const union bpf_attr
*attr
,
2504 union bpf_attr __user
*uattr
)
2506 pid_t pid
= attr
->task_fd_query
.pid
;
2507 u32 fd
= attr
->task_fd_query
.fd
;
2508 const struct perf_event
*event
;
2509 struct files_struct
*files
;
2510 struct task_struct
*task
;
2514 if (CHECK_ATTR(BPF_TASK_FD_QUERY
))
2517 if (!capable(CAP_SYS_ADMIN
))
2520 if (attr
->task_fd_query
.flags
!= 0)
2523 task
= get_pid_task(find_vpid(pid
), PIDTYPE_PID
);
2527 files
= get_files_struct(task
);
2528 put_task_struct(task
);
2533 spin_lock(&files
->file_lock
);
2534 file
= fcheck_files(files
, fd
);
2539 spin_unlock(&files
->file_lock
);
2540 put_files_struct(files
);
2545 if (file
->f_op
== &bpf_raw_tp_fops
) {
2546 struct bpf_raw_tracepoint
*raw_tp
= file
->private_data
;
2547 struct bpf_raw_event_map
*btp
= raw_tp
->btp
;
2549 err
= bpf_task_fd_query_copy(attr
, uattr
,
2550 raw_tp
->prog
->aux
->id
,
2551 BPF_FD_TYPE_RAW_TRACEPOINT
,
2552 btp
->tp
->name
, 0, 0);
2556 event
= perf_get_event(file
);
2557 if (!IS_ERR(event
)) {
2558 u64 probe_offset
, probe_addr
;
2559 u32 prog_id
, fd_type
;
2562 err
= bpf_get_perf_event_info(event
, &prog_id
, &fd_type
,
2563 &buf
, &probe_offset
,
2566 err
= bpf_task_fd_query_copy(attr
, uattr
, prog_id
,
2580 SYSCALL_DEFINE3(bpf
, int, cmd
, union bpf_attr __user
*, uattr
, unsigned int, size
)
2582 union bpf_attr attr
= {};
2585 if (sysctl_unprivileged_bpf_disabled
&& !capable(CAP_SYS_ADMIN
))
2588 err
= bpf_check_uarg_tail_zero(uattr
, sizeof(attr
), size
);
2591 size
= min_t(u32
, size
, sizeof(attr
));
2593 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
2594 if (copy_from_user(&attr
, uattr
, size
) != 0)
2597 err
= security_bpf(cmd
, &attr
, size
);
2602 case BPF_MAP_CREATE
:
2603 err
= map_create(&attr
);
2605 case BPF_MAP_LOOKUP_ELEM
:
2606 err
= map_lookup_elem(&attr
);
2608 case BPF_MAP_UPDATE_ELEM
:
2609 err
= map_update_elem(&attr
);
2611 case BPF_MAP_DELETE_ELEM
:
2612 err
= map_delete_elem(&attr
);
2614 case BPF_MAP_GET_NEXT_KEY
:
2615 err
= map_get_next_key(&attr
);
2618 err
= bpf_prog_load(&attr
, uattr
);
2621 err
= bpf_obj_pin(&attr
);
2624 err
= bpf_obj_get(&attr
);
2626 case BPF_PROG_ATTACH
:
2627 err
= bpf_prog_attach(&attr
);
2629 case BPF_PROG_DETACH
:
2630 err
= bpf_prog_detach(&attr
);
2632 case BPF_PROG_QUERY
:
2633 err
= bpf_prog_query(&attr
, uattr
);
2635 case BPF_PROG_TEST_RUN
:
2636 err
= bpf_prog_test_run(&attr
, uattr
);
2638 case BPF_PROG_GET_NEXT_ID
:
2639 err
= bpf_obj_get_next_id(&attr
, uattr
,
2640 &prog_idr
, &prog_idr_lock
);
2642 case BPF_MAP_GET_NEXT_ID
:
2643 err
= bpf_obj_get_next_id(&attr
, uattr
,
2644 &map_idr
, &map_idr_lock
);
2646 case BPF_PROG_GET_FD_BY_ID
:
2647 err
= bpf_prog_get_fd_by_id(&attr
);
2649 case BPF_MAP_GET_FD_BY_ID
:
2650 err
= bpf_map_get_fd_by_id(&attr
);
2652 case BPF_OBJ_GET_INFO_BY_FD
:
2653 err
= bpf_obj_get_info_by_fd(&attr
, uattr
);
2655 case BPF_RAW_TRACEPOINT_OPEN
:
2656 err
= bpf_raw_tracepoint_open(&attr
);
2659 err
= bpf_btf_load(&attr
);
2661 case BPF_BTF_GET_FD_BY_ID
:
2662 err
= bpf_btf_get_fd_by_id(&attr
);
2664 case BPF_TASK_FD_QUERY
:
2665 err
= bpf_task_fd_query(&attr
, uattr
);
2667 case BPF_MAP_LOOKUP_AND_DELETE_ELEM
:
2668 err
= map_lookup_and_delete_elem(&attr
);