1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
5 #include <linux/bpf_trace.h>
6 #include <linux/bpf_lirc.h>
7 #include <linux/bpf_verifier.h>
9 #include <linux/syscalls.h>
10 #include <linux/slab.h>
11 #include <linux/sched/signal.h>
12 #include <linux/vmalloc.h>
13 #include <linux/mmzone.h>
14 #include <linux/anon_inodes.h>
15 #include <linux/fdtable.h>
16 #include <linux/file.h>
18 #include <linux/license.h>
19 #include <linux/filter.h>
20 #include <linux/version.h>
21 #include <linux/kernel.h>
22 #include <linux/idr.h>
23 #include <linux/cred.h>
24 #include <linux/timekeeping.h>
25 #include <linux/ctype.h>
26 #include <linux/nospec.h>
27 #include <linux/audit.h>
28 #include <uapi/linux/btf.h>
29 #include <linux/pgtable.h>
30 #include <linux/bpf_lsm.h>
31 #include <linux/poll.h>
32 #include <linux/bpf-netns.h>
33 #include <linux/rcupdate_trace.h>
34 #include <linux/memcontrol.h>
36 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
37 (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
38 (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
39 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
40 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
41 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
44 #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
46 DEFINE_PER_CPU(int, bpf_prog_active
);
47 static DEFINE_IDR(prog_idr
);
48 static DEFINE_SPINLOCK(prog_idr_lock
);
49 static DEFINE_IDR(map_idr
);
50 static DEFINE_SPINLOCK(map_idr_lock
);
51 static DEFINE_IDR(link_idr
);
52 static DEFINE_SPINLOCK(link_idr_lock
);
54 int sysctl_unprivileged_bpf_disabled __read_mostly
;
56 static const struct bpf_map_ops
* const bpf_map_types
[] = {
57 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
58 #define BPF_MAP_TYPE(_id, _ops) \
60 #define BPF_LINK_TYPE(_id, _name)
61 #include <linux/bpf_types.h>
68 * If we're handed a bigger struct than we know of, ensure all the unknown bits
69 * are 0 - i.e. new user-space does not rely on any kernel feature extensions
70 * we don't know about yet.
72 * There is a ToCToU between this function call and the following
73 * copy_from_user() call. However, this is not a concern since this function is
74 * meant to be a future-proofing of bits.
76 int bpf_check_uarg_tail_zero(void __user
*uaddr
,
80 unsigned char __user
*addr
= uaddr
+ expected_size
;
83 if (unlikely(actual_size
> PAGE_SIZE
)) /* silly large */
86 if (actual_size
<= expected_size
)
89 res
= check_zeroed_user(addr
, actual_size
- expected_size
);
92 return res
? 0 : -E2BIG
;
95 const struct bpf_map_ops bpf_map_offload_ops
= {
96 .map_meta_equal
= bpf_map_meta_equal
,
97 .map_alloc
= bpf_map_offload_map_alloc
,
98 .map_free
= bpf_map_offload_map_free
,
99 .map_check_btf
= map_check_no_btf
,
102 static struct bpf_map
*find_and_alloc_map(union bpf_attr
*attr
)
104 const struct bpf_map_ops
*ops
;
105 u32 type
= attr
->map_type
;
109 if (type
>= ARRAY_SIZE(bpf_map_types
))
110 return ERR_PTR(-EINVAL
);
111 type
= array_index_nospec(type
, ARRAY_SIZE(bpf_map_types
));
112 ops
= bpf_map_types
[type
];
114 return ERR_PTR(-EINVAL
);
116 if (ops
->map_alloc_check
) {
117 err
= ops
->map_alloc_check(attr
);
121 if (attr
->map_ifindex
)
122 ops
= &bpf_map_offload_ops
;
123 map
= ops
->map_alloc(attr
);
127 map
->map_type
= type
;
131 static u32
bpf_map_value_size(const struct bpf_map
*map
)
133 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
134 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
||
135 map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
||
136 map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
)
137 return round_up(map
->value_size
, 8) * num_possible_cpus();
138 else if (IS_FD_MAP(map
))
141 return map
->value_size
;
144 static void maybe_wait_bpf_programs(struct bpf_map
*map
)
146 /* Wait for any running BPF programs to complete so that
147 * userspace, when we return to it, knows that all programs
148 * that could be running use the new map value.
150 if (map
->map_type
== BPF_MAP_TYPE_HASH_OF_MAPS
||
151 map
->map_type
== BPF_MAP_TYPE_ARRAY_OF_MAPS
)
155 static int bpf_map_update_value(struct bpf_map
*map
, struct fd f
, void *key
,
156 void *value
, __u64 flags
)
160 /* Need to create a kthread, thus must support schedule */
161 if (bpf_map_is_dev_bound(map
)) {
162 return bpf_map_offload_update_elem(map
, key
, value
, flags
);
163 } else if (map
->map_type
== BPF_MAP_TYPE_CPUMAP
||
164 map
->map_type
== BPF_MAP_TYPE_STRUCT_OPS
) {
165 return map
->ops
->map_update_elem(map
, key
, value
, flags
);
166 } else if (map
->map_type
== BPF_MAP_TYPE_SOCKHASH
||
167 map
->map_type
== BPF_MAP_TYPE_SOCKMAP
) {
168 return sock_map_update_elem_sys(map
, key
, value
, flags
);
169 } else if (IS_FD_PROG_ARRAY(map
)) {
170 return bpf_fd_array_map_update_elem(map
, f
.file
, key
, value
,
174 bpf_disable_instrumentation();
175 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
176 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
) {
177 err
= bpf_percpu_hash_update(map
, key
, value
, flags
);
178 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
) {
179 err
= bpf_percpu_array_update(map
, key
, value
, flags
);
180 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
) {
181 err
= bpf_percpu_cgroup_storage_update(map
, key
, value
,
183 } else if (IS_FD_ARRAY(map
)) {
185 err
= bpf_fd_array_map_update_elem(map
, f
.file
, key
, value
,
188 } else if (map
->map_type
== BPF_MAP_TYPE_HASH_OF_MAPS
) {
190 err
= bpf_fd_htab_map_update_elem(map
, f
.file
, key
, value
,
193 } else if (map
->map_type
== BPF_MAP_TYPE_REUSEPORT_SOCKARRAY
) {
194 /* rcu_read_lock() is not needed */
195 err
= bpf_fd_reuseport_array_update_elem(map
, key
, value
,
197 } else if (map
->map_type
== BPF_MAP_TYPE_QUEUE
||
198 map
->map_type
== BPF_MAP_TYPE_STACK
) {
199 err
= map
->ops
->map_push_elem(map
, value
, flags
);
202 err
= map
->ops
->map_update_elem(map
, key
, value
, flags
);
205 bpf_enable_instrumentation();
206 maybe_wait_bpf_programs(map
);
211 static int bpf_map_copy_value(struct bpf_map
*map
, void *key
, void *value
,
217 if (bpf_map_is_dev_bound(map
))
218 return bpf_map_offload_lookup_elem(map
, key
, value
);
220 bpf_disable_instrumentation();
221 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
222 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
) {
223 err
= bpf_percpu_hash_copy(map
, key
, value
);
224 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
) {
225 err
= bpf_percpu_array_copy(map
, key
, value
);
226 } else if (map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
) {
227 err
= bpf_percpu_cgroup_storage_copy(map
, key
, value
);
228 } else if (map
->map_type
== BPF_MAP_TYPE_STACK_TRACE
) {
229 err
= bpf_stackmap_copy(map
, key
, value
);
230 } else if (IS_FD_ARRAY(map
) || IS_FD_PROG_ARRAY(map
)) {
231 err
= bpf_fd_array_map_lookup_elem(map
, key
, value
);
232 } else if (IS_FD_HASH(map
)) {
233 err
= bpf_fd_htab_map_lookup_elem(map
, key
, value
);
234 } else if (map
->map_type
== BPF_MAP_TYPE_REUSEPORT_SOCKARRAY
) {
235 err
= bpf_fd_reuseport_array_lookup_elem(map
, key
, value
);
236 } else if (map
->map_type
== BPF_MAP_TYPE_QUEUE
||
237 map
->map_type
== BPF_MAP_TYPE_STACK
) {
238 err
= map
->ops
->map_peek_elem(map
, value
);
239 } else if (map
->map_type
== BPF_MAP_TYPE_STRUCT_OPS
) {
240 /* struct_ops map requires directly updating "value" */
241 err
= bpf_struct_ops_map_sys_lookup_elem(map
, key
, value
);
244 if (map
->ops
->map_lookup_elem_sys_only
)
245 ptr
= map
->ops
->map_lookup_elem_sys_only(map
, key
);
247 ptr
= map
->ops
->map_lookup_elem(map
, key
);
254 if (flags
& BPF_F_LOCK
)
255 /* lock 'ptr' and copy everything but lock */
256 copy_map_value_locked(map
, value
, ptr
, true);
258 copy_map_value(map
, value
, ptr
);
259 /* mask lock, since value wasn't zero inited */
260 check_and_init_map_lock(map
, value
);
265 bpf_enable_instrumentation();
266 maybe_wait_bpf_programs(map
);
271 /* Please, do not use this function outside from the map creation path
272 * (e.g. in map update path) without taking care of setting the active
273 * memory cgroup (see at bpf_map_kmalloc_node() for example).
275 static void *__bpf_map_area_alloc(u64 size
, int numa_node
, bool mmapable
)
277 /* We really just want to fail instead of triggering OOM killer
278 * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
279 * which is used for lower order allocation requests.
281 * It has been observed that higher order allocation requests done by
282 * vmalloc with __GFP_NORETRY being set might fail due to not trying
283 * to reclaim memory from the page cache, thus we set
284 * __GFP_RETRY_MAYFAIL to avoid such situations.
287 const gfp_t gfp
= __GFP_NOWARN
| __GFP_ZERO
| __GFP_ACCOUNT
;
288 unsigned int flags
= 0;
289 unsigned long align
= 1;
292 if (size
>= SIZE_MAX
)
295 /* kmalloc()'ed memory can't be mmap()'ed */
297 BUG_ON(!PAGE_ALIGNED(size
));
300 } else if (size
<= (PAGE_SIZE
<< PAGE_ALLOC_COSTLY_ORDER
)) {
301 area
= kmalloc_node(size
, gfp
| GFP_USER
| __GFP_NORETRY
,
307 return __vmalloc_node_range(size
, align
, VMALLOC_START
, VMALLOC_END
,
308 gfp
| GFP_KERNEL
| __GFP_RETRY_MAYFAIL
, PAGE_KERNEL
,
309 flags
, numa_node
, __builtin_return_address(0));
312 void *bpf_map_area_alloc(u64 size
, int numa_node
)
314 return __bpf_map_area_alloc(size
, numa_node
, false);
317 void *bpf_map_area_mmapable_alloc(u64 size
, int numa_node
)
319 return __bpf_map_area_alloc(size
, numa_node
, true);
322 void bpf_map_area_free(void *area
)
327 static u32
bpf_map_flags_retain_permanent(u32 flags
)
329 /* Some map creation flags are not tied to the map object but
330 * rather to the map fd instead, so they have no meaning upon
331 * map object inspection since multiple file descriptors with
332 * different (access) properties can exist here. Thus, given
333 * this has zero meaning for the map itself, lets clear these
336 return flags
& ~(BPF_F_RDONLY
| BPF_F_WRONLY
);
339 void bpf_map_init_from_attr(struct bpf_map
*map
, union bpf_attr
*attr
)
341 map
->map_type
= attr
->map_type
;
342 map
->key_size
= attr
->key_size
;
343 map
->value_size
= attr
->value_size
;
344 map
->max_entries
= attr
->max_entries
;
345 map
->map_flags
= bpf_map_flags_retain_permanent(attr
->map_flags
);
346 map
->numa_node
= bpf_map_attr_numa_node(attr
);
349 static int bpf_map_alloc_id(struct bpf_map
*map
)
353 idr_preload(GFP_KERNEL
);
354 spin_lock_bh(&map_idr_lock
);
355 id
= idr_alloc_cyclic(&map_idr
, map
, 1, INT_MAX
, GFP_ATOMIC
);
358 spin_unlock_bh(&map_idr_lock
);
361 if (WARN_ON_ONCE(!id
))
364 return id
> 0 ? 0 : id
;
367 void bpf_map_free_id(struct bpf_map
*map
, bool do_idr_lock
)
371 /* Offloaded maps are removed from the IDR store when their device
372 * disappears - even if someone holds an fd to them they are unusable,
373 * the memory is gone, all ops will fail; they are simply waiting for
374 * refcnt to drop to be freed.
380 spin_lock_irqsave(&map_idr_lock
, flags
);
382 __acquire(&map_idr_lock
);
384 idr_remove(&map_idr
, map
->id
);
388 spin_unlock_irqrestore(&map_idr_lock
, flags
);
390 __release(&map_idr_lock
);
393 #ifdef CONFIG_MEMCG_KMEM
394 static void bpf_map_save_memcg(struct bpf_map
*map
)
396 map
->memcg
= get_mem_cgroup_from_mm(current
->mm
);
399 static void bpf_map_release_memcg(struct bpf_map
*map
)
401 mem_cgroup_put(map
->memcg
);
404 void *bpf_map_kmalloc_node(const struct bpf_map
*map
, size_t size
, gfp_t flags
,
407 struct mem_cgroup
*old_memcg
;
410 old_memcg
= set_active_memcg(map
->memcg
);
411 ptr
= kmalloc_node(size
, flags
| __GFP_ACCOUNT
, node
);
412 set_active_memcg(old_memcg
);
417 void *bpf_map_kzalloc(const struct bpf_map
*map
, size_t size
, gfp_t flags
)
419 struct mem_cgroup
*old_memcg
;
422 old_memcg
= set_active_memcg(map
->memcg
);
423 ptr
= kzalloc(size
, flags
| __GFP_ACCOUNT
);
424 set_active_memcg(old_memcg
);
429 void __percpu
*bpf_map_alloc_percpu(const struct bpf_map
*map
, size_t size
,
430 size_t align
, gfp_t flags
)
432 struct mem_cgroup
*old_memcg
;
435 old_memcg
= set_active_memcg(map
->memcg
);
436 ptr
= __alloc_percpu_gfp(size
, align
, flags
| __GFP_ACCOUNT
);
437 set_active_memcg(old_memcg
);
443 static void bpf_map_save_memcg(struct bpf_map
*map
)
447 static void bpf_map_release_memcg(struct bpf_map
*map
)
452 /* called from workqueue */
453 static void bpf_map_free_deferred(struct work_struct
*work
)
455 struct bpf_map
*map
= container_of(work
, struct bpf_map
, work
);
457 security_bpf_map_free(map
);
458 bpf_map_release_memcg(map
);
459 /* implementation dependent freeing */
460 map
->ops
->map_free(map
);
463 static void bpf_map_put_uref(struct bpf_map
*map
)
465 if (atomic64_dec_and_test(&map
->usercnt
)) {
466 if (map
->ops
->map_release_uref
)
467 map
->ops
->map_release_uref(map
);
471 /* decrement map refcnt and schedule it for freeing via workqueue
472 * (unrelying map implementation ops->map_free() might sleep)
474 static void __bpf_map_put(struct bpf_map
*map
, bool do_idr_lock
)
476 if (atomic64_dec_and_test(&map
->refcnt
)) {
477 /* bpf_map_free_id() must be called first */
478 bpf_map_free_id(map
, do_idr_lock
);
480 INIT_WORK(&map
->work
, bpf_map_free_deferred
);
481 schedule_work(&map
->work
);
485 void bpf_map_put(struct bpf_map
*map
)
487 __bpf_map_put(map
, true);
489 EXPORT_SYMBOL_GPL(bpf_map_put
);
491 void bpf_map_put_with_uref(struct bpf_map
*map
)
493 bpf_map_put_uref(map
);
497 static int bpf_map_release(struct inode
*inode
, struct file
*filp
)
499 struct bpf_map
*map
= filp
->private_data
;
501 if (map
->ops
->map_release
)
502 map
->ops
->map_release(map
, filp
);
504 bpf_map_put_with_uref(map
);
508 static fmode_t
map_get_sys_perms(struct bpf_map
*map
, struct fd f
)
510 fmode_t mode
= f
.file
->f_mode
;
512 /* Our file permissions may have been overridden by global
513 * map permissions facing syscall side.
515 if (READ_ONCE(map
->frozen
))
516 mode
&= ~FMODE_CAN_WRITE
;
520 #ifdef CONFIG_PROC_FS
521 /* Provides an approximation of the map's memory footprint.
522 * Used only to provide a backward compatibility and display
523 * a reasonable "memlock" info.
525 static unsigned long bpf_map_memory_footprint(const struct bpf_map
*map
)
529 size
= round_up(map
->key_size
+ bpf_map_value_size(map
), 8);
531 return round_up(map
->max_entries
* size
, PAGE_SIZE
);
534 static void bpf_map_show_fdinfo(struct seq_file
*m
, struct file
*filp
)
536 const struct bpf_map
*map
= filp
->private_data
;
537 const struct bpf_array
*array
;
538 u32 type
= 0, jited
= 0;
540 if (map
->map_type
== BPF_MAP_TYPE_PROG_ARRAY
) {
541 array
= container_of(map
, struct bpf_array
, map
);
542 type
= array
->aux
->type
;
543 jited
= array
->aux
->jited
;
560 bpf_map_memory_footprint(map
),
562 READ_ONCE(map
->frozen
));
564 seq_printf(m
, "owner_prog_type:\t%u\n", type
);
565 seq_printf(m
, "owner_jited:\t%u\n", jited
);
570 static ssize_t
bpf_dummy_read(struct file
*filp
, char __user
*buf
, size_t siz
,
573 /* We need this handler such that alloc_file() enables
574 * f_mode with FMODE_CAN_READ.
579 static ssize_t
bpf_dummy_write(struct file
*filp
, const char __user
*buf
,
580 size_t siz
, loff_t
*ppos
)
582 /* We need this handler such that alloc_file() enables
583 * f_mode with FMODE_CAN_WRITE.
588 /* called for any extra memory-mapped regions (except initial) */
589 static void bpf_map_mmap_open(struct vm_area_struct
*vma
)
591 struct bpf_map
*map
= vma
->vm_file
->private_data
;
593 if (vma
->vm_flags
& VM_MAYWRITE
) {
594 mutex_lock(&map
->freeze_mutex
);
596 mutex_unlock(&map
->freeze_mutex
);
600 /* called for all unmapped memory region (including initial) */
601 static void bpf_map_mmap_close(struct vm_area_struct
*vma
)
603 struct bpf_map
*map
= vma
->vm_file
->private_data
;
605 if (vma
->vm_flags
& VM_MAYWRITE
) {
606 mutex_lock(&map
->freeze_mutex
);
608 mutex_unlock(&map
->freeze_mutex
);
612 static const struct vm_operations_struct bpf_map_default_vmops
= {
613 .open
= bpf_map_mmap_open
,
614 .close
= bpf_map_mmap_close
,
617 static int bpf_map_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
619 struct bpf_map
*map
= filp
->private_data
;
622 if (!map
->ops
->map_mmap
|| map_value_has_spin_lock(map
))
625 if (!(vma
->vm_flags
& VM_SHARED
))
628 mutex_lock(&map
->freeze_mutex
);
630 if (vma
->vm_flags
& VM_WRITE
) {
635 /* map is meant to be read-only, so do not allow mapping as
636 * writable, because it's possible to leak a writable page
637 * reference and allows user-space to still modify it after
638 * freezing, while verifier will assume contents do not change
640 if (map
->map_flags
& BPF_F_RDONLY_PROG
) {
646 /* set default open/close callbacks */
647 vma
->vm_ops
= &bpf_map_default_vmops
;
648 vma
->vm_private_data
= map
;
649 vma
->vm_flags
&= ~VM_MAYEXEC
;
650 if (!(vma
->vm_flags
& VM_WRITE
))
651 /* disallow re-mapping with PROT_WRITE */
652 vma
->vm_flags
&= ~VM_MAYWRITE
;
654 err
= map
->ops
->map_mmap(map
, vma
);
658 if (vma
->vm_flags
& VM_MAYWRITE
)
661 mutex_unlock(&map
->freeze_mutex
);
665 static __poll_t
bpf_map_poll(struct file
*filp
, struct poll_table_struct
*pts
)
667 struct bpf_map
*map
= filp
->private_data
;
669 if (map
->ops
->map_poll
)
670 return map
->ops
->map_poll(map
, filp
, pts
);
675 const struct file_operations bpf_map_fops
= {
676 #ifdef CONFIG_PROC_FS
677 .show_fdinfo
= bpf_map_show_fdinfo
,
679 .release
= bpf_map_release
,
680 .read
= bpf_dummy_read
,
681 .write
= bpf_dummy_write
,
682 .mmap
= bpf_map_mmap
,
683 .poll
= bpf_map_poll
,
686 int bpf_map_new_fd(struct bpf_map
*map
, int flags
)
690 ret
= security_bpf_map(map
, OPEN_FMODE(flags
));
694 return anon_inode_getfd("bpf-map", &bpf_map_fops
, map
,
698 int bpf_get_file_flag(int flags
)
700 if ((flags
& BPF_F_RDONLY
) && (flags
& BPF_F_WRONLY
))
702 if (flags
& BPF_F_RDONLY
)
704 if (flags
& BPF_F_WRONLY
)
709 /* helper macro to check that unused fields 'union bpf_attr' are zero */
710 #define CHECK_ATTR(CMD) \
711 memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
712 sizeof(attr->CMD##_LAST_FIELD), 0, \
714 offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
715 sizeof(attr->CMD##_LAST_FIELD)) != NULL
717 /* dst and src must have at least "size" number of bytes.
718 * Return strlen on success and < 0 on error.
720 int bpf_obj_name_cpy(char *dst
, const char *src
, unsigned int size
)
722 const char *end
= src
+ size
;
723 const char *orig_src
= src
;
725 memset(dst
, 0, size
);
726 /* Copy all isalnum(), '_' and '.' chars. */
727 while (src
< end
&& *src
) {
728 if (!isalnum(*src
) &&
729 *src
!= '_' && *src
!= '.')
734 /* No '\0' found in "size" number of bytes */
738 return src
- orig_src
;
741 int map_check_no_btf(const struct bpf_map
*map
,
742 const struct btf
*btf
,
743 const struct btf_type
*key_type
,
744 const struct btf_type
*value_type
)
749 static int map_check_btf(struct bpf_map
*map
, const struct btf
*btf
,
750 u32 btf_key_id
, u32 btf_value_id
)
752 const struct btf_type
*key_type
, *value_type
;
753 u32 key_size
, value_size
;
756 /* Some maps allow key to be unspecified. */
758 key_type
= btf_type_id_size(btf
, &btf_key_id
, &key_size
);
759 if (!key_type
|| key_size
!= map
->key_size
)
762 key_type
= btf_type_by_id(btf
, 0);
763 if (!map
->ops
->map_check_btf
)
767 value_type
= btf_type_id_size(btf
, &btf_value_id
, &value_size
);
768 if (!value_type
|| value_size
!= map
->value_size
)
771 map
->spin_lock_off
= btf_find_spin_lock(btf
, value_type
);
773 if (map_value_has_spin_lock(map
)) {
774 if (map
->map_flags
& BPF_F_RDONLY_PROG
)
776 if (map
->map_type
!= BPF_MAP_TYPE_HASH
&&
777 map
->map_type
!= BPF_MAP_TYPE_ARRAY
&&
778 map
->map_type
!= BPF_MAP_TYPE_CGROUP_STORAGE
&&
779 map
->map_type
!= BPF_MAP_TYPE_SK_STORAGE
&&
780 map
->map_type
!= BPF_MAP_TYPE_INODE_STORAGE
&&
781 map
->map_type
!= BPF_MAP_TYPE_TASK_STORAGE
)
783 if (map
->spin_lock_off
+ sizeof(struct bpf_spin_lock
) >
786 "verifier bug spin_lock_off %d value_size %d\n",
787 map
->spin_lock_off
, map
->value_size
);
792 if (map
->ops
->map_check_btf
)
793 ret
= map
->ops
->map_check_btf(map
, btf
, key_type
, value_type
);
798 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
799 /* called via syscall */
800 static int map_create(union bpf_attr
*attr
)
802 int numa_node
= bpf_map_attr_numa_node(attr
);
807 err
= CHECK_ATTR(BPF_MAP_CREATE
);
811 if (attr
->btf_vmlinux_value_type_id
) {
812 if (attr
->map_type
!= BPF_MAP_TYPE_STRUCT_OPS
||
813 attr
->btf_key_type_id
|| attr
->btf_value_type_id
)
815 } else if (attr
->btf_key_type_id
&& !attr
->btf_value_type_id
) {
819 f_flags
= bpf_get_file_flag(attr
->map_flags
);
823 if (numa_node
!= NUMA_NO_NODE
&&
824 ((unsigned int)numa_node
>= nr_node_ids
||
825 !node_online(numa_node
)))
828 /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
829 map
= find_and_alloc_map(attr
);
833 err
= bpf_obj_name_cpy(map
->name
, attr
->map_name
,
834 sizeof(attr
->map_name
));
838 atomic64_set(&map
->refcnt
, 1);
839 atomic64_set(&map
->usercnt
, 1);
840 mutex_init(&map
->freeze_mutex
);
842 map
->spin_lock_off
= -EINVAL
;
843 if (attr
->btf_key_type_id
|| attr
->btf_value_type_id
||
844 /* Even the map's value is a kernel's struct,
845 * the bpf_prog.o must have BTF to begin with
846 * to figure out the corresponding kernel's
847 * counter part. Thus, attr->btf_fd has
850 attr
->btf_vmlinux_value_type_id
) {
853 btf
= btf_get_by_fd(attr
->btf_fd
);
860 if (attr
->btf_value_type_id
) {
861 err
= map_check_btf(map
, btf
, attr
->btf_key_type_id
,
862 attr
->btf_value_type_id
);
867 map
->btf_key_type_id
= attr
->btf_key_type_id
;
868 map
->btf_value_type_id
= attr
->btf_value_type_id
;
869 map
->btf_vmlinux_value_type_id
=
870 attr
->btf_vmlinux_value_type_id
;
873 err
= security_bpf_map_alloc(map
);
877 err
= bpf_map_alloc_id(map
);
881 bpf_map_save_memcg(map
);
883 err
= bpf_map_new_fd(map
, f_flags
);
885 /* failed to allocate fd.
886 * bpf_map_put_with_uref() is needed because the above
887 * bpf_map_alloc_id() has published the map
888 * to the userspace and the userspace may
889 * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
891 bpf_map_put_with_uref(map
);
898 security_bpf_map_free(map
);
901 map
->ops
->map_free(map
);
905 /* if error is returned, fd is released.
906 * On success caller should complete fd access with matching fdput()
908 struct bpf_map
*__bpf_map_get(struct fd f
)
911 return ERR_PTR(-EBADF
);
912 if (f
.file
->f_op
!= &bpf_map_fops
) {
914 return ERR_PTR(-EINVAL
);
917 return f
.file
->private_data
;
920 void bpf_map_inc(struct bpf_map
*map
)
922 atomic64_inc(&map
->refcnt
);
924 EXPORT_SYMBOL_GPL(bpf_map_inc
);
926 void bpf_map_inc_with_uref(struct bpf_map
*map
)
928 atomic64_inc(&map
->refcnt
);
929 atomic64_inc(&map
->usercnt
);
931 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref
);
933 struct bpf_map
*bpf_map_get(u32 ufd
)
935 struct fd f
= fdget(ufd
);
938 map
= __bpf_map_get(f
);
948 struct bpf_map
*bpf_map_get_with_uref(u32 ufd
)
950 struct fd f
= fdget(ufd
);
953 map
= __bpf_map_get(f
);
957 bpf_map_inc_with_uref(map
);
963 /* map_idr_lock should have been held */
964 static struct bpf_map
*__bpf_map_inc_not_zero(struct bpf_map
*map
, bool uref
)
968 refold
= atomic64_fetch_add_unless(&map
->refcnt
, 1, 0);
970 return ERR_PTR(-ENOENT
);
972 atomic64_inc(&map
->usercnt
);
977 struct bpf_map
*bpf_map_inc_not_zero(struct bpf_map
*map
)
979 spin_lock_bh(&map_idr_lock
);
980 map
= __bpf_map_inc_not_zero(map
, false);
981 spin_unlock_bh(&map_idr_lock
);
985 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero
);
987 int __weak
bpf_stackmap_copy(struct bpf_map
*map
, void *key
, void *value
)
992 static void *__bpf_copy_key(void __user
*ukey
, u64 key_size
)
995 return memdup_user(ukey
, key_size
);
998 return ERR_PTR(-EINVAL
);
1003 /* last field in 'union bpf_attr' used by this command */
1004 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
1006 static int map_lookup_elem(union bpf_attr
*attr
)
1008 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
1009 void __user
*uvalue
= u64_to_user_ptr(attr
->value
);
1010 int ufd
= attr
->map_fd
;
1011 struct bpf_map
*map
;
1017 if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM
))
1020 if (attr
->flags
& ~BPF_F_LOCK
)
1024 map
= __bpf_map_get(f
);
1026 return PTR_ERR(map
);
1027 if (!(map_get_sys_perms(map
, f
) & FMODE_CAN_READ
)) {
1032 if ((attr
->flags
& BPF_F_LOCK
) &&
1033 !map_value_has_spin_lock(map
)) {
1038 key
= __bpf_copy_key(ukey
, map
->key_size
);
1044 value_size
= bpf_map_value_size(map
);
1047 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
1051 err
= bpf_map_copy_value(map
, key
, value
, attr
->flags
);
1056 if (copy_to_user(uvalue
, value
, value_size
) != 0)
1071 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1073 static int map_update_elem(union bpf_attr
*attr
)
1075 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
1076 void __user
*uvalue
= u64_to_user_ptr(attr
->value
);
1077 int ufd
= attr
->map_fd
;
1078 struct bpf_map
*map
;
1084 if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM
))
1088 map
= __bpf_map_get(f
);
1090 return PTR_ERR(map
);
1091 if (!(map_get_sys_perms(map
, f
) & FMODE_CAN_WRITE
)) {
1096 if ((attr
->flags
& BPF_F_LOCK
) &&
1097 !map_value_has_spin_lock(map
)) {
1102 key
= __bpf_copy_key(ukey
, map
->key_size
);
1108 if (map
->map_type
== BPF_MAP_TYPE_PERCPU_HASH
||
1109 map
->map_type
== BPF_MAP_TYPE_LRU_PERCPU_HASH
||
1110 map
->map_type
== BPF_MAP_TYPE_PERCPU_ARRAY
||
1111 map
->map_type
== BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
)
1112 value_size
= round_up(map
->value_size
, 8) * num_possible_cpus();
1114 value_size
= map
->value_size
;
1117 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
1122 if (copy_from_user(value
, uvalue
, value_size
) != 0)
1125 err
= bpf_map_update_value(map
, f
, key
, value
, attr
->flags
);
1136 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1138 static int map_delete_elem(union bpf_attr
*attr
)
1140 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
1141 int ufd
= attr
->map_fd
;
1142 struct bpf_map
*map
;
1147 if (CHECK_ATTR(BPF_MAP_DELETE_ELEM
))
1151 map
= __bpf_map_get(f
);
1153 return PTR_ERR(map
);
1154 if (!(map_get_sys_perms(map
, f
) & FMODE_CAN_WRITE
)) {
1159 key
= __bpf_copy_key(ukey
, map
->key_size
);
1165 if (bpf_map_is_dev_bound(map
)) {
1166 err
= bpf_map_offload_delete_elem(map
, key
);
1168 } else if (IS_FD_PROG_ARRAY(map
) ||
1169 map
->map_type
== BPF_MAP_TYPE_STRUCT_OPS
) {
1170 /* These maps require sleepable context */
1171 err
= map
->ops
->map_delete_elem(map
, key
);
1175 bpf_disable_instrumentation();
1177 err
= map
->ops
->map_delete_elem(map
, key
);
1179 bpf_enable_instrumentation();
1180 maybe_wait_bpf_programs(map
);
1188 /* last field in 'union bpf_attr' used by this command */
1189 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1191 static int map_get_next_key(union bpf_attr
*attr
)
1193 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
1194 void __user
*unext_key
= u64_to_user_ptr(attr
->next_key
);
1195 int ufd
= attr
->map_fd
;
1196 struct bpf_map
*map
;
1197 void *key
, *next_key
;
1201 if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY
))
1205 map
= __bpf_map_get(f
);
1207 return PTR_ERR(map
);
1208 if (!(map_get_sys_perms(map
, f
) & FMODE_CAN_READ
)) {
1214 key
= __bpf_copy_key(ukey
, map
->key_size
);
1224 next_key
= kmalloc(map
->key_size
, GFP_USER
);
1228 if (bpf_map_is_dev_bound(map
)) {
1229 err
= bpf_map_offload_get_next_key(map
, key
, next_key
);
1234 err
= map
->ops
->map_get_next_key(map
, key
, next_key
);
1241 if (copy_to_user(unext_key
, next_key
, map
->key_size
) != 0)
1255 int generic_map_delete_batch(struct bpf_map
*map
,
1256 const union bpf_attr
*attr
,
1257 union bpf_attr __user
*uattr
)
1259 void __user
*keys
= u64_to_user_ptr(attr
->batch
.keys
);
1264 if (attr
->batch
.elem_flags
& ~BPF_F_LOCK
)
1267 if ((attr
->batch
.elem_flags
& BPF_F_LOCK
) &&
1268 !map_value_has_spin_lock(map
)) {
1272 max_count
= attr
->batch
.count
;
1276 key
= kmalloc(map
->key_size
, GFP_USER
| __GFP_NOWARN
);
1280 for (cp
= 0; cp
< max_count
; cp
++) {
1282 if (copy_from_user(key
, keys
+ cp
* map
->key_size
,
1286 if (bpf_map_is_dev_bound(map
)) {
1287 err
= bpf_map_offload_delete_elem(map
, key
);
1291 bpf_disable_instrumentation();
1293 err
= map
->ops
->map_delete_elem(map
, key
);
1295 bpf_enable_instrumentation();
1296 maybe_wait_bpf_programs(map
);
1300 if (copy_to_user(&uattr
->batch
.count
, &cp
, sizeof(cp
)))
1307 int generic_map_update_batch(struct bpf_map
*map
,
1308 const union bpf_attr
*attr
,
1309 union bpf_attr __user
*uattr
)
1311 void __user
*values
= u64_to_user_ptr(attr
->batch
.values
);
1312 void __user
*keys
= u64_to_user_ptr(attr
->batch
.keys
);
1313 u32 value_size
, cp
, max_count
;
1314 int ufd
= attr
->map_fd
;
1320 if (attr
->batch
.elem_flags
& ~BPF_F_LOCK
)
1323 if ((attr
->batch
.elem_flags
& BPF_F_LOCK
) &&
1324 !map_value_has_spin_lock(map
)) {
1328 value_size
= bpf_map_value_size(map
);
1330 max_count
= attr
->batch
.count
;
1334 key
= kmalloc(map
->key_size
, GFP_USER
| __GFP_NOWARN
);
1338 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
1344 for (cp
= 0; cp
< max_count
; cp
++) {
1346 if (copy_from_user(key
, keys
+ cp
* map
->key_size
,
1348 copy_from_user(value
, values
+ cp
* value_size
, value_size
))
1351 err
= bpf_map_update_value(map
, f
, key
, value
,
1352 attr
->batch
.elem_flags
);
1358 if (copy_to_user(&uattr
->batch
.count
, &cp
, sizeof(cp
)))
1366 #define MAP_LOOKUP_RETRIES 3
1368 int generic_map_lookup_batch(struct bpf_map
*map
,
1369 const union bpf_attr
*attr
,
1370 union bpf_attr __user
*uattr
)
1372 void __user
*uobatch
= u64_to_user_ptr(attr
->batch
.out_batch
);
1373 void __user
*ubatch
= u64_to_user_ptr(attr
->batch
.in_batch
);
1374 void __user
*values
= u64_to_user_ptr(attr
->batch
.values
);
1375 void __user
*keys
= u64_to_user_ptr(attr
->batch
.keys
);
1376 void *buf
, *buf_prevkey
, *prev_key
, *key
, *value
;
1377 int err
, retry
= MAP_LOOKUP_RETRIES
;
1378 u32 value_size
, cp
, max_count
;
1380 if (attr
->batch
.elem_flags
& ~BPF_F_LOCK
)
1383 if ((attr
->batch
.elem_flags
& BPF_F_LOCK
) &&
1384 !map_value_has_spin_lock(map
))
1387 value_size
= bpf_map_value_size(map
);
1389 max_count
= attr
->batch
.count
;
1393 if (put_user(0, &uattr
->batch
.count
))
1396 buf_prevkey
= kmalloc(map
->key_size
, GFP_USER
| __GFP_NOWARN
);
1400 buf
= kmalloc(map
->key_size
+ value_size
, GFP_USER
| __GFP_NOWARN
);
1408 if (ubatch
&& copy_from_user(buf_prevkey
, ubatch
, map
->key_size
))
1411 value
= key
+ map
->key_size
;
1413 prev_key
= buf_prevkey
;
1415 for (cp
= 0; cp
< max_count
;) {
1417 err
= map
->ops
->map_get_next_key(map
, prev_key
, key
);
1421 err
= bpf_map_copy_value(map
, key
, value
,
1422 attr
->batch
.elem_flags
);
1424 if (err
== -ENOENT
) {
1436 if (copy_to_user(keys
+ cp
* map
->key_size
, key
,
1441 if (copy_to_user(values
+ cp
* value_size
, value
, value_size
)) {
1447 prev_key
= buf_prevkey
;
1449 swap(prev_key
, key
);
1450 retry
= MAP_LOOKUP_RETRIES
;
1457 if ((copy_to_user(&uattr
->batch
.count
, &cp
, sizeof(cp
)) ||
1458 (cp
&& copy_to_user(uobatch
, prev_key
, map
->key_size
))))
1467 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
1469 static int map_lookup_and_delete_elem(union bpf_attr
*attr
)
1471 void __user
*ukey
= u64_to_user_ptr(attr
->key
);
1472 void __user
*uvalue
= u64_to_user_ptr(attr
->value
);
1473 int ufd
= attr
->map_fd
;
1474 struct bpf_map
*map
;
1480 if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM
))
1484 map
= __bpf_map_get(f
);
1486 return PTR_ERR(map
);
1487 if (!(map_get_sys_perms(map
, f
) & FMODE_CAN_READ
) ||
1488 !(map_get_sys_perms(map
, f
) & FMODE_CAN_WRITE
)) {
1493 key
= __bpf_copy_key(ukey
, map
->key_size
);
1499 value_size
= map
->value_size
;
1502 value
= kmalloc(value_size
, GFP_USER
| __GFP_NOWARN
);
1506 if (map
->map_type
== BPF_MAP_TYPE_QUEUE
||
1507 map
->map_type
== BPF_MAP_TYPE_STACK
) {
1508 err
= map
->ops
->map_pop_elem(map
, value
);
1516 if (copy_to_user(uvalue
, value
, value_size
) != 0) {
1532 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1534 static int map_freeze(const union bpf_attr
*attr
)
1536 int err
= 0, ufd
= attr
->map_fd
;
1537 struct bpf_map
*map
;
1540 if (CHECK_ATTR(BPF_MAP_FREEZE
))
1544 map
= __bpf_map_get(f
);
1546 return PTR_ERR(map
);
1548 if (map
->map_type
== BPF_MAP_TYPE_STRUCT_OPS
) {
1553 mutex_lock(&map
->freeze_mutex
);
1555 if (map
->writecnt
) {
1559 if (READ_ONCE(map
->frozen
)) {
1563 if (!bpf_capable()) {
1568 WRITE_ONCE(map
->frozen
, true);
1570 mutex_unlock(&map
->freeze_mutex
);
1575 static const struct bpf_prog_ops
* const bpf_prog_types
[] = {
1576 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
1577 [_id] = & _name ## _prog_ops,
1578 #define BPF_MAP_TYPE(_id, _ops)
1579 #define BPF_LINK_TYPE(_id, _name)
1580 #include <linux/bpf_types.h>
1581 #undef BPF_PROG_TYPE
1583 #undef BPF_LINK_TYPE
1586 static int find_prog_type(enum bpf_prog_type type
, struct bpf_prog
*prog
)
1588 const struct bpf_prog_ops
*ops
;
1590 if (type
>= ARRAY_SIZE(bpf_prog_types
))
1592 type
= array_index_nospec(type
, ARRAY_SIZE(bpf_prog_types
));
1593 ops
= bpf_prog_types
[type
];
1597 if (!bpf_prog_is_dev_bound(prog
->aux
))
1598 prog
->aux
->ops
= ops
;
1600 prog
->aux
->ops
= &bpf_offload_prog_ops
;
1611 static const char * const bpf_audit_str
[BPF_AUDIT_MAX
] = {
1612 [BPF_AUDIT_LOAD
] = "LOAD",
1613 [BPF_AUDIT_UNLOAD
] = "UNLOAD",
1616 static void bpf_audit_prog(const struct bpf_prog
*prog
, unsigned int op
)
1618 struct audit_context
*ctx
= NULL
;
1619 struct audit_buffer
*ab
;
1621 if (WARN_ON_ONCE(op
>= BPF_AUDIT_MAX
))
1623 if (audit_enabled
== AUDIT_OFF
)
1625 if (op
== BPF_AUDIT_LOAD
)
1626 ctx
= audit_context();
1627 ab
= audit_log_start(ctx
, GFP_ATOMIC
, AUDIT_BPF
);
1630 audit_log_format(ab
, "prog-id=%u op=%s",
1631 prog
->aux
->id
, bpf_audit_str
[op
]);
1635 static int bpf_prog_alloc_id(struct bpf_prog
*prog
)
1639 idr_preload(GFP_KERNEL
);
1640 spin_lock_bh(&prog_idr_lock
);
1641 id
= idr_alloc_cyclic(&prog_idr
, prog
, 1, INT_MAX
, GFP_ATOMIC
);
1644 spin_unlock_bh(&prog_idr_lock
);
1647 /* id is in [1, INT_MAX) */
1648 if (WARN_ON_ONCE(!id
))
1651 return id
> 0 ? 0 : id
;
1654 void bpf_prog_free_id(struct bpf_prog
*prog
, bool do_idr_lock
)
1656 /* cBPF to eBPF migrations are currently not in the idr store.
1657 * Offloaded programs are removed from the store when their device
1658 * disappears - even if someone grabs an fd to them they are unusable,
1659 * simply waiting for refcnt to drop to be freed.
1665 spin_lock_bh(&prog_idr_lock
);
1667 __acquire(&prog_idr_lock
);
1669 idr_remove(&prog_idr
, prog
->aux
->id
);
1673 spin_unlock_bh(&prog_idr_lock
);
1675 __release(&prog_idr_lock
);
1678 static void __bpf_prog_put_rcu(struct rcu_head
*rcu
)
1680 struct bpf_prog_aux
*aux
= container_of(rcu
, struct bpf_prog_aux
, rcu
);
1682 kvfree(aux
->func_info
);
1683 kfree(aux
->func_info_aux
);
1684 free_uid(aux
->user
);
1685 security_bpf_prog_free(aux
);
1686 bpf_prog_free(aux
->prog
);
1689 static void __bpf_prog_put_noref(struct bpf_prog
*prog
, bool deferred
)
1691 bpf_prog_kallsyms_del_all(prog
);
1692 btf_put(prog
->aux
->btf
);
1693 bpf_prog_free_linfo(prog
);
1694 if (prog
->aux
->attach_btf
)
1695 btf_put(prog
->aux
->attach_btf
);
1698 if (prog
->aux
->sleepable
)
1699 call_rcu_tasks_trace(&prog
->aux
->rcu
, __bpf_prog_put_rcu
);
1701 call_rcu(&prog
->aux
->rcu
, __bpf_prog_put_rcu
);
1703 __bpf_prog_put_rcu(&prog
->aux
->rcu
);
1707 static void __bpf_prog_put(struct bpf_prog
*prog
, bool do_idr_lock
)
1709 if (atomic64_dec_and_test(&prog
->aux
->refcnt
)) {
1710 perf_event_bpf_event(prog
, PERF_BPF_EVENT_PROG_UNLOAD
, 0);
1711 bpf_audit_prog(prog
, BPF_AUDIT_UNLOAD
);
1712 /* bpf_prog_free_id() must be called first */
1713 bpf_prog_free_id(prog
, do_idr_lock
);
1714 __bpf_prog_put_noref(prog
, true);
1718 void bpf_prog_put(struct bpf_prog
*prog
)
1720 __bpf_prog_put(prog
, true);
1722 EXPORT_SYMBOL_GPL(bpf_prog_put
);
1724 static int bpf_prog_release(struct inode
*inode
, struct file
*filp
)
1726 struct bpf_prog
*prog
= filp
->private_data
;
1732 static void bpf_prog_get_stats(const struct bpf_prog
*prog
,
1733 struct bpf_prog_stats
*stats
)
1735 u64 nsecs
= 0, cnt
= 0;
1738 for_each_possible_cpu(cpu
) {
1739 const struct bpf_prog_stats
*st
;
1743 st
= per_cpu_ptr(prog
->aux
->stats
, cpu
);
1745 start
= u64_stats_fetch_begin_irq(&st
->syncp
);
1748 } while (u64_stats_fetch_retry_irq(&st
->syncp
, start
));
1752 stats
->nsecs
= nsecs
;
1756 #ifdef CONFIG_PROC_FS
1757 static void bpf_prog_show_fdinfo(struct seq_file
*m
, struct file
*filp
)
1759 const struct bpf_prog
*prog
= filp
->private_data
;
1760 char prog_tag
[sizeof(prog
->tag
) * 2 + 1] = { };
1761 struct bpf_prog_stats stats
;
1763 bpf_prog_get_stats(prog
, &stats
);
1764 bin2hex(prog_tag
, prog
->tag
, sizeof(prog
->tag
));
1771 "run_time_ns:\t%llu\n"
1776 prog
->pages
* 1ULL << PAGE_SHIFT
,
1783 const struct file_operations bpf_prog_fops
= {
1784 #ifdef CONFIG_PROC_FS
1785 .show_fdinfo
= bpf_prog_show_fdinfo
,
1787 .release
= bpf_prog_release
,
1788 .read
= bpf_dummy_read
,
1789 .write
= bpf_dummy_write
,
1792 int bpf_prog_new_fd(struct bpf_prog
*prog
)
1796 ret
= security_bpf_prog(prog
);
1800 return anon_inode_getfd("bpf-prog", &bpf_prog_fops
, prog
,
1801 O_RDWR
| O_CLOEXEC
);
1804 static struct bpf_prog
*____bpf_prog_get(struct fd f
)
1807 return ERR_PTR(-EBADF
);
1808 if (f
.file
->f_op
!= &bpf_prog_fops
) {
1810 return ERR_PTR(-EINVAL
);
1813 return f
.file
->private_data
;
1816 void bpf_prog_add(struct bpf_prog
*prog
, int i
)
1818 atomic64_add(i
, &prog
->aux
->refcnt
);
1820 EXPORT_SYMBOL_GPL(bpf_prog_add
);
1822 void bpf_prog_sub(struct bpf_prog
*prog
, int i
)
1824 /* Only to be used for undoing previous bpf_prog_add() in some
1825 * error path. We still know that another entity in our call
1826 * path holds a reference to the program, thus atomic_sub() can
1827 * be safely used in such cases!
1829 WARN_ON(atomic64_sub_return(i
, &prog
->aux
->refcnt
) == 0);
1831 EXPORT_SYMBOL_GPL(bpf_prog_sub
);
1833 void bpf_prog_inc(struct bpf_prog
*prog
)
1835 atomic64_inc(&prog
->aux
->refcnt
);
1837 EXPORT_SYMBOL_GPL(bpf_prog_inc
);
1839 /* prog_idr_lock should have been held */
1840 struct bpf_prog
*bpf_prog_inc_not_zero(struct bpf_prog
*prog
)
1844 refold
= atomic64_fetch_add_unless(&prog
->aux
->refcnt
, 1, 0);
1847 return ERR_PTR(-ENOENT
);
1851 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero
);
1853 bool bpf_prog_get_ok(struct bpf_prog
*prog
,
1854 enum bpf_prog_type
*attach_type
, bool attach_drv
)
1856 /* not an attachment, just a refcount inc, always allow */
1860 if (prog
->type
!= *attach_type
)
1862 if (bpf_prog_is_dev_bound(prog
->aux
) && !attach_drv
)
1868 static struct bpf_prog
*__bpf_prog_get(u32 ufd
, enum bpf_prog_type
*attach_type
,
1871 struct fd f
= fdget(ufd
);
1872 struct bpf_prog
*prog
;
1874 prog
= ____bpf_prog_get(f
);
1877 if (!bpf_prog_get_ok(prog
, attach_type
, attach_drv
)) {
1878 prog
= ERR_PTR(-EINVAL
);
1888 struct bpf_prog
*bpf_prog_get(u32 ufd
)
1890 return __bpf_prog_get(ufd
, NULL
, false);
1893 struct bpf_prog
*bpf_prog_get_type_dev(u32 ufd
, enum bpf_prog_type type
,
1896 return __bpf_prog_get(ufd
, &type
, attach_drv
);
1898 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev
);
1900 /* Initially all BPF programs could be loaded w/o specifying
1901 * expected_attach_type. Later for some of them specifying expected_attach_type
1902 * at load time became required so that program could be validated properly.
1903 * Programs of types that are allowed to be loaded both w/ and w/o (for
1904 * backward compatibility) expected_attach_type, should have the default attach
1905 * type assigned to expected_attach_type for the latter case, so that it can be
1906 * validated later at attach time.
1908 * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1909 * prog type requires it but has some attach types that have to be backward
1912 static void bpf_prog_load_fixup_attach_type(union bpf_attr
*attr
)
1914 switch (attr
->prog_type
) {
1915 case BPF_PROG_TYPE_CGROUP_SOCK
:
1916 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
1917 * exist so checking for non-zero is the way to go here.
1919 if (!attr
->expected_attach_type
)
1920 attr
->expected_attach_type
=
1921 BPF_CGROUP_INET_SOCK_CREATE
;
1927 bpf_prog_load_check_attach(enum bpf_prog_type prog_type
,
1928 enum bpf_attach_type expected_attach_type
,
1929 struct btf
*attach_btf
, u32 btf_id
,
1930 struct bpf_prog
*dst_prog
)
1933 if (btf_id
> BTF_MAX_TYPE
)
1936 if (!attach_btf
&& !dst_prog
)
1939 switch (prog_type
) {
1940 case BPF_PROG_TYPE_TRACING
:
1941 case BPF_PROG_TYPE_LSM
:
1942 case BPF_PROG_TYPE_STRUCT_OPS
:
1943 case BPF_PROG_TYPE_EXT
:
1950 if (attach_btf
&& (!btf_id
|| dst_prog
))
1953 if (dst_prog
&& prog_type
!= BPF_PROG_TYPE_TRACING
&&
1954 prog_type
!= BPF_PROG_TYPE_EXT
)
1957 switch (prog_type
) {
1958 case BPF_PROG_TYPE_CGROUP_SOCK
:
1959 switch (expected_attach_type
) {
1960 case BPF_CGROUP_INET_SOCK_CREATE
:
1961 case BPF_CGROUP_INET_SOCK_RELEASE
:
1962 case BPF_CGROUP_INET4_POST_BIND
:
1963 case BPF_CGROUP_INET6_POST_BIND
:
1968 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
1969 switch (expected_attach_type
) {
1970 case BPF_CGROUP_INET4_BIND
:
1971 case BPF_CGROUP_INET6_BIND
:
1972 case BPF_CGROUP_INET4_CONNECT
:
1973 case BPF_CGROUP_INET6_CONNECT
:
1974 case BPF_CGROUP_INET4_GETPEERNAME
:
1975 case BPF_CGROUP_INET6_GETPEERNAME
:
1976 case BPF_CGROUP_INET4_GETSOCKNAME
:
1977 case BPF_CGROUP_INET6_GETSOCKNAME
:
1978 case BPF_CGROUP_UDP4_SENDMSG
:
1979 case BPF_CGROUP_UDP6_SENDMSG
:
1980 case BPF_CGROUP_UDP4_RECVMSG
:
1981 case BPF_CGROUP_UDP6_RECVMSG
:
1986 case BPF_PROG_TYPE_CGROUP_SKB
:
1987 switch (expected_attach_type
) {
1988 case BPF_CGROUP_INET_INGRESS
:
1989 case BPF_CGROUP_INET_EGRESS
:
1994 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
1995 switch (expected_attach_type
) {
1996 case BPF_CGROUP_SETSOCKOPT
:
1997 case BPF_CGROUP_GETSOCKOPT
:
2002 case BPF_PROG_TYPE_SK_LOOKUP
:
2003 if (expected_attach_type
== BPF_SK_LOOKUP
)
2006 case BPF_PROG_TYPE_EXT
:
2007 if (expected_attach_type
)
2015 static bool is_net_admin_prog_type(enum bpf_prog_type prog_type
)
2017 switch (prog_type
) {
2018 case BPF_PROG_TYPE_SCHED_CLS
:
2019 case BPF_PROG_TYPE_SCHED_ACT
:
2020 case BPF_PROG_TYPE_XDP
:
2021 case BPF_PROG_TYPE_LWT_IN
:
2022 case BPF_PROG_TYPE_LWT_OUT
:
2023 case BPF_PROG_TYPE_LWT_XMIT
:
2024 case BPF_PROG_TYPE_LWT_SEG6LOCAL
:
2025 case BPF_PROG_TYPE_SK_SKB
:
2026 case BPF_PROG_TYPE_SK_MSG
:
2027 case BPF_PROG_TYPE_LIRC_MODE2
:
2028 case BPF_PROG_TYPE_FLOW_DISSECTOR
:
2029 case BPF_PROG_TYPE_CGROUP_DEVICE
:
2030 case BPF_PROG_TYPE_CGROUP_SOCK
:
2031 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
2032 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
2033 case BPF_PROG_TYPE_CGROUP_SYSCTL
:
2034 case BPF_PROG_TYPE_SOCK_OPS
:
2035 case BPF_PROG_TYPE_EXT
: /* extends any prog */
2037 case BPF_PROG_TYPE_CGROUP_SKB
:
2039 case BPF_PROG_TYPE_SK_REUSEPORT
:
2040 /* equivalent to SOCKET_FILTER. need CAP_BPF only */
2046 static bool is_perfmon_prog_type(enum bpf_prog_type prog_type
)
2048 switch (prog_type
) {
2049 case BPF_PROG_TYPE_KPROBE
:
2050 case BPF_PROG_TYPE_TRACEPOINT
:
2051 case BPF_PROG_TYPE_PERF_EVENT
:
2052 case BPF_PROG_TYPE_RAW_TRACEPOINT
:
2053 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE
:
2054 case BPF_PROG_TYPE_TRACING
:
2055 case BPF_PROG_TYPE_LSM
:
2056 case BPF_PROG_TYPE_STRUCT_OPS
: /* has access to struct sock */
2057 case BPF_PROG_TYPE_EXT
: /* extends any prog */
2064 /* last field in 'union bpf_attr' used by this command */
2065 #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd
2067 static int bpf_prog_load(union bpf_attr
*attr
, union bpf_attr __user
*uattr
)
2069 enum bpf_prog_type type
= attr
->prog_type
;
2070 struct bpf_prog
*prog
, *dst_prog
= NULL
;
2071 struct btf
*attach_btf
= NULL
;
2076 if (CHECK_ATTR(BPF_PROG_LOAD
))
2079 if (attr
->prog_flags
& ~(BPF_F_STRICT_ALIGNMENT
|
2080 BPF_F_ANY_ALIGNMENT
|
2081 BPF_F_TEST_STATE_FREQ
|
2083 BPF_F_TEST_RND_HI32
))
2086 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
) &&
2087 (attr
->prog_flags
& BPF_F_ANY_ALIGNMENT
) &&
2091 /* copy eBPF program license from user space */
2092 if (strncpy_from_user(license
, u64_to_user_ptr(attr
->license
),
2093 sizeof(license
) - 1) < 0)
2095 license
[sizeof(license
) - 1] = 0;
2097 /* eBPF programs must be GPL compatible to use GPL-ed functions */
2098 is_gpl
= license_is_gpl_compatible(license
);
2100 if (attr
->insn_cnt
== 0 ||
2101 attr
->insn_cnt
> (bpf_capable() ? BPF_COMPLEXITY_LIMIT_INSNS
: BPF_MAXINSNS
))
2103 if (type
!= BPF_PROG_TYPE_SOCKET_FILTER
&&
2104 type
!= BPF_PROG_TYPE_CGROUP_SKB
&&
2108 if (is_net_admin_prog_type(type
) && !capable(CAP_NET_ADMIN
) && !capable(CAP_SYS_ADMIN
))
2110 if (is_perfmon_prog_type(type
) && !perfmon_capable())
2113 /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog
2114 * or btf, we need to check which one it is
2116 if (attr
->attach_prog_fd
) {
2117 dst_prog
= bpf_prog_get(attr
->attach_prog_fd
);
2118 if (IS_ERR(dst_prog
)) {
2120 attach_btf
= btf_get_by_fd(attr
->attach_btf_obj_fd
);
2121 if (IS_ERR(attach_btf
))
2123 if (!btf_is_kernel(attach_btf
)) {
2124 /* attaching through specifying bpf_prog's BTF
2125 * objects directly might be supported eventually
2127 btf_put(attach_btf
);
2131 } else if (attr
->attach_btf_id
) {
2132 /* fall back to vmlinux BTF, if BTF type ID is specified */
2133 attach_btf
= bpf_get_btf_vmlinux();
2134 if (IS_ERR(attach_btf
))
2135 return PTR_ERR(attach_btf
);
2138 btf_get(attach_btf
);
2141 bpf_prog_load_fixup_attach_type(attr
);
2142 if (bpf_prog_load_check_attach(type
, attr
->expected_attach_type
,
2143 attach_btf
, attr
->attach_btf_id
,
2146 bpf_prog_put(dst_prog
);
2148 btf_put(attach_btf
);
2152 /* plain bpf_prog allocation */
2153 prog
= bpf_prog_alloc(bpf_prog_size(attr
->insn_cnt
), GFP_USER
);
2156 bpf_prog_put(dst_prog
);
2158 btf_put(attach_btf
);
2162 prog
->expected_attach_type
= attr
->expected_attach_type
;
2163 prog
->aux
->attach_btf
= attach_btf
;
2164 prog
->aux
->attach_btf_id
= attr
->attach_btf_id
;
2165 prog
->aux
->dst_prog
= dst_prog
;
2166 prog
->aux
->offload_requested
= !!attr
->prog_ifindex
;
2167 prog
->aux
->sleepable
= attr
->prog_flags
& BPF_F_SLEEPABLE
;
2169 err
= security_bpf_prog_alloc(prog
->aux
);
2173 prog
->aux
->user
= get_current_user();
2174 prog
->len
= attr
->insn_cnt
;
2177 if (copy_from_user(prog
->insns
, u64_to_user_ptr(attr
->insns
),
2178 bpf_prog_insn_size(prog
)) != 0)
2181 prog
->orig_prog
= NULL
;
2184 atomic64_set(&prog
->aux
->refcnt
, 1);
2185 prog
->gpl_compatible
= is_gpl
? 1 : 0;
2187 if (bpf_prog_is_dev_bound(prog
->aux
)) {
2188 err
= bpf_prog_offload_init(prog
, attr
);
2193 /* find program type: socket_filter vs tracing_filter */
2194 err
= find_prog_type(type
, prog
);
2198 prog
->aux
->load_time
= ktime_get_boottime_ns();
2199 err
= bpf_obj_name_cpy(prog
->aux
->name
, attr
->prog_name
,
2200 sizeof(attr
->prog_name
));
2204 /* run eBPF verifier */
2205 err
= bpf_check(&prog
, attr
, uattr
);
2207 goto free_used_maps
;
2209 prog
= bpf_prog_select_runtime(prog
, &err
);
2211 goto free_used_maps
;
2213 err
= bpf_prog_alloc_id(prog
);
2215 goto free_used_maps
;
2217 /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2218 * effectively publicly exposed. However, retrieving via
2219 * bpf_prog_get_fd_by_id() will take another reference,
2220 * therefore it cannot be gone underneath us.
2222 * Only for the time /after/ successful bpf_prog_new_fd()
2223 * and before returning to userspace, we might just hold
2224 * one reference and any parallel close on that fd could
2225 * rip everything out. Hence, below notifications must
2226 * happen before bpf_prog_new_fd().
2228 * Also, any failure handling from this point onwards must
2229 * be using bpf_prog_put() given the program is exposed.
2231 bpf_prog_kallsyms_add(prog
);
2232 perf_event_bpf_event(prog
, PERF_BPF_EVENT_PROG_LOAD
, 0);
2233 bpf_audit_prog(prog
, BPF_AUDIT_LOAD
);
2235 err
= bpf_prog_new_fd(prog
);
2241 /* In case we have subprogs, we need to wait for a grace
2242 * period before we can tear down JIT memory since symbols
2243 * are already exposed under kallsyms.
2245 __bpf_prog_put_noref(prog
, prog
->aux
->func_cnt
);
2248 free_uid(prog
->aux
->user
);
2249 security_bpf_prog_free(prog
->aux
);
2251 if (prog
->aux
->attach_btf
)
2252 btf_put(prog
->aux
->attach_btf
);
2253 bpf_prog_free(prog
);
2257 #define BPF_OBJ_LAST_FIELD file_flags
2259 static int bpf_obj_pin(const union bpf_attr
*attr
)
2261 if (CHECK_ATTR(BPF_OBJ
) || attr
->file_flags
!= 0)
2264 return bpf_obj_pin_user(attr
->bpf_fd
, u64_to_user_ptr(attr
->pathname
));
2267 static int bpf_obj_get(const union bpf_attr
*attr
)
2269 if (CHECK_ATTR(BPF_OBJ
) || attr
->bpf_fd
!= 0 ||
2270 attr
->file_flags
& ~BPF_OBJ_FLAG_MASK
)
2273 return bpf_obj_get_user(u64_to_user_ptr(attr
->pathname
),
2277 void bpf_link_init(struct bpf_link
*link
, enum bpf_link_type type
,
2278 const struct bpf_link_ops
*ops
, struct bpf_prog
*prog
)
2280 atomic64_set(&link
->refcnt
, 1);
2287 static void bpf_link_free_id(int id
)
2292 spin_lock_bh(&link_idr_lock
);
2293 idr_remove(&link_idr
, id
);
2294 spin_unlock_bh(&link_idr_lock
);
2297 /* Clean up bpf_link and corresponding anon_inode file and FD. After
2298 * anon_inode is created, bpf_link can't be just kfree()'d due to deferred
2299 * anon_inode's release() call. This helper marksbpf_link as
2300 * defunct, releases anon_inode file and puts reserved FD. bpf_prog's refcnt
2301 * is not decremented, it's the responsibility of a calling code that failed
2302 * to complete bpf_link initialization.
2304 void bpf_link_cleanup(struct bpf_link_primer
*primer
)
2306 primer
->link
->prog
= NULL
;
2307 bpf_link_free_id(primer
->id
);
2309 put_unused_fd(primer
->fd
);
2312 void bpf_link_inc(struct bpf_link
*link
)
2314 atomic64_inc(&link
->refcnt
);
2317 /* bpf_link_free is guaranteed to be called from process context */
2318 static void bpf_link_free(struct bpf_link
*link
)
2320 bpf_link_free_id(link
->id
);
2322 /* detach BPF program, clean up used resources */
2323 link
->ops
->release(link
);
2324 bpf_prog_put(link
->prog
);
2326 /* free bpf_link and its containing memory */
2327 link
->ops
->dealloc(link
);
2330 static void bpf_link_put_deferred(struct work_struct
*work
)
2332 struct bpf_link
*link
= container_of(work
, struct bpf_link
, work
);
2334 bpf_link_free(link
);
2337 /* bpf_link_put can be called from atomic context, but ensures that resources
2338 * are freed from process context
2340 void bpf_link_put(struct bpf_link
*link
)
2342 if (!atomic64_dec_and_test(&link
->refcnt
))
2346 INIT_WORK(&link
->work
, bpf_link_put_deferred
);
2347 schedule_work(&link
->work
);
2349 bpf_link_free(link
);
2353 static int bpf_link_release(struct inode
*inode
, struct file
*filp
)
2355 struct bpf_link
*link
= filp
->private_data
;
2361 #ifdef CONFIG_PROC_FS
2362 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
2363 #define BPF_MAP_TYPE(_id, _ops)
2364 #define BPF_LINK_TYPE(_id, _name) [_id] = #_name,
2365 static const char *bpf_link_type_strs
[] = {
2366 [BPF_LINK_TYPE_UNSPEC
] = "<invalid>",
2367 #include <linux/bpf_types.h>
2369 #undef BPF_PROG_TYPE
2371 #undef BPF_LINK_TYPE
2373 static void bpf_link_show_fdinfo(struct seq_file
*m
, struct file
*filp
)
2375 const struct bpf_link
*link
= filp
->private_data
;
2376 const struct bpf_prog
*prog
= link
->prog
;
2377 char prog_tag
[sizeof(prog
->tag
) * 2 + 1] = { };
2379 bin2hex(prog_tag
, prog
->tag
, sizeof(prog
->tag
));
2385 bpf_link_type_strs
[link
->type
],
2389 if (link
->ops
->show_fdinfo
)
2390 link
->ops
->show_fdinfo(link
, m
);
2394 static const struct file_operations bpf_link_fops
= {
2395 #ifdef CONFIG_PROC_FS
2396 .show_fdinfo
= bpf_link_show_fdinfo
,
2398 .release
= bpf_link_release
,
2399 .read
= bpf_dummy_read
,
2400 .write
= bpf_dummy_write
,
2403 static int bpf_link_alloc_id(struct bpf_link
*link
)
2407 idr_preload(GFP_KERNEL
);
2408 spin_lock_bh(&link_idr_lock
);
2409 id
= idr_alloc_cyclic(&link_idr
, link
, 1, INT_MAX
, GFP_ATOMIC
);
2410 spin_unlock_bh(&link_idr_lock
);
2416 /* Prepare bpf_link to be exposed to user-space by allocating anon_inode file,
2417 * reserving unused FD and allocating ID from link_idr. This is to be paired
2418 * with bpf_link_settle() to install FD and ID and expose bpf_link to
2419 * user-space, if bpf_link is successfully attached. If not, bpf_link and
2420 * pre-allocated resources are to be freed with bpf_cleanup() call. All the
2421 * transient state is passed around in struct bpf_link_primer.
2422 * This is preferred way to create and initialize bpf_link, especially when
2423 * there are complicated and expensive operations inbetween creating bpf_link
2424 * itself and attaching it to BPF hook. By using bpf_link_prime() and
2425 * bpf_link_settle() kernel code using bpf_link doesn't have to perform
2426 * expensive (and potentially failing) roll back operations in a rare case
2427 * that file, FD, or ID can't be allocated.
2429 int bpf_link_prime(struct bpf_link
*link
, struct bpf_link_primer
*primer
)
2434 fd
= get_unused_fd_flags(O_CLOEXEC
);
2439 id
= bpf_link_alloc_id(link
);
2445 file
= anon_inode_getfile("bpf_link", &bpf_link_fops
, link
, O_CLOEXEC
);
2447 bpf_link_free_id(id
);
2449 return PTR_ERR(file
);
2452 primer
->link
= link
;
2453 primer
->file
= file
;
2459 int bpf_link_settle(struct bpf_link_primer
*primer
)
2461 /* make bpf_link fetchable by ID */
2462 spin_lock_bh(&link_idr_lock
);
2463 primer
->link
->id
= primer
->id
;
2464 spin_unlock_bh(&link_idr_lock
);
2465 /* make bpf_link fetchable by FD */
2466 fd_install(primer
->fd
, primer
->file
);
2467 /* pass through installed FD */
2471 int bpf_link_new_fd(struct bpf_link
*link
)
2473 return anon_inode_getfd("bpf-link", &bpf_link_fops
, link
, O_CLOEXEC
);
2476 struct bpf_link
*bpf_link_get_from_fd(u32 ufd
)
2478 struct fd f
= fdget(ufd
);
2479 struct bpf_link
*link
;
2482 return ERR_PTR(-EBADF
);
2483 if (f
.file
->f_op
!= &bpf_link_fops
) {
2485 return ERR_PTR(-EINVAL
);
2488 link
= f
.file
->private_data
;
2495 struct bpf_tracing_link
{
2496 struct bpf_link link
;
2497 enum bpf_attach_type attach_type
;
2498 struct bpf_trampoline
*trampoline
;
2499 struct bpf_prog
*tgt_prog
;
2502 static void bpf_tracing_link_release(struct bpf_link
*link
)
2504 struct bpf_tracing_link
*tr_link
=
2505 container_of(link
, struct bpf_tracing_link
, link
);
2507 WARN_ON_ONCE(bpf_trampoline_unlink_prog(link
->prog
,
2508 tr_link
->trampoline
));
2510 bpf_trampoline_put(tr_link
->trampoline
);
2512 /* tgt_prog is NULL if target is a kernel function */
2513 if (tr_link
->tgt_prog
)
2514 bpf_prog_put(tr_link
->tgt_prog
);
2517 static void bpf_tracing_link_dealloc(struct bpf_link
*link
)
2519 struct bpf_tracing_link
*tr_link
=
2520 container_of(link
, struct bpf_tracing_link
, link
);
2525 static void bpf_tracing_link_show_fdinfo(const struct bpf_link
*link
,
2526 struct seq_file
*seq
)
2528 struct bpf_tracing_link
*tr_link
=
2529 container_of(link
, struct bpf_tracing_link
, link
);
2532 "attach_type:\t%d\n",
2533 tr_link
->attach_type
);
2536 static int bpf_tracing_link_fill_link_info(const struct bpf_link
*link
,
2537 struct bpf_link_info
*info
)
2539 struct bpf_tracing_link
*tr_link
=
2540 container_of(link
, struct bpf_tracing_link
, link
);
2542 info
->tracing
.attach_type
= tr_link
->attach_type
;
2547 static const struct bpf_link_ops bpf_tracing_link_lops
= {
2548 .release
= bpf_tracing_link_release
,
2549 .dealloc
= bpf_tracing_link_dealloc
,
2550 .show_fdinfo
= bpf_tracing_link_show_fdinfo
,
2551 .fill_link_info
= bpf_tracing_link_fill_link_info
,
2554 static int bpf_tracing_prog_attach(struct bpf_prog
*prog
,
2558 struct bpf_link_primer link_primer
;
2559 struct bpf_prog
*tgt_prog
= NULL
;
2560 struct bpf_trampoline
*tr
= NULL
;
2561 struct bpf_tracing_link
*link
;
2565 switch (prog
->type
) {
2566 case BPF_PROG_TYPE_TRACING
:
2567 if (prog
->expected_attach_type
!= BPF_TRACE_FENTRY
&&
2568 prog
->expected_attach_type
!= BPF_TRACE_FEXIT
&&
2569 prog
->expected_attach_type
!= BPF_MODIFY_RETURN
) {
2574 case BPF_PROG_TYPE_EXT
:
2575 if (prog
->expected_attach_type
!= 0) {
2580 case BPF_PROG_TYPE_LSM
:
2581 if (prog
->expected_attach_type
!= BPF_LSM_MAC
) {
2591 if (!!tgt_prog_fd
!= !!btf_id
) {
2597 /* For now we only allow new targets for BPF_PROG_TYPE_EXT */
2598 if (prog
->type
!= BPF_PROG_TYPE_EXT
) {
2603 tgt_prog
= bpf_prog_get(tgt_prog_fd
);
2604 if (IS_ERR(tgt_prog
)) {
2605 err
= PTR_ERR(tgt_prog
);
2610 key
= bpf_trampoline_compute_key(tgt_prog
, NULL
, btf_id
);
2613 link
= kzalloc(sizeof(*link
), GFP_USER
);
2618 bpf_link_init(&link
->link
, BPF_LINK_TYPE_TRACING
,
2619 &bpf_tracing_link_lops
, prog
);
2620 link
->attach_type
= prog
->expected_attach_type
;
2622 mutex_lock(&prog
->aux
->dst_mutex
);
2624 /* There are a few possible cases here:
2626 * - if prog->aux->dst_trampoline is set, the program was just loaded
2627 * and not yet attached to anything, so we can use the values stored
2630 * - if prog->aux->dst_trampoline is NULL, the program has already been
2631 * attached to a target and its initial target was cleared (below)
2633 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
2634 * target_btf_id using the link_create API.
2636 * - if tgt_prog == NULL when this function was called using the old
2637 * raw_tracepoint_open API, and we need a target from prog->aux
2639 * The combination of no saved target in prog->aux, and no target
2640 * specified on load is illegal, and we reject that here.
2642 if (!prog
->aux
->dst_trampoline
&& !tgt_prog
) {
2647 if (!prog
->aux
->dst_trampoline
||
2648 (key
&& key
!= prog
->aux
->dst_trampoline
->key
)) {
2649 /* If there is no saved target, or the specified target is
2650 * different from the destination specified at load time, we
2651 * need a new trampoline and a check for compatibility
2653 struct bpf_attach_target_info tgt_info
= {};
2655 err
= bpf_check_attach_target(NULL
, prog
, tgt_prog
, btf_id
,
2660 tr
= bpf_trampoline_get(key
, &tgt_info
);
2666 /* The caller didn't specify a target, or the target was the
2667 * same as the destination supplied during program load. This
2668 * means we can reuse the trampoline and reference from program
2669 * load time, and there is no need to allocate a new one. This
2670 * can only happen once for any program, as the saved values in
2671 * prog->aux are cleared below.
2673 tr
= prog
->aux
->dst_trampoline
;
2674 tgt_prog
= prog
->aux
->dst_prog
;
2677 err
= bpf_link_prime(&link
->link
, &link_primer
);
2681 err
= bpf_trampoline_link_prog(prog
, tr
);
2683 bpf_link_cleanup(&link_primer
);
2688 link
->tgt_prog
= tgt_prog
;
2689 link
->trampoline
= tr
;
2691 /* Always clear the trampoline and target prog from prog->aux to make
2692 * sure the original attach destination is not kept alive after a
2693 * program is (re-)attached to another target.
2695 if (prog
->aux
->dst_prog
&&
2696 (tgt_prog_fd
|| tr
!= prog
->aux
->dst_trampoline
))
2697 /* got extra prog ref from syscall, or attaching to different prog */
2698 bpf_prog_put(prog
->aux
->dst_prog
);
2699 if (prog
->aux
->dst_trampoline
&& tr
!= prog
->aux
->dst_trampoline
)
2700 /* we allocated a new trampoline, so free the old one */
2701 bpf_trampoline_put(prog
->aux
->dst_trampoline
);
2703 prog
->aux
->dst_prog
= NULL
;
2704 prog
->aux
->dst_trampoline
= NULL
;
2705 mutex_unlock(&prog
->aux
->dst_mutex
);
2707 return bpf_link_settle(&link_primer
);
2709 if (tr
&& tr
!= prog
->aux
->dst_trampoline
)
2710 bpf_trampoline_put(tr
);
2711 mutex_unlock(&prog
->aux
->dst_mutex
);
2714 if (tgt_prog_fd
&& tgt_prog
)
2715 bpf_prog_put(tgt_prog
);
2720 struct bpf_raw_tp_link
{
2721 struct bpf_link link
;
2722 struct bpf_raw_event_map
*btp
;
2725 static void bpf_raw_tp_link_release(struct bpf_link
*link
)
2727 struct bpf_raw_tp_link
*raw_tp
=
2728 container_of(link
, struct bpf_raw_tp_link
, link
);
2730 bpf_probe_unregister(raw_tp
->btp
, raw_tp
->link
.prog
);
2731 bpf_put_raw_tracepoint(raw_tp
->btp
);
2734 static void bpf_raw_tp_link_dealloc(struct bpf_link
*link
)
2736 struct bpf_raw_tp_link
*raw_tp
=
2737 container_of(link
, struct bpf_raw_tp_link
, link
);
2742 static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link
*link
,
2743 struct seq_file
*seq
)
2745 struct bpf_raw_tp_link
*raw_tp_link
=
2746 container_of(link
, struct bpf_raw_tp_link
, link
);
2750 raw_tp_link
->btp
->tp
->name
);
2753 static int bpf_raw_tp_link_fill_link_info(const struct bpf_link
*link
,
2754 struct bpf_link_info
*info
)
2756 struct bpf_raw_tp_link
*raw_tp_link
=
2757 container_of(link
, struct bpf_raw_tp_link
, link
);
2758 char __user
*ubuf
= u64_to_user_ptr(info
->raw_tracepoint
.tp_name
);
2759 const char *tp_name
= raw_tp_link
->btp
->tp
->name
;
2760 u32 ulen
= info
->raw_tracepoint
.tp_name_len
;
2761 size_t tp_len
= strlen(tp_name
);
2766 info
->raw_tracepoint
.tp_name_len
= tp_len
+ 1;
2771 if (ulen
>= tp_len
+ 1) {
2772 if (copy_to_user(ubuf
, tp_name
, tp_len
+ 1))
2777 if (copy_to_user(ubuf
, tp_name
, ulen
- 1))
2779 if (put_user(zero
, ubuf
+ ulen
- 1))
2787 static const struct bpf_link_ops bpf_raw_tp_link_lops
= {
2788 .release
= bpf_raw_tp_link_release
,
2789 .dealloc
= bpf_raw_tp_link_dealloc
,
2790 .show_fdinfo
= bpf_raw_tp_link_show_fdinfo
,
2791 .fill_link_info
= bpf_raw_tp_link_fill_link_info
,
2794 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
2796 static int bpf_raw_tracepoint_open(const union bpf_attr
*attr
)
2798 struct bpf_link_primer link_primer
;
2799 struct bpf_raw_tp_link
*link
;
2800 struct bpf_raw_event_map
*btp
;
2801 struct bpf_prog
*prog
;
2802 const char *tp_name
;
2806 if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN
))
2809 prog
= bpf_prog_get(attr
->raw_tracepoint
.prog_fd
);
2811 return PTR_ERR(prog
);
2813 switch (prog
->type
) {
2814 case BPF_PROG_TYPE_TRACING
:
2815 case BPF_PROG_TYPE_EXT
:
2816 case BPF_PROG_TYPE_LSM
:
2817 if (attr
->raw_tracepoint
.name
) {
2818 /* The attach point for this category of programs
2819 * should be specified via btf_id during program load.
2824 if (prog
->type
== BPF_PROG_TYPE_TRACING
&&
2825 prog
->expected_attach_type
== BPF_TRACE_RAW_TP
) {
2826 tp_name
= prog
->aux
->attach_func_name
;
2829 return bpf_tracing_prog_attach(prog
, 0, 0);
2830 case BPF_PROG_TYPE_RAW_TRACEPOINT
:
2831 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE
:
2832 if (strncpy_from_user(buf
,
2833 u64_to_user_ptr(attr
->raw_tracepoint
.name
),
2834 sizeof(buf
) - 1) < 0) {
2838 buf
[sizeof(buf
) - 1] = 0;
2846 btp
= bpf_get_raw_tracepoint(tp_name
);
2852 link
= kzalloc(sizeof(*link
), GFP_USER
);
2857 bpf_link_init(&link
->link
, BPF_LINK_TYPE_RAW_TRACEPOINT
,
2858 &bpf_raw_tp_link_lops
, prog
);
2861 err
= bpf_link_prime(&link
->link
, &link_primer
);
2867 err
= bpf_probe_register(link
->btp
, prog
);
2869 bpf_link_cleanup(&link_primer
);
2873 return bpf_link_settle(&link_primer
);
2876 bpf_put_raw_tracepoint(btp
);
2882 static int bpf_prog_attach_check_attach_type(const struct bpf_prog
*prog
,
2883 enum bpf_attach_type attach_type
)
2885 switch (prog
->type
) {
2886 case BPF_PROG_TYPE_CGROUP_SOCK
:
2887 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
2888 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
2889 case BPF_PROG_TYPE_SK_LOOKUP
:
2890 return attach_type
== prog
->expected_attach_type
? 0 : -EINVAL
;
2891 case BPF_PROG_TYPE_CGROUP_SKB
:
2892 if (!capable(CAP_NET_ADMIN
))
2893 /* cg-skb progs can be loaded by unpriv user.
2894 * check permissions at attach time.
2897 return prog
->enforce_expected_attach_type
&&
2898 prog
->expected_attach_type
!= attach_type
?
2905 static enum bpf_prog_type
2906 attach_type_to_prog_type(enum bpf_attach_type attach_type
)
2908 switch (attach_type
) {
2909 case BPF_CGROUP_INET_INGRESS
:
2910 case BPF_CGROUP_INET_EGRESS
:
2911 return BPF_PROG_TYPE_CGROUP_SKB
;
2912 case BPF_CGROUP_INET_SOCK_CREATE
:
2913 case BPF_CGROUP_INET_SOCK_RELEASE
:
2914 case BPF_CGROUP_INET4_POST_BIND
:
2915 case BPF_CGROUP_INET6_POST_BIND
:
2916 return BPF_PROG_TYPE_CGROUP_SOCK
;
2917 case BPF_CGROUP_INET4_BIND
:
2918 case BPF_CGROUP_INET6_BIND
:
2919 case BPF_CGROUP_INET4_CONNECT
:
2920 case BPF_CGROUP_INET6_CONNECT
:
2921 case BPF_CGROUP_INET4_GETPEERNAME
:
2922 case BPF_CGROUP_INET6_GETPEERNAME
:
2923 case BPF_CGROUP_INET4_GETSOCKNAME
:
2924 case BPF_CGROUP_INET6_GETSOCKNAME
:
2925 case BPF_CGROUP_UDP4_SENDMSG
:
2926 case BPF_CGROUP_UDP6_SENDMSG
:
2927 case BPF_CGROUP_UDP4_RECVMSG
:
2928 case BPF_CGROUP_UDP6_RECVMSG
:
2929 return BPF_PROG_TYPE_CGROUP_SOCK_ADDR
;
2930 case BPF_CGROUP_SOCK_OPS
:
2931 return BPF_PROG_TYPE_SOCK_OPS
;
2932 case BPF_CGROUP_DEVICE
:
2933 return BPF_PROG_TYPE_CGROUP_DEVICE
;
2934 case BPF_SK_MSG_VERDICT
:
2935 return BPF_PROG_TYPE_SK_MSG
;
2936 case BPF_SK_SKB_STREAM_PARSER
:
2937 case BPF_SK_SKB_STREAM_VERDICT
:
2938 return BPF_PROG_TYPE_SK_SKB
;
2939 case BPF_LIRC_MODE2
:
2940 return BPF_PROG_TYPE_LIRC_MODE2
;
2941 case BPF_FLOW_DISSECTOR
:
2942 return BPF_PROG_TYPE_FLOW_DISSECTOR
;
2943 case BPF_CGROUP_SYSCTL
:
2944 return BPF_PROG_TYPE_CGROUP_SYSCTL
;
2945 case BPF_CGROUP_GETSOCKOPT
:
2946 case BPF_CGROUP_SETSOCKOPT
:
2947 return BPF_PROG_TYPE_CGROUP_SOCKOPT
;
2948 case BPF_TRACE_ITER
:
2949 return BPF_PROG_TYPE_TRACING
;
2951 return BPF_PROG_TYPE_SK_LOOKUP
;
2953 return BPF_PROG_TYPE_XDP
;
2955 return BPF_PROG_TYPE_UNSPEC
;
2959 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd
2961 #define BPF_F_ATTACH_MASK \
2962 (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE)
2964 static int bpf_prog_attach(const union bpf_attr
*attr
)
2966 enum bpf_prog_type ptype
;
2967 struct bpf_prog
*prog
;
2970 if (CHECK_ATTR(BPF_PROG_ATTACH
))
2973 if (attr
->attach_flags
& ~BPF_F_ATTACH_MASK
)
2976 ptype
= attach_type_to_prog_type(attr
->attach_type
);
2977 if (ptype
== BPF_PROG_TYPE_UNSPEC
)
2980 prog
= bpf_prog_get_type(attr
->attach_bpf_fd
, ptype
);
2982 return PTR_ERR(prog
);
2984 if (bpf_prog_attach_check_attach_type(prog
, attr
->attach_type
)) {
2990 case BPF_PROG_TYPE_SK_SKB
:
2991 case BPF_PROG_TYPE_SK_MSG
:
2992 ret
= sock_map_get_from_fd(attr
, prog
);
2994 case BPF_PROG_TYPE_LIRC_MODE2
:
2995 ret
= lirc_prog_attach(attr
, prog
);
2997 case BPF_PROG_TYPE_FLOW_DISSECTOR
:
2998 ret
= netns_bpf_prog_attach(attr
, prog
);
3000 case BPF_PROG_TYPE_CGROUP_DEVICE
:
3001 case BPF_PROG_TYPE_CGROUP_SKB
:
3002 case BPF_PROG_TYPE_CGROUP_SOCK
:
3003 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
3004 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
3005 case BPF_PROG_TYPE_CGROUP_SYSCTL
:
3006 case BPF_PROG_TYPE_SOCK_OPS
:
3007 ret
= cgroup_bpf_prog_attach(attr
, ptype
, prog
);
3018 #define BPF_PROG_DETACH_LAST_FIELD attach_type
3020 static int bpf_prog_detach(const union bpf_attr
*attr
)
3022 enum bpf_prog_type ptype
;
3024 if (CHECK_ATTR(BPF_PROG_DETACH
))
3027 ptype
= attach_type_to_prog_type(attr
->attach_type
);
3030 case BPF_PROG_TYPE_SK_MSG
:
3031 case BPF_PROG_TYPE_SK_SKB
:
3032 return sock_map_prog_detach(attr
, ptype
);
3033 case BPF_PROG_TYPE_LIRC_MODE2
:
3034 return lirc_prog_detach(attr
);
3035 case BPF_PROG_TYPE_FLOW_DISSECTOR
:
3036 return netns_bpf_prog_detach(attr
, ptype
);
3037 case BPF_PROG_TYPE_CGROUP_DEVICE
:
3038 case BPF_PROG_TYPE_CGROUP_SKB
:
3039 case BPF_PROG_TYPE_CGROUP_SOCK
:
3040 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
3041 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
3042 case BPF_PROG_TYPE_CGROUP_SYSCTL
:
3043 case BPF_PROG_TYPE_SOCK_OPS
:
3044 return cgroup_bpf_prog_detach(attr
, ptype
);
3050 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
3052 static int bpf_prog_query(const union bpf_attr
*attr
,
3053 union bpf_attr __user
*uattr
)
3055 if (!capable(CAP_NET_ADMIN
))
3057 if (CHECK_ATTR(BPF_PROG_QUERY
))
3059 if (attr
->query
.query_flags
& ~BPF_F_QUERY_EFFECTIVE
)
3062 switch (attr
->query
.attach_type
) {
3063 case BPF_CGROUP_INET_INGRESS
:
3064 case BPF_CGROUP_INET_EGRESS
:
3065 case BPF_CGROUP_INET_SOCK_CREATE
:
3066 case BPF_CGROUP_INET_SOCK_RELEASE
:
3067 case BPF_CGROUP_INET4_BIND
:
3068 case BPF_CGROUP_INET6_BIND
:
3069 case BPF_CGROUP_INET4_POST_BIND
:
3070 case BPF_CGROUP_INET6_POST_BIND
:
3071 case BPF_CGROUP_INET4_CONNECT
:
3072 case BPF_CGROUP_INET6_CONNECT
:
3073 case BPF_CGROUP_INET4_GETPEERNAME
:
3074 case BPF_CGROUP_INET6_GETPEERNAME
:
3075 case BPF_CGROUP_INET4_GETSOCKNAME
:
3076 case BPF_CGROUP_INET6_GETSOCKNAME
:
3077 case BPF_CGROUP_UDP4_SENDMSG
:
3078 case BPF_CGROUP_UDP6_SENDMSG
:
3079 case BPF_CGROUP_UDP4_RECVMSG
:
3080 case BPF_CGROUP_UDP6_RECVMSG
:
3081 case BPF_CGROUP_SOCK_OPS
:
3082 case BPF_CGROUP_DEVICE
:
3083 case BPF_CGROUP_SYSCTL
:
3084 case BPF_CGROUP_GETSOCKOPT
:
3085 case BPF_CGROUP_SETSOCKOPT
:
3086 return cgroup_bpf_prog_query(attr
, uattr
);
3087 case BPF_LIRC_MODE2
:
3088 return lirc_prog_query(attr
, uattr
);
3089 case BPF_FLOW_DISSECTOR
:
3091 return netns_bpf_prog_query(attr
, uattr
);
3097 #define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
3099 static int bpf_prog_test_run(const union bpf_attr
*attr
,
3100 union bpf_attr __user
*uattr
)
3102 struct bpf_prog
*prog
;
3103 int ret
= -ENOTSUPP
;
3105 if (CHECK_ATTR(BPF_PROG_TEST_RUN
))
3108 if ((attr
->test
.ctx_size_in
&& !attr
->test
.ctx_in
) ||
3109 (!attr
->test
.ctx_size_in
&& attr
->test
.ctx_in
))
3112 if ((attr
->test
.ctx_size_out
&& !attr
->test
.ctx_out
) ||
3113 (!attr
->test
.ctx_size_out
&& attr
->test
.ctx_out
))
3116 prog
= bpf_prog_get(attr
->test
.prog_fd
);
3118 return PTR_ERR(prog
);
3120 if (prog
->aux
->ops
->test_run
)
3121 ret
= prog
->aux
->ops
->test_run(prog
, attr
, uattr
);
3127 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
3129 static int bpf_obj_get_next_id(const union bpf_attr
*attr
,
3130 union bpf_attr __user
*uattr
,
3134 u32 next_id
= attr
->start_id
;
3137 if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID
) || next_id
>= INT_MAX
)
3140 if (!capable(CAP_SYS_ADMIN
))
3145 if (!idr_get_next(idr
, &next_id
))
3147 spin_unlock_bh(lock
);
3150 err
= put_user(next_id
, &uattr
->next_id
);
3155 struct bpf_map
*bpf_map_get_curr_or_next(u32
*id
)
3157 struct bpf_map
*map
;
3159 spin_lock_bh(&map_idr_lock
);
3161 map
= idr_get_next(&map_idr
, id
);
3163 map
= __bpf_map_inc_not_zero(map
, false);
3169 spin_unlock_bh(&map_idr_lock
);
3174 struct bpf_prog
*bpf_prog_get_curr_or_next(u32
*id
)
3176 struct bpf_prog
*prog
;
3178 spin_lock_bh(&prog_idr_lock
);
3180 prog
= idr_get_next(&prog_idr
, id
);
3182 prog
= bpf_prog_inc_not_zero(prog
);
3188 spin_unlock_bh(&prog_idr_lock
);
3193 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
3195 struct bpf_prog
*bpf_prog_by_id(u32 id
)
3197 struct bpf_prog
*prog
;
3200 return ERR_PTR(-ENOENT
);
3202 spin_lock_bh(&prog_idr_lock
);
3203 prog
= idr_find(&prog_idr
, id
);
3205 prog
= bpf_prog_inc_not_zero(prog
);
3207 prog
= ERR_PTR(-ENOENT
);
3208 spin_unlock_bh(&prog_idr_lock
);
3212 static int bpf_prog_get_fd_by_id(const union bpf_attr
*attr
)
3214 struct bpf_prog
*prog
;
3215 u32 id
= attr
->prog_id
;
3218 if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID
))
3221 if (!capable(CAP_SYS_ADMIN
))
3224 prog
= bpf_prog_by_id(id
);
3226 return PTR_ERR(prog
);
3228 fd
= bpf_prog_new_fd(prog
);
3235 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
3237 static int bpf_map_get_fd_by_id(const union bpf_attr
*attr
)
3239 struct bpf_map
*map
;
3240 u32 id
= attr
->map_id
;
3244 if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID
) ||
3245 attr
->open_flags
& ~BPF_OBJ_FLAG_MASK
)
3248 if (!capable(CAP_SYS_ADMIN
))
3251 f_flags
= bpf_get_file_flag(attr
->open_flags
);
3255 spin_lock_bh(&map_idr_lock
);
3256 map
= idr_find(&map_idr
, id
);
3258 map
= __bpf_map_inc_not_zero(map
, true);
3260 map
= ERR_PTR(-ENOENT
);
3261 spin_unlock_bh(&map_idr_lock
);
3264 return PTR_ERR(map
);
3266 fd
= bpf_map_new_fd(map
, f_flags
);
3268 bpf_map_put_with_uref(map
);
3273 static const struct bpf_map
*bpf_map_from_imm(const struct bpf_prog
*prog
,
3274 unsigned long addr
, u32
*off
,
3277 const struct bpf_map
*map
;
3280 mutex_lock(&prog
->aux
->used_maps_mutex
);
3281 for (i
= 0, *off
= 0; i
< prog
->aux
->used_map_cnt
; i
++) {
3282 map
= prog
->aux
->used_maps
[i
];
3283 if (map
== (void *)addr
) {
3284 *type
= BPF_PSEUDO_MAP_FD
;
3287 if (!map
->ops
->map_direct_value_meta
)
3289 if (!map
->ops
->map_direct_value_meta(map
, addr
, off
)) {
3290 *type
= BPF_PSEUDO_MAP_VALUE
;
3297 mutex_unlock(&prog
->aux
->used_maps_mutex
);
3301 static struct bpf_insn
*bpf_insn_prepare_dump(const struct bpf_prog
*prog
,
3302 const struct cred
*f_cred
)
3304 const struct bpf_map
*map
;
3305 struct bpf_insn
*insns
;
3311 insns
= kmemdup(prog
->insnsi
, bpf_prog_insn_size(prog
),
3316 for (i
= 0; i
< prog
->len
; i
++) {
3317 code
= insns
[i
].code
;
3319 if (code
== (BPF_JMP
| BPF_TAIL_CALL
)) {
3320 insns
[i
].code
= BPF_JMP
| BPF_CALL
;
3321 insns
[i
].imm
= BPF_FUNC_tail_call
;
3324 if (code
== (BPF_JMP
| BPF_CALL
) ||
3325 code
== (BPF_JMP
| BPF_CALL_ARGS
)) {
3326 if (code
== (BPF_JMP
| BPF_CALL_ARGS
))
3327 insns
[i
].code
= BPF_JMP
| BPF_CALL
;
3328 if (!bpf_dump_raw_ok(f_cred
))
3332 if (BPF_CLASS(code
) == BPF_LDX
&& BPF_MODE(code
) == BPF_PROBE_MEM
) {
3333 insns
[i
].code
= BPF_LDX
| BPF_SIZE(code
) | BPF_MEM
;
3337 if (code
!= (BPF_LD
| BPF_IMM
| BPF_DW
))
3340 imm
= ((u64
)insns
[i
+ 1].imm
<< 32) | (u32
)insns
[i
].imm
;
3341 map
= bpf_map_from_imm(prog
, imm
, &off
, &type
);
3343 insns
[i
].src_reg
= type
;
3344 insns
[i
].imm
= map
->id
;
3345 insns
[i
+ 1].imm
= off
;
3353 static int set_info_rec_size(struct bpf_prog_info
*info
)
3356 * Ensure info.*_rec_size is the same as kernel expected size
3360 * Only allow zero *_rec_size if both _rec_size and _cnt are
3361 * zero. In this case, the kernel will set the expected
3362 * _rec_size back to the info.
3365 if ((info
->nr_func_info
|| info
->func_info_rec_size
) &&
3366 info
->func_info_rec_size
!= sizeof(struct bpf_func_info
))
3369 if ((info
->nr_line_info
|| info
->line_info_rec_size
) &&
3370 info
->line_info_rec_size
!= sizeof(struct bpf_line_info
))
3373 if ((info
->nr_jited_line_info
|| info
->jited_line_info_rec_size
) &&
3374 info
->jited_line_info_rec_size
!= sizeof(__u64
))
3377 info
->func_info_rec_size
= sizeof(struct bpf_func_info
);
3378 info
->line_info_rec_size
= sizeof(struct bpf_line_info
);
3379 info
->jited_line_info_rec_size
= sizeof(__u64
);
3384 static int bpf_prog_get_info_by_fd(struct file
*file
,
3385 struct bpf_prog
*prog
,
3386 const union bpf_attr
*attr
,
3387 union bpf_attr __user
*uattr
)
3389 struct bpf_prog_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
3390 struct bpf_prog_info info
;
3391 u32 info_len
= attr
->info
.info_len
;
3392 struct bpf_prog_stats stats
;
3393 char __user
*uinsns
;
3397 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(info
), info_len
);
3400 info_len
= min_t(u32
, sizeof(info
), info_len
);
3402 memset(&info
, 0, sizeof(info
));
3403 if (copy_from_user(&info
, uinfo
, info_len
))
3406 info
.type
= prog
->type
;
3407 info
.id
= prog
->aux
->id
;
3408 info
.load_time
= prog
->aux
->load_time
;
3409 info
.created_by_uid
= from_kuid_munged(current_user_ns(),
3410 prog
->aux
->user
->uid
);
3411 info
.gpl_compatible
= prog
->gpl_compatible
;
3413 memcpy(info
.tag
, prog
->tag
, sizeof(prog
->tag
));
3414 memcpy(info
.name
, prog
->aux
->name
, sizeof(prog
->aux
->name
));
3416 mutex_lock(&prog
->aux
->used_maps_mutex
);
3417 ulen
= info
.nr_map_ids
;
3418 info
.nr_map_ids
= prog
->aux
->used_map_cnt
;
3419 ulen
= min_t(u32
, info
.nr_map_ids
, ulen
);
3421 u32 __user
*user_map_ids
= u64_to_user_ptr(info
.map_ids
);
3424 for (i
= 0; i
< ulen
; i
++)
3425 if (put_user(prog
->aux
->used_maps
[i
]->id
,
3426 &user_map_ids
[i
])) {
3427 mutex_unlock(&prog
->aux
->used_maps_mutex
);
3431 mutex_unlock(&prog
->aux
->used_maps_mutex
);
3433 err
= set_info_rec_size(&info
);
3437 bpf_prog_get_stats(prog
, &stats
);
3438 info
.run_time_ns
= stats
.nsecs
;
3439 info
.run_cnt
= stats
.cnt
;
3441 if (!bpf_capable()) {
3442 info
.jited_prog_len
= 0;
3443 info
.xlated_prog_len
= 0;
3444 info
.nr_jited_ksyms
= 0;
3445 info
.nr_jited_func_lens
= 0;
3446 info
.nr_func_info
= 0;
3447 info
.nr_line_info
= 0;
3448 info
.nr_jited_line_info
= 0;
3452 ulen
= info
.xlated_prog_len
;
3453 info
.xlated_prog_len
= bpf_prog_insn_size(prog
);
3454 if (info
.xlated_prog_len
&& ulen
) {
3455 struct bpf_insn
*insns_sanitized
;
3458 if (prog
->blinded
&& !bpf_dump_raw_ok(file
->f_cred
)) {
3459 info
.xlated_prog_insns
= 0;
3462 insns_sanitized
= bpf_insn_prepare_dump(prog
, file
->f_cred
);
3463 if (!insns_sanitized
)
3465 uinsns
= u64_to_user_ptr(info
.xlated_prog_insns
);
3466 ulen
= min_t(u32
, info
.xlated_prog_len
, ulen
);
3467 fault
= copy_to_user(uinsns
, insns_sanitized
, ulen
);
3468 kfree(insns_sanitized
);
3473 if (bpf_prog_is_dev_bound(prog
->aux
)) {
3474 err
= bpf_prog_offload_info_fill(&info
, prog
);
3480 /* NOTE: the following code is supposed to be skipped for offload.
3481 * bpf_prog_offload_info_fill() is the place to fill similar fields
3484 ulen
= info
.jited_prog_len
;
3485 if (prog
->aux
->func_cnt
) {
3488 info
.jited_prog_len
= 0;
3489 for (i
= 0; i
< prog
->aux
->func_cnt
; i
++)
3490 info
.jited_prog_len
+= prog
->aux
->func
[i
]->jited_len
;
3492 info
.jited_prog_len
= prog
->jited_len
;
3495 if (info
.jited_prog_len
&& ulen
) {
3496 if (bpf_dump_raw_ok(file
->f_cred
)) {
3497 uinsns
= u64_to_user_ptr(info
.jited_prog_insns
);
3498 ulen
= min_t(u32
, info
.jited_prog_len
, ulen
);
3500 /* for multi-function programs, copy the JITed
3501 * instructions for all the functions
3503 if (prog
->aux
->func_cnt
) {
3508 for (i
= 0; i
< prog
->aux
->func_cnt
; i
++) {
3509 len
= prog
->aux
->func
[i
]->jited_len
;
3510 len
= min_t(u32
, len
, free
);
3511 img
= (u8
*) prog
->aux
->func
[i
]->bpf_func
;
3512 if (copy_to_user(uinsns
, img
, len
))
3520 if (copy_to_user(uinsns
, prog
->bpf_func
, ulen
))
3524 info
.jited_prog_insns
= 0;
3528 ulen
= info
.nr_jited_ksyms
;
3529 info
.nr_jited_ksyms
= prog
->aux
->func_cnt
? : 1;
3531 if (bpf_dump_raw_ok(file
->f_cred
)) {
3532 unsigned long ksym_addr
;
3533 u64 __user
*user_ksyms
;
3536 /* copy the address of the kernel symbol
3537 * corresponding to each function
3539 ulen
= min_t(u32
, info
.nr_jited_ksyms
, ulen
);
3540 user_ksyms
= u64_to_user_ptr(info
.jited_ksyms
);
3541 if (prog
->aux
->func_cnt
) {
3542 for (i
= 0; i
< ulen
; i
++) {
3543 ksym_addr
= (unsigned long)
3544 prog
->aux
->func
[i
]->bpf_func
;
3545 if (put_user((u64
) ksym_addr
,
3550 ksym_addr
= (unsigned long) prog
->bpf_func
;
3551 if (put_user((u64
) ksym_addr
, &user_ksyms
[0]))
3555 info
.jited_ksyms
= 0;
3559 ulen
= info
.nr_jited_func_lens
;
3560 info
.nr_jited_func_lens
= prog
->aux
->func_cnt
? : 1;
3562 if (bpf_dump_raw_ok(file
->f_cred
)) {
3563 u32 __user
*user_lens
;
3566 /* copy the JITed image lengths for each function */
3567 ulen
= min_t(u32
, info
.nr_jited_func_lens
, ulen
);
3568 user_lens
= u64_to_user_ptr(info
.jited_func_lens
);
3569 if (prog
->aux
->func_cnt
) {
3570 for (i
= 0; i
< ulen
; i
++) {
3572 prog
->aux
->func
[i
]->jited_len
;
3573 if (put_user(func_len
, &user_lens
[i
]))
3577 func_len
= prog
->jited_len
;
3578 if (put_user(func_len
, &user_lens
[0]))
3582 info
.jited_func_lens
= 0;
3587 info
.btf_id
= btf_obj_id(prog
->aux
->btf
);
3589 ulen
= info
.nr_func_info
;
3590 info
.nr_func_info
= prog
->aux
->func_info_cnt
;
3591 if (info
.nr_func_info
&& ulen
) {
3592 char __user
*user_finfo
;
3594 user_finfo
= u64_to_user_ptr(info
.func_info
);
3595 ulen
= min_t(u32
, info
.nr_func_info
, ulen
);
3596 if (copy_to_user(user_finfo
, prog
->aux
->func_info
,
3597 info
.func_info_rec_size
* ulen
))
3601 ulen
= info
.nr_line_info
;
3602 info
.nr_line_info
= prog
->aux
->nr_linfo
;
3603 if (info
.nr_line_info
&& ulen
) {
3604 __u8 __user
*user_linfo
;
3606 user_linfo
= u64_to_user_ptr(info
.line_info
);
3607 ulen
= min_t(u32
, info
.nr_line_info
, ulen
);
3608 if (copy_to_user(user_linfo
, prog
->aux
->linfo
,
3609 info
.line_info_rec_size
* ulen
))
3613 ulen
= info
.nr_jited_line_info
;
3614 if (prog
->aux
->jited_linfo
)
3615 info
.nr_jited_line_info
= prog
->aux
->nr_linfo
;
3617 info
.nr_jited_line_info
= 0;
3618 if (info
.nr_jited_line_info
&& ulen
) {
3619 if (bpf_dump_raw_ok(file
->f_cred
)) {
3620 __u64 __user
*user_linfo
;
3623 user_linfo
= u64_to_user_ptr(info
.jited_line_info
);
3624 ulen
= min_t(u32
, info
.nr_jited_line_info
, ulen
);
3625 for (i
= 0; i
< ulen
; i
++) {
3626 if (put_user((__u64
)(long)prog
->aux
->jited_linfo
[i
],
3631 info
.jited_line_info
= 0;
3635 ulen
= info
.nr_prog_tags
;
3636 info
.nr_prog_tags
= prog
->aux
->func_cnt
? : 1;
3638 __u8
__user (*user_prog_tags
)[BPF_TAG_SIZE
];
3641 user_prog_tags
= u64_to_user_ptr(info
.prog_tags
);
3642 ulen
= min_t(u32
, info
.nr_prog_tags
, ulen
);
3643 if (prog
->aux
->func_cnt
) {
3644 for (i
= 0; i
< ulen
; i
++) {
3645 if (copy_to_user(user_prog_tags
[i
],
3646 prog
->aux
->func
[i
]->tag
,
3651 if (copy_to_user(user_prog_tags
[0],
3652 prog
->tag
, BPF_TAG_SIZE
))
3658 if (copy_to_user(uinfo
, &info
, info_len
) ||
3659 put_user(info_len
, &uattr
->info
.info_len
))
3665 static int bpf_map_get_info_by_fd(struct file
*file
,
3666 struct bpf_map
*map
,
3667 const union bpf_attr
*attr
,
3668 union bpf_attr __user
*uattr
)
3670 struct bpf_map_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
3671 struct bpf_map_info info
;
3672 u32 info_len
= attr
->info
.info_len
;
3675 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(info
), info_len
);
3678 info_len
= min_t(u32
, sizeof(info
), info_len
);
3680 memset(&info
, 0, sizeof(info
));
3681 info
.type
= map
->map_type
;
3683 info
.key_size
= map
->key_size
;
3684 info
.value_size
= map
->value_size
;
3685 info
.max_entries
= map
->max_entries
;
3686 info
.map_flags
= map
->map_flags
;
3687 memcpy(info
.name
, map
->name
, sizeof(map
->name
));
3690 info
.btf_id
= btf_obj_id(map
->btf
);
3691 info
.btf_key_type_id
= map
->btf_key_type_id
;
3692 info
.btf_value_type_id
= map
->btf_value_type_id
;
3694 info
.btf_vmlinux_value_type_id
= map
->btf_vmlinux_value_type_id
;
3696 if (bpf_map_is_dev_bound(map
)) {
3697 err
= bpf_map_offload_info_fill(&info
, map
);
3702 if (copy_to_user(uinfo
, &info
, info_len
) ||
3703 put_user(info_len
, &uattr
->info
.info_len
))
3709 static int bpf_btf_get_info_by_fd(struct file
*file
,
3711 const union bpf_attr
*attr
,
3712 union bpf_attr __user
*uattr
)
3714 struct bpf_btf_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
3715 u32 info_len
= attr
->info
.info_len
;
3718 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(*uinfo
), info_len
);
3722 return btf_get_info_by_fd(btf
, attr
, uattr
);
3725 static int bpf_link_get_info_by_fd(struct file
*file
,
3726 struct bpf_link
*link
,
3727 const union bpf_attr
*attr
,
3728 union bpf_attr __user
*uattr
)
3730 struct bpf_link_info __user
*uinfo
= u64_to_user_ptr(attr
->info
.info
);
3731 struct bpf_link_info info
;
3732 u32 info_len
= attr
->info
.info_len
;
3735 err
= bpf_check_uarg_tail_zero(uinfo
, sizeof(info
), info_len
);
3738 info_len
= min_t(u32
, sizeof(info
), info_len
);
3740 memset(&info
, 0, sizeof(info
));
3741 if (copy_from_user(&info
, uinfo
, info_len
))
3744 info
.type
= link
->type
;
3746 info
.prog_id
= link
->prog
->aux
->id
;
3748 if (link
->ops
->fill_link_info
) {
3749 err
= link
->ops
->fill_link_info(link
, &info
);
3754 if (copy_to_user(uinfo
, &info
, info_len
) ||
3755 put_user(info_len
, &uattr
->info
.info_len
))
3762 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
3764 static int bpf_obj_get_info_by_fd(const union bpf_attr
*attr
,
3765 union bpf_attr __user
*uattr
)
3767 int ufd
= attr
->info
.bpf_fd
;
3771 if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD
))
3778 if (f
.file
->f_op
== &bpf_prog_fops
)
3779 err
= bpf_prog_get_info_by_fd(f
.file
, f
.file
->private_data
, attr
,
3781 else if (f
.file
->f_op
== &bpf_map_fops
)
3782 err
= bpf_map_get_info_by_fd(f
.file
, f
.file
->private_data
, attr
,
3784 else if (f
.file
->f_op
== &btf_fops
)
3785 err
= bpf_btf_get_info_by_fd(f
.file
, f
.file
->private_data
, attr
, uattr
);
3786 else if (f
.file
->f_op
== &bpf_link_fops
)
3787 err
= bpf_link_get_info_by_fd(f
.file
, f
.file
->private_data
,
3796 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
3798 static int bpf_btf_load(const union bpf_attr
*attr
)
3800 if (CHECK_ATTR(BPF_BTF_LOAD
))
3806 return btf_new_fd(attr
);
3809 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
3811 static int bpf_btf_get_fd_by_id(const union bpf_attr
*attr
)
3813 if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID
))
3816 if (!capable(CAP_SYS_ADMIN
))
3819 return btf_get_fd_by_id(attr
->btf_id
);
3822 static int bpf_task_fd_query_copy(const union bpf_attr
*attr
,
3823 union bpf_attr __user
*uattr
,
3824 u32 prog_id
, u32 fd_type
,
3825 const char *buf
, u64 probe_offset
,
3828 char __user
*ubuf
= u64_to_user_ptr(attr
->task_fd_query
.buf
);
3829 u32 len
= buf
? strlen(buf
) : 0, input_len
;
3832 if (put_user(len
, &uattr
->task_fd_query
.buf_len
))
3834 input_len
= attr
->task_fd_query
.buf_len
;
3835 if (input_len
&& ubuf
) {
3837 /* nothing to copy, just make ubuf NULL terminated */
3840 if (put_user(zero
, ubuf
))
3842 } else if (input_len
>= len
+ 1) {
3843 /* ubuf can hold the string with NULL terminator */
3844 if (copy_to_user(ubuf
, buf
, len
+ 1))
3847 /* ubuf cannot hold the string with NULL terminator,
3848 * do a partial copy with NULL terminator.
3853 if (copy_to_user(ubuf
, buf
, input_len
- 1))
3855 if (put_user(zero
, ubuf
+ input_len
- 1))
3860 if (put_user(prog_id
, &uattr
->task_fd_query
.prog_id
) ||
3861 put_user(fd_type
, &uattr
->task_fd_query
.fd_type
) ||
3862 put_user(probe_offset
, &uattr
->task_fd_query
.probe_offset
) ||
3863 put_user(probe_addr
, &uattr
->task_fd_query
.probe_addr
))
3869 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
3871 static int bpf_task_fd_query(const union bpf_attr
*attr
,
3872 union bpf_attr __user
*uattr
)
3874 pid_t pid
= attr
->task_fd_query
.pid
;
3875 u32 fd
= attr
->task_fd_query
.fd
;
3876 const struct perf_event
*event
;
3877 struct files_struct
*files
;
3878 struct task_struct
*task
;
3882 if (CHECK_ATTR(BPF_TASK_FD_QUERY
))
3885 if (!capable(CAP_SYS_ADMIN
))
3888 if (attr
->task_fd_query
.flags
!= 0)
3891 task
= get_pid_task(find_vpid(pid
), PIDTYPE_PID
);
3895 files
= get_files_struct(task
);
3896 put_task_struct(task
);
3901 spin_lock(&files
->file_lock
);
3902 file
= fcheck_files(files
, fd
);
3907 spin_unlock(&files
->file_lock
);
3908 put_files_struct(files
);
3913 if (file
->f_op
== &bpf_link_fops
) {
3914 struct bpf_link
*link
= file
->private_data
;
3916 if (link
->ops
== &bpf_raw_tp_link_lops
) {
3917 struct bpf_raw_tp_link
*raw_tp
=
3918 container_of(link
, struct bpf_raw_tp_link
, link
);
3919 struct bpf_raw_event_map
*btp
= raw_tp
->btp
;
3921 err
= bpf_task_fd_query_copy(attr
, uattr
,
3922 raw_tp
->link
.prog
->aux
->id
,
3923 BPF_FD_TYPE_RAW_TRACEPOINT
,
3924 btp
->tp
->name
, 0, 0);
3930 event
= perf_get_event(file
);
3931 if (!IS_ERR(event
)) {
3932 u64 probe_offset
, probe_addr
;
3933 u32 prog_id
, fd_type
;
3936 err
= bpf_get_perf_event_info(event
, &prog_id
, &fd_type
,
3937 &buf
, &probe_offset
,
3940 err
= bpf_task_fd_query_copy(attr
, uattr
, prog_id
,
3955 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
3957 #define BPF_DO_BATCH(fn) \
3963 err = fn(map, attr, uattr); \
3966 static int bpf_map_do_batch(const union bpf_attr
*attr
,
3967 union bpf_attr __user
*uattr
,
3970 struct bpf_map
*map
;
3974 if (CHECK_ATTR(BPF_MAP_BATCH
))
3977 ufd
= attr
->batch
.map_fd
;
3979 map
= __bpf_map_get(f
);
3981 return PTR_ERR(map
);
3983 if ((cmd
== BPF_MAP_LOOKUP_BATCH
||
3984 cmd
== BPF_MAP_LOOKUP_AND_DELETE_BATCH
) &&
3985 !(map_get_sys_perms(map
, f
) & FMODE_CAN_READ
)) {
3990 if (cmd
!= BPF_MAP_LOOKUP_BATCH
&&
3991 !(map_get_sys_perms(map
, f
) & FMODE_CAN_WRITE
)) {
3996 if (cmd
== BPF_MAP_LOOKUP_BATCH
)
3997 BPF_DO_BATCH(map
->ops
->map_lookup_batch
);
3998 else if (cmd
== BPF_MAP_LOOKUP_AND_DELETE_BATCH
)
3999 BPF_DO_BATCH(map
->ops
->map_lookup_and_delete_batch
);
4000 else if (cmd
== BPF_MAP_UPDATE_BATCH
)
4001 BPF_DO_BATCH(map
->ops
->map_update_batch
);
4003 BPF_DO_BATCH(map
->ops
->map_delete_batch
);
4010 static int tracing_bpf_link_attach(const union bpf_attr
*attr
, struct bpf_prog
*prog
)
4012 if (attr
->link_create
.attach_type
!= prog
->expected_attach_type
)
4015 if (prog
->expected_attach_type
== BPF_TRACE_ITER
)
4016 return bpf_iter_link_attach(attr
, prog
);
4017 else if (prog
->type
== BPF_PROG_TYPE_EXT
)
4018 return bpf_tracing_prog_attach(prog
,
4019 attr
->link_create
.target_fd
,
4020 attr
->link_create
.target_btf_id
);
4024 #define BPF_LINK_CREATE_LAST_FIELD link_create.iter_info_len
4025 static int link_create(union bpf_attr
*attr
)
4027 enum bpf_prog_type ptype
;
4028 struct bpf_prog
*prog
;
4031 if (CHECK_ATTR(BPF_LINK_CREATE
))
4034 prog
= bpf_prog_get(attr
->link_create
.prog_fd
);
4036 return PTR_ERR(prog
);
4038 ret
= bpf_prog_attach_check_attach_type(prog
,
4039 attr
->link_create
.attach_type
);
4043 if (prog
->type
== BPF_PROG_TYPE_EXT
) {
4044 ret
= tracing_bpf_link_attach(attr
, prog
);
4048 ptype
= attach_type_to_prog_type(attr
->link_create
.attach_type
);
4049 if (ptype
== BPF_PROG_TYPE_UNSPEC
|| ptype
!= prog
->type
) {
4055 case BPF_PROG_TYPE_CGROUP_SKB
:
4056 case BPF_PROG_TYPE_CGROUP_SOCK
:
4057 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR
:
4058 case BPF_PROG_TYPE_SOCK_OPS
:
4059 case BPF_PROG_TYPE_CGROUP_DEVICE
:
4060 case BPF_PROG_TYPE_CGROUP_SYSCTL
:
4061 case BPF_PROG_TYPE_CGROUP_SOCKOPT
:
4062 ret
= cgroup_bpf_link_attach(attr
, prog
);
4064 case BPF_PROG_TYPE_TRACING
:
4065 ret
= tracing_bpf_link_attach(attr
, prog
);
4067 case BPF_PROG_TYPE_FLOW_DISSECTOR
:
4068 case BPF_PROG_TYPE_SK_LOOKUP
:
4069 ret
= netns_bpf_link_create(attr
, prog
);
4072 case BPF_PROG_TYPE_XDP
:
4073 ret
= bpf_xdp_link_attach(attr
, prog
);
4086 #define BPF_LINK_UPDATE_LAST_FIELD link_update.old_prog_fd
4088 static int link_update(union bpf_attr
*attr
)
4090 struct bpf_prog
*old_prog
= NULL
, *new_prog
;
4091 struct bpf_link
*link
;
4095 if (CHECK_ATTR(BPF_LINK_UPDATE
))
4098 flags
= attr
->link_update
.flags
;
4099 if (flags
& ~BPF_F_REPLACE
)
4102 link
= bpf_link_get_from_fd(attr
->link_update
.link_fd
);
4104 return PTR_ERR(link
);
4106 new_prog
= bpf_prog_get(attr
->link_update
.new_prog_fd
);
4107 if (IS_ERR(new_prog
)) {
4108 ret
= PTR_ERR(new_prog
);
4112 if (flags
& BPF_F_REPLACE
) {
4113 old_prog
= bpf_prog_get(attr
->link_update
.old_prog_fd
);
4114 if (IS_ERR(old_prog
)) {
4115 ret
= PTR_ERR(old_prog
);
4119 } else if (attr
->link_update
.old_prog_fd
) {
4124 if (link
->ops
->update_prog
)
4125 ret
= link
->ops
->update_prog(link
, new_prog
, old_prog
);
4131 bpf_prog_put(old_prog
);
4133 bpf_prog_put(new_prog
);
4139 #define BPF_LINK_DETACH_LAST_FIELD link_detach.link_fd
4141 static int link_detach(union bpf_attr
*attr
)
4143 struct bpf_link
*link
;
4146 if (CHECK_ATTR(BPF_LINK_DETACH
))
4149 link
= bpf_link_get_from_fd(attr
->link_detach
.link_fd
);
4151 return PTR_ERR(link
);
4153 if (link
->ops
->detach
)
4154 ret
= link
->ops
->detach(link
);
4162 static struct bpf_link
*bpf_link_inc_not_zero(struct bpf_link
*link
)
4164 return atomic64_fetch_add_unless(&link
->refcnt
, 1, 0) ? link
: ERR_PTR(-ENOENT
);
4167 struct bpf_link
*bpf_link_by_id(u32 id
)
4169 struct bpf_link
*link
;
4172 return ERR_PTR(-ENOENT
);
4174 spin_lock_bh(&link_idr_lock
);
4175 /* before link is "settled", ID is 0, pretend it doesn't exist yet */
4176 link
= idr_find(&link_idr
, id
);
4179 link
= bpf_link_inc_not_zero(link
);
4181 link
= ERR_PTR(-EAGAIN
);
4183 link
= ERR_PTR(-ENOENT
);
4185 spin_unlock_bh(&link_idr_lock
);
4189 #define BPF_LINK_GET_FD_BY_ID_LAST_FIELD link_id
4191 static int bpf_link_get_fd_by_id(const union bpf_attr
*attr
)
4193 struct bpf_link
*link
;
4194 u32 id
= attr
->link_id
;
4197 if (CHECK_ATTR(BPF_LINK_GET_FD_BY_ID
))
4200 if (!capable(CAP_SYS_ADMIN
))
4203 link
= bpf_link_by_id(id
);
4205 return PTR_ERR(link
);
4207 fd
= bpf_link_new_fd(link
);
4214 DEFINE_MUTEX(bpf_stats_enabled_mutex
);
4216 static int bpf_stats_release(struct inode
*inode
, struct file
*file
)
4218 mutex_lock(&bpf_stats_enabled_mutex
);
4219 static_key_slow_dec(&bpf_stats_enabled_key
.key
);
4220 mutex_unlock(&bpf_stats_enabled_mutex
);
4224 static const struct file_operations bpf_stats_fops
= {
4225 .release
= bpf_stats_release
,
4228 static int bpf_enable_runtime_stats(void)
4232 mutex_lock(&bpf_stats_enabled_mutex
);
4234 /* Set a very high limit to avoid overflow */
4235 if (static_key_count(&bpf_stats_enabled_key
.key
) > INT_MAX
/ 2) {
4236 mutex_unlock(&bpf_stats_enabled_mutex
);
4240 fd
= anon_inode_getfd("bpf-stats", &bpf_stats_fops
, NULL
, O_CLOEXEC
);
4242 static_key_slow_inc(&bpf_stats_enabled_key
.key
);
4244 mutex_unlock(&bpf_stats_enabled_mutex
);
4248 #define BPF_ENABLE_STATS_LAST_FIELD enable_stats.type
4250 static int bpf_enable_stats(union bpf_attr
*attr
)
4253 if (CHECK_ATTR(BPF_ENABLE_STATS
))
4256 if (!capable(CAP_SYS_ADMIN
))
4259 switch (attr
->enable_stats
.type
) {
4260 case BPF_STATS_RUN_TIME
:
4261 return bpf_enable_runtime_stats();
4268 #define BPF_ITER_CREATE_LAST_FIELD iter_create.flags
4270 static int bpf_iter_create(union bpf_attr
*attr
)
4272 struct bpf_link
*link
;
4275 if (CHECK_ATTR(BPF_ITER_CREATE
))
4278 if (attr
->iter_create
.flags
)
4281 link
= bpf_link_get_from_fd(attr
->iter_create
.link_fd
);
4283 return PTR_ERR(link
);
4285 err
= bpf_iter_new_fd(link
);
4291 #define BPF_PROG_BIND_MAP_LAST_FIELD prog_bind_map.flags
4293 static int bpf_prog_bind_map(union bpf_attr
*attr
)
4295 struct bpf_prog
*prog
;
4296 struct bpf_map
*map
;
4297 struct bpf_map
**used_maps_old
, **used_maps_new
;
4300 if (CHECK_ATTR(BPF_PROG_BIND_MAP
))
4303 if (attr
->prog_bind_map
.flags
)
4306 prog
= bpf_prog_get(attr
->prog_bind_map
.prog_fd
);
4308 return PTR_ERR(prog
);
4310 map
= bpf_map_get(attr
->prog_bind_map
.map_fd
);
4316 mutex_lock(&prog
->aux
->used_maps_mutex
);
4318 used_maps_old
= prog
->aux
->used_maps
;
4320 for (i
= 0; i
< prog
->aux
->used_map_cnt
; i
++)
4321 if (used_maps_old
[i
] == map
) {
4326 used_maps_new
= kmalloc_array(prog
->aux
->used_map_cnt
+ 1,
4327 sizeof(used_maps_new
[0]),
4329 if (!used_maps_new
) {
4334 memcpy(used_maps_new
, used_maps_old
,
4335 sizeof(used_maps_old
[0]) * prog
->aux
->used_map_cnt
);
4336 used_maps_new
[prog
->aux
->used_map_cnt
] = map
;
4338 prog
->aux
->used_map_cnt
++;
4339 prog
->aux
->used_maps
= used_maps_new
;
4341 kfree(used_maps_old
);
4344 mutex_unlock(&prog
->aux
->used_maps_mutex
);
4353 SYSCALL_DEFINE3(bpf
, int, cmd
, union bpf_attr __user
*, uattr
, unsigned int, size
)
4355 union bpf_attr attr
;
4358 if (sysctl_unprivileged_bpf_disabled
&& !bpf_capable())
4361 err
= bpf_check_uarg_tail_zero(uattr
, sizeof(attr
), size
);
4364 size
= min_t(u32
, size
, sizeof(attr
));
4366 /* copy attributes from user space, may be less than sizeof(bpf_attr) */
4367 memset(&attr
, 0, sizeof(attr
));
4368 if (copy_from_user(&attr
, uattr
, size
) != 0)
4371 err
= security_bpf(cmd
, &attr
, size
);
4376 case BPF_MAP_CREATE
:
4377 err
= map_create(&attr
);
4379 case BPF_MAP_LOOKUP_ELEM
:
4380 err
= map_lookup_elem(&attr
);
4382 case BPF_MAP_UPDATE_ELEM
:
4383 err
= map_update_elem(&attr
);
4385 case BPF_MAP_DELETE_ELEM
:
4386 err
= map_delete_elem(&attr
);
4388 case BPF_MAP_GET_NEXT_KEY
:
4389 err
= map_get_next_key(&attr
);
4391 case BPF_MAP_FREEZE
:
4392 err
= map_freeze(&attr
);
4395 err
= bpf_prog_load(&attr
, uattr
);
4398 err
= bpf_obj_pin(&attr
);
4401 err
= bpf_obj_get(&attr
);
4403 case BPF_PROG_ATTACH
:
4404 err
= bpf_prog_attach(&attr
);
4406 case BPF_PROG_DETACH
:
4407 err
= bpf_prog_detach(&attr
);
4409 case BPF_PROG_QUERY
:
4410 err
= bpf_prog_query(&attr
, uattr
);
4412 case BPF_PROG_TEST_RUN
:
4413 err
= bpf_prog_test_run(&attr
, uattr
);
4415 case BPF_PROG_GET_NEXT_ID
:
4416 err
= bpf_obj_get_next_id(&attr
, uattr
,
4417 &prog_idr
, &prog_idr_lock
);
4419 case BPF_MAP_GET_NEXT_ID
:
4420 err
= bpf_obj_get_next_id(&attr
, uattr
,
4421 &map_idr
, &map_idr_lock
);
4423 case BPF_BTF_GET_NEXT_ID
:
4424 err
= bpf_obj_get_next_id(&attr
, uattr
,
4425 &btf_idr
, &btf_idr_lock
);
4427 case BPF_PROG_GET_FD_BY_ID
:
4428 err
= bpf_prog_get_fd_by_id(&attr
);
4430 case BPF_MAP_GET_FD_BY_ID
:
4431 err
= bpf_map_get_fd_by_id(&attr
);
4433 case BPF_OBJ_GET_INFO_BY_FD
:
4434 err
= bpf_obj_get_info_by_fd(&attr
, uattr
);
4436 case BPF_RAW_TRACEPOINT_OPEN
:
4437 err
= bpf_raw_tracepoint_open(&attr
);
4440 err
= bpf_btf_load(&attr
);
4442 case BPF_BTF_GET_FD_BY_ID
:
4443 err
= bpf_btf_get_fd_by_id(&attr
);
4445 case BPF_TASK_FD_QUERY
:
4446 err
= bpf_task_fd_query(&attr
, uattr
);
4448 case BPF_MAP_LOOKUP_AND_DELETE_ELEM
:
4449 err
= map_lookup_and_delete_elem(&attr
);
4451 case BPF_MAP_LOOKUP_BATCH
:
4452 err
= bpf_map_do_batch(&attr
, uattr
, BPF_MAP_LOOKUP_BATCH
);
4454 case BPF_MAP_LOOKUP_AND_DELETE_BATCH
:
4455 err
= bpf_map_do_batch(&attr
, uattr
,
4456 BPF_MAP_LOOKUP_AND_DELETE_BATCH
);
4458 case BPF_MAP_UPDATE_BATCH
:
4459 err
= bpf_map_do_batch(&attr
, uattr
, BPF_MAP_UPDATE_BATCH
);
4461 case BPF_MAP_DELETE_BATCH
:
4462 err
= bpf_map_do_batch(&attr
, uattr
, BPF_MAP_DELETE_BATCH
);
4464 case BPF_LINK_CREATE
:
4465 err
= link_create(&attr
);
4467 case BPF_LINK_UPDATE
:
4468 err
= link_update(&attr
);
4470 case BPF_LINK_GET_FD_BY_ID
:
4471 err
= bpf_link_get_fd_by_id(&attr
);
4473 case BPF_LINK_GET_NEXT_ID
:
4474 err
= bpf_obj_get_next_id(&attr
, uattr
,
4475 &link_idr
, &link_idr_lock
);
4477 case BPF_ENABLE_STATS
:
4478 err
= bpf_enable_stats(&attr
);
4480 case BPF_ITER_CREATE
:
4481 err
= bpf_iter_create(&attr
);
4483 case BPF_LINK_DETACH
:
4484 err
= link_detach(&attr
);
4486 case BPF_PROG_BIND_MAP
:
4487 err
= bpf_prog_bind_map(&attr
);