1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2019 Facebook */
3 #include <linux/hash.h>
5 #include <linux/filter.h>
6 #include <linux/ftrace.h>
8 /* btf_vmlinux has ~22k attachable functions. 1k htab is enough. */
9 #define TRAMPOLINE_HASH_BITS 10
10 #define TRAMPOLINE_TABLE_SIZE (1 << TRAMPOLINE_HASH_BITS)
12 static struct hlist_head trampoline_table
[TRAMPOLINE_TABLE_SIZE
];
14 /* serializes access to trampoline_table */
15 static DEFINE_MUTEX(trampoline_mutex
);
17 struct bpf_trampoline
*bpf_trampoline_lookup(u64 key
)
19 struct bpf_trampoline
*tr
;
20 struct hlist_head
*head
;
24 mutex_lock(&trampoline_mutex
);
25 head
= &trampoline_table
[hash_64(key
, TRAMPOLINE_HASH_BITS
)];
26 hlist_for_each_entry(tr
, head
, hlist
) {
28 refcount_inc(&tr
->refcnt
);
32 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
36 /* is_root was checked earlier. No need for bpf_jit_charge_modmem() */
37 image
= bpf_jit_alloc_exec(PAGE_SIZE
);
45 INIT_HLIST_NODE(&tr
->hlist
);
46 hlist_add_head(&tr
->hlist
, head
);
47 refcount_set(&tr
->refcnt
, 1);
48 mutex_init(&tr
->mutex
);
49 for (i
= 0; i
< BPF_TRAMP_MAX
; i
++)
50 INIT_HLIST_HEAD(&tr
->progs_hlist
[i
]);
52 set_vm_flush_reset_perms(image
);
53 /* Keep image as writeable. The alternative is to keep flipping ro/rw
54 * everytime new program is attached or detached.
56 set_memory_x((long)image
, 1);
59 mutex_unlock(&trampoline_mutex
);
63 static int is_ftrace_location(void *ip
)
67 addr
= ftrace_location((long)ip
);
70 if (WARN_ON_ONCE(addr
!= (long)ip
))
75 static int unregister_fentry(struct bpf_trampoline
*tr
, void *old_addr
)
77 void *ip
= tr
->func
.addr
;
80 if (tr
->func
.ftrace_managed
)
81 ret
= unregister_ftrace_direct((long)ip
, (long)old_addr
);
83 ret
= bpf_arch_text_poke(ip
, BPF_MOD_CALL
, old_addr
, NULL
);
87 static int modify_fentry(struct bpf_trampoline
*tr
, void *old_addr
, void *new_addr
)
89 void *ip
= tr
->func
.addr
;
92 if (tr
->func
.ftrace_managed
)
93 ret
= modify_ftrace_direct((long)ip
, (long)old_addr
, (long)new_addr
);
95 ret
= bpf_arch_text_poke(ip
, BPF_MOD_CALL
, old_addr
, new_addr
);
99 /* first time registering */
100 static int register_fentry(struct bpf_trampoline
*tr
, void *new_addr
)
102 void *ip
= tr
->func
.addr
;
105 ret
= is_ftrace_location(ip
);
108 tr
->func
.ftrace_managed
= ret
;
110 if (tr
->func
.ftrace_managed
)
111 ret
= register_ftrace_direct((long)ip
, (long)new_addr
);
113 ret
= bpf_arch_text_poke(ip
, BPF_MOD_CALL
, NULL
, new_addr
);
117 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
118 * bytes on x86. Pick a number to fit into PAGE_SIZE / 2
120 #define BPF_MAX_TRAMP_PROGS 40
122 static int bpf_trampoline_update(struct bpf_trampoline
*tr
)
124 void *old_image
= tr
->image
+ ((tr
->selector
+ 1) & 1) * PAGE_SIZE
/2;
125 void *new_image
= tr
->image
+ (tr
->selector
& 1) * PAGE_SIZE
/2;
126 struct bpf_prog
*progs_to_run
[BPF_MAX_TRAMP_PROGS
];
127 int fentry_cnt
= tr
->progs_cnt
[BPF_TRAMP_FENTRY
];
128 int fexit_cnt
= tr
->progs_cnt
[BPF_TRAMP_FEXIT
];
129 struct bpf_prog
**progs
, **fentry
, **fexit
;
130 u32 flags
= BPF_TRAMP_F_RESTORE_REGS
;
131 struct bpf_prog_aux
*aux
;
134 if (fentry_cnt
+ fexit_cnt
== 0) {
135 err
= unregister_fentry(tr
, old_image
);
140 /* populate fentry progs */
141 fentry
= progs
= progs_to_run
;
142 hlist_for_each_entry(aux
, &tr
->progs_hlist
[BPF_TRAMP_FENTRY
], tramp_hlist
)
143 *progs
++ = aux
->prog
;
145 /* populate fexit progs */
147 hlist_for_each_entry(aux
, &tr
->progs_hlist
[BPF_TRAMP_FEXIT
], tramp_hlist
)
148 *progs
++ = aux
->prog
;
151 flags
= BPF_TRAMP_F_CALL_ORIG
| BPF_TRAMP_F_SKIP_FRAME
;
153 err
= arch_prepare_bpf_trampoline(new_image
, &tr
->func
.model
, flags
,
161 /* progs already running at this address */
162 err
= modify_fentry(tr
, old_image
, new_image
);
164 /* first time registering */
165 err
= register_fentry(tr
, new_image
);
173 static enum bpf_tramp_prog_type
bpf_attach_type_to_tramp(enum bpf_attach_type t
)
176 case BPF_TRACE_FENTRY
:
177 return BPF_TRAMP_FENTRY
;
179 return BPF_TRAMP_FEXIT
;
183 int bpf_trampoline_link_prog(struct bpf_prog
*prog
)
185 enum bpf_tramp_prog_type kind
;
186 struct bpf_trampoline
*tr
;
189 tr
= prog
->aux
->trampoline
;
190 kind
= bpf_attach_type_to_tramp(prog
->expected_attach_type
);
191 mutex_lock(&tr
->mutex
);
192 if (tr
->progs_cnt
[BPF_TRAMP_FENTRY
] + tr
->progs_cnt
[BPF_TRAMP_FEXIT
]
193 >= BPF_MAX_TRAMP_PROGS
) {
197 if (!hlist_unhashed(&prog
->aux
->tramp_hlist
)) {
198 /* prog already linked */
202 hlist_add_head(&prog
->aux
->tramp_hlist
, &tr
->progs_hlist
[kind
]);
203 tr
->progs_cnt
[kind
]++;
204 err
= bpf_trampoline_update(prog
->aux
->trampoline
);
206 hlist_del(&prog
->aux
->tramp_hlist
);
207 tr
->progs_cnt
[kind
]--;
210 mutex_unlock(&tr
->mutex
);
214 /* bpf_trampoline_unlink_prog() should never fail. */
215 int bpf_trampoline_unlink_prog(struct bpf_prog
*prog
)
217 enum bpf_tramp_prog_type kind
;
218 struct bpf_trampoline
*tr
;
221 tr
= prog
->aux
->trampoline
;
222 kind
= bpf_attach_type_to_tramp(prog
->expected_attach_type
);
223 mutex_lock(&tr
->mutex
);
224 hlist_del(&prog
->aux
->tramp_hlist
);
225 tr
->progs_cnt
[kind
]--;
226 err
= bpf_trampoline_update(prog
->aux
->trampoline
);
227 mutex_unlock(&tr
->mutex
);
231 void bpf_trampoline_put(struct bpf_trampoline
*tr
)
235 mutex_lock(&trampoline_mutex
);
236 if (!refcount_dec_and_test(&tr
->refcnt
))
238 WARN_ON_ONCE(mutex_is_locked(&tr
->mutex
));
239 if (WARN_ON_ONCE(!hlist_empty(&tr
->progs_hlist
[BPF_TRAMP_FENTRY
])))
241 if (WARN_ON_ONCE(!hlist_empty(&tr
->progs_hlist
[BPF_TRAMP_FEXIT
])))
243 bpf_jit_free_exec(tr
->image
);
244 hlist_del(&tr
->hlist
);
247 mutex_unlock(&trampoline_mutex
);
250 /* The logic is similar to BPF_PROG_RUN, but with explicit rcu and preempt that
251 * are needed for trampoline. The macro is split into
252 * call _bpf_prog_enter
253 * call prog->bpf_func
254 * call __bpf_prog_exit
256 u64 notrace
__bpf_prog_enter(void)
262 if (static_branch_unlikely(&bpf_stats_enabled_key
))
263 start
= sched_clock();
267 void notrace
__bpf_prog_exit(struct bpf_prog
*prog
, u64 start
)
269 struct bpf_prog_stats
*stats
;
271 if (static_branch_unlikely(&bpf_stats_enabled_key
) &&
272 /* static_key could be enabled in __bpf_prog_enter
273 * and disabled in __bpf_prog_exit.
275 * Hence check that 'start' is not zero.
278 stats
= this_cpu_ptr(prog
->aux
->stats
);
279 u64_stats_update_begin(&stats
->syncp
);
281 stats
->nsecs
+= sched_clock() - start
;
282 u64_stats_update_end(&stats
->syncp
);
289 arch_prepare_bpf_trampoline(void *image
, struct btf_func_model
*m
, u32 flags
,
290 struct bpf_prog
**fentry_progs
, int fentry_cnt
,
291 struct bpf_prog
**fexit_progs
, int fexit_cnt
,
297 static int __init
init_trampolines(void)
301 for (i
= 0; i
< TRAMPOLINE_TABLE_SIZE
; i
++)
302 INIT_HLIST_HEAD(&trampoline_table
[i
]);
305 late_initcall(init_trampolines
);