]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - kernel/kprobes.c
tools/libbpf: handle issues with bpf ELF objects containing .eh_frames
[mirror_ubuntu-bionic-kernel.git] / kernel / kprobes.c
CommitLineData
1da177e4
LT
1/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
b94cce92
HN
30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
1da177e4
LT
33 */
34#include <linux/kprobes.h>
1da177e4
LT
35#include <linux/hash.h>
36#include <linux/init.h>
4e57b681 37#include <linux/slab.h>
e3869792 38#include <linux/stddef.h>
9984de1a 39#include <linux/export.h>
9ec4b1f3 40#include <linux/moduleloader.h>
3a872d89 41#include <linux/kallsyms.h>
b4c6c34a 42#include <linux/freezer.h>
346fd59b
SD
43#include <linux/seq_file.h>
44#include <linux/debugfs.h>
b2be84df 45#include <linux/sysctl.h>
1eeb66a1 46#include <linux/kdebug.h>
4460fdad 47#include <linux/memory.h>
4554dbcb 48#include <linux/ftrace.h>
afd66255 49#include <linux/cpu.h>
bf5438fc 50#include <linux/jump_label.h>
bf8f6e5b 51
bfd45be0 52#include <asm/sections.h>
1da177e4
LT
53#include <asm/cacheflush.h>
54#include <asm/errno.h>
7c0f6ba6 55#include <linux/uaccess.h>
1da177e4
LT
56
57#define KPROBE_HASH_BITS 6
58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
3a872d89 60
ef53d9c5 61static int kprobes_initialized;
1da177e4 62static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
b94cce92 63static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
1da177e4 64
bf8f6e5b 65/* NOTE: change this value only with kprobe_mutex held */
e579abeb 66static bool kprobes_all_disarmed;
bf8f6e5b 67
43948f50
MH
68/* This protects kprobe_table and optimizing_list */
69static DEFINE_MUTEX(kprobe_mutex);
e6584523 70static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
ef53d9c5 71static struct {
ec484608 72 raw_spinlock_t lock ____cacheline_aligned_in_smp;
ef53d9c5
S
73} kretprobe_table_locks[KPROBE_TABLE_SIZE];
74
290e3070
NR
75kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
76 unsigned int __unused)
49e0b465
NR
77{
78 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
79}
80
ec484608 81static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
ef53d9c5
S
82{
83 return &(kretprobe_table_locks[hash].lock);
84}
1da177e4 85
376e2424
MH
86/* Blacklist -- list of struct kprobe_blacklist_entry */
87static LIST_HEAD(kprobe_blacklist);
3d8d996e 88
2d14e39d 89#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
9ec4b1f3
AM
90/*
91 * kprobe->ainsn.insn points to the copy of the instruction to be
92 * single-stepped. x86_64, POWER4 and above have no-exec support and
93 * stepping on the instruction on a vmalloced/kmalloced/data page
94 * is a recipe for disaster
95 */
9ec4b1f3 96struct kprobe_insn_page {
c5cb5a2d 97 struct list_head list;
9ec4b1f3 98 kprobe_opcode_t *insns; /* Page of instruction slots */
af96397d 99 struct kprobe_insn_cache *cache;
9ec4b1f3 100 int nused;
b4c6c34a 101 int ngarbage;
4610ee1d 102 char slot_used[];
9ec4b1f3
AM
103};
104
4610ee1d
MH
105#define KPROBE_INSN_PAGE_SIZE(slots) \
106 (offsetof(struct kprobe_insn_page, slot_used) + \
107 (sizeof(char) * (slots)))
108
4610ee1d
MH
109static int slots_per_page(struct kprobe_insn_cache *c)
110{
111 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
112}
113
ab40c5c6
MH
114enum kprobe_slot_state {
115 SLOT_CLEAN = 0,
116 SLOT_DIRTY = 1,
117 SLOT_USED = 2,
118};
119
63fef14f 120void __weak *alloc_insn_page(void)
af96397d
HC
121{
122 return module_alloc(PAGE_SIZE);
123}
124
c93f5cf5 125void __weak free_insn_page(void *page)
af96397d 126{
be1f221c 127 module_memfree(page);
af96397d
HC
128}
129
c802d64a
HC
130struct kprobe_insn_cache kprobe_insn_slots = {
131 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
af96397d
HC
132 .alloc = alloc_insn_page,
133 .free = free_insn_page,
4610ee1d
MH
134 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135 .insn_size = MAX_INSN_SIZE,
136 .nr_garbage = 0,
137};
55479f64 138static int collect_garbage_slots(struct kprobe_insn_cache *c);
b4c6c34a 139
9ec4b1f3 140/**
12941560 141 * __get_insn_slot() - Find a slot on an executable page for an instruction.
9ec4b1f3
AM
142 * We allocate an executable page if there's no room on existing ones.
143 */
55479f64 144kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
9ec4b1f3
AM
145{
146 struct kprobe_insn_page *kip;
c802d64a 147 kprobe_opcode_t *slot = NULL;
9ec4b1f3 148
5b485629 149 /* Since the slot array is not protected by rcu, we need a mutex */
c802d64a 150 mutex_lock(&c->mutex);
6f716acd 151 retry:
5b485629
MH
152 rcu_read_lock();
153 list_for_each_entry_rcu(kip, &c->pages, list) {
4610ee1d 154 if (kip->nused < slots_per_page(c)) {
9ec4b1f3 155 int i;
4610ee1d 156 for (i = 0; i < slots_per_page(c); i++) {
ab40c5c6
MH
157 if (kip->slot_used[i] == SLOT_CLEAN) {
158 kip->slot_used[i] = SLOT_USED;
9ec4b1f3 159 kip->nused++;
c802d64a 160 slot = kip->insns + (i * c->insn_size);
5b485629 161 rcu_read_unlock();
c802d64a 162 goto out;
9ec4b1f3
AM
163 }
164 }
4610ee1d
MH
165 /* kip->nused is broken. Fix it. */
166 kip->nused = slots_per_page(c);
167 WARN_ON(1);
9ec4b1f3
AM
168 }
169 }
5b485629 170 rcu_read_unlock();
9ec4b1f3 171
b4c6c34a 172 /* If there are any garbage slots, collect it and try again. */
4610ee1d 173 if (c->nr_garbage && collect_garbage_slots(c) == 0)
b4c6c34a 174 goto retry;
4610ee1d
MH
175
176 /* All out of space. Need to allocate a new page. */
177 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
6f716acd 178 if (!kip)
c802d64a 179 goto out;
9ec4b1f3
AM
180
181 /*
182 * Use module_alloc so this page is within +/- 2GB of where the
183 * kernel image and loaded module images reside. This is required
184 * so x86_64 can correctly handle the %rip-relative fixups.
185 */
af96397d 186 kip->insns = c->alloc();
9ec4b1f3
AM
187 if (!kip->insns) {
188 kfree(kip);
c802d64a 189 goto out;
9ec4b1f3 190 }
c5cb5a2d 191 INIT_LIST_HEAD(&kip->list);
4610ee1d 192 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
ab40c5c6 193 kip->slot_used[0] = SLOT_USED;
9ec4b1f3 194 kip->nused = 1;
b4c6c34a 195 kip->ngarbage = 0;
af96397d 196 kip->cache = c;
5b485629 197 list_add_rcu(&kip->list, &c->pages);
c802d64a
HC
198 slot = kip->insns;
199out:
200 mutex_unlock(&c->mutex);
201 return slot;
12941560
MH
202}
203
b4c6c34a 204/* Return 1 if all garbages are collected, otherwise 0. */
55479f64 205static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
b4c6c34a 206{
ab40c5c6 207 kip->slot_used[idx] = SLOT_CLEAN;
b4c6c34a
MH
208 kip->nused--;
209 if (kip->nused == 0) {
210 /*
211 * Page is no longer in use. Free it unless
212 * it's the last one. We keep the last one
213 * so as not to have to set it up again the
214 * next time somebody inserts a probe.
215 */
4610ee1d 216 if (!list_is_singular(&kip->list)) {
5b485629
MH
217 list_del_rcu(&kip->list);
218 synchronize_rcu();
af96397d 219 kip->cache->free(kip->insns);
b4c6c34a
MH
220 kfree(kip);
221 }
222 return 1;
223 }
224 return 0;
225}
226
55479f64 227static int collect_garbage_slots(struct kprobe_insn_cache *c)
b4c6c34a 228{
c5cb5a2d 229 struct kprobe_insn_page *kip, *next;
b4c6c34a 230
615d0ebb
MH
231 /* Ensure no-one is interrupted on the garbages */
232 synchronize_sched();
b4c6c34a 233
4610ee1d 234 list_for_each_entry_safe(kip, next, &c->pages, list) {
b4c6c34a 235 int i;
b4c6c34a
MH
236 if (kip->ngarbage == 0)
237 continue;
238 kip->ngarbage = 0; /* we will collect all garbages */
4610ee1d 239 for (i = 0; i < slots_per_page(c); i++) {
5b485629 240 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
b4c6c34a
MH
241 break;
242 }
243 }
4610ee1d 244 c->nr_garbage = 0;
b4c6c34a
MH
245 return 0;
246}
247
55479f64
MH
248void __free_insn_slot(struct kprobe_insn_cache *c,
249 kprobe_opcode_t *slot, int dirty)
9ec4b1f3
AM
250{
251 struct kprobe_insn_page *kip;
5b485629 252 long idx;
9ec4b1f3 253
c802d64a 254 mutex_lock(&c->mutex);
5b485629
MH
255 rcu_read_lock();
256 list_for_each_entry_rcu(kip, &c->pages, list) {
257 idx = ((long)slot - (long)kip->insns) /
258 (c->insn_size * sizeof(kprobe_opcode_t));
259 if (idx >= 0 && idx < slots_per_page(c))
c802d64a 260 goto out;
9ec4b1f3 261 }
5b485629 262 /* Could not find this slot. */
4610ee1d 263 WARN_ON(1);
5b485629 264 kip = NULL;
c802d64a 265out:
5b485629
MH
266 rcu_read_unlock();
267 /* Mark and sweep: this may sleep */
268 if (kip) {
269 /* Check double free */
270 WARN_ON(kip->slot_used[idx] != SLOT_USED);
271 if (dirty) {
272 kip->slot_used[idx] = SLOT_DIRTY;
273 kip->ngarbage++;
274 if (++c->nr_garbage > slots_per_page(c))
275 collect_garbage_slots(c);
276 } else {
277 collect_one_slot(kip, idx);
278 }
279 }
c802d64a 280 mutex_unlock(&c->mutex);
4610ee1d 281}
6f716acd 282
5b485629
MH
283/*
284 * Check given address is on the page of kprobe instruction slots.
285 * This will be used for checking whether the address on a stack
286 * is on a text area or not.
287 */
288bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
289{
290 struct kprobe_insn_page *kip;
291 bool ret = false;
292
293 rcu_read_lock();
294 list_for_each_entry_rcu(kip, &c->pages, list) {
295 if (addr >= (unsigned long)kip->insns &&
296 addr < (unsigned long)kip->insns + PAGE_SIZE) {
297 ret = true;
298 break;
299 }
300 }
301 rcu_read_unlock();
302
303 return ret;
304}
305
afd66255
MH
306#ifdef CONFIG_OPTPROBES
307/* For optimized_kprobe buffer */
c802d64a
HC
308struct kprobe_insn_cache kprobe_optinsn_slots = {
309 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
af96397d
HC
310 .alloc = alloc_insn_page,
311 .free = free_insn_page,
afd66255
MH
312 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
313 /* .insn_size is initialized later */
314 .nr_garbage = 0,
315};
afd66255 316#endif
2d14e39d 317#endif
9ec4b1f3 318
e6584523
AM
319/* We have preemption disabled.. so it is safe to use __ versions */
320static inline void set_kprobe_instance(struct kprobe *kp)
321{
b76834bc 322 __this_cpu_write(kprobe_instance, kp);
e6584523
AM
323}
324
325static inline void reset_kprobe_instance(void)
326{
b76834bc 327 __this_cpu_write(kprobe_instance, NULL);
e6584523
AM
328}
329
3516a460
AM
330/*
331 * This routine is called either:
49a2a1b8 332 * - under the kprobe_mutex - during kprobe_[un]register()
3516a460 333 * OR
d217d545 334 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
3516a460 335 */
820aede0 336struct kprobe *get_kprobe(void *addr)
1da177e4
LT
337{
338 struct hlist_head *head;
3516a460 339 struct kprobe *p;
1da177e4
LT
340
341 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
b67bfe0d 342 hlist_for_each_entry_rcu(p, head, hlist) {
1da177e4
LT
343 if (p->addr == addr)
344 return p;
345 }
afd66255 346
1da177e4
LT
347 return NULL;
348}
820aede0 349NOKPROBE_SYMBOL(get_kprobe);
1da177e4 350
820aede0 351static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
afd66255
MH
352
353/* Return true if the kprobe is an aggregator */
354static inline int kprobe_aggrprobe(struct kprobe *p)
355{
356 return p->pre_handler == aggr_pre_handler;
357}
358
6274de49
MH
359/* Return true(!0) if the kprobe is unused */
360static inline int kprobe_unused(struct kprobe *p)
361{
362 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
363 list_empty(&p->list);
364}
365
afd66255
MH
366/*
367 * Keep all fields in the kprobe consistent
368 */
6d8e40a8 369static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
afd66255 370{
6d8e40a8
MH
371 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
372 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
afd66255
MH
373}
374
375#ifdef CONFIG_OPTPROBES
b2be84df
MH
376/* NOTE: change this value only with kprobe_mutex held */
377static bool kprobes_allow_optimization;
378
afd66255
MH
379/*
380 * Call all pre_handler on the list, but ignores its return value.
381 * This must be called from arch-dep optimized caller.
382 */
820aede0 383void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
afd66255
MH
384{
385 struct kprobe *kp;
386
387 list_for_each_entry_rcu(kp, &p->list, list) {
388 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
389 set_kprobe_instance(kp);
4f3a8714 390 kp->pre_handler(kp, regs);
afd66255
MH
391 }
392 reset_kprobe_instance();
393 }
394}
820aede0 395NOKPROBE_SYMBOL(opt_pre_handler);
afd66255 396
6274de49 397/* Free optimized instructions and optimized_kprobe */
55479f64 398static void free_aggr_kprobe(struct kprobe *p)
6274de49
MH
399{
400 struct optimized_kprobe *op;
401
402 op = container_of(p, struct optimized_kprobe, kp);
403 arch_remove_optimized_kprobe(op);
404 arch_remove_kprobe(p);
405 kfree(op);
406}
407
afd66255
MH
408/* Return true(!0) if the kprobe is ready for optimization. */
409static inline int kprobe_optready(struct kprobe *p)
410{
411 struct optimized_kprobe *op;
412
413 if (kprobe_aggrprobe(p)) {
414 op = container_of(p, struct optimized_kprobe, kp);
415 return arch_prepared_optinsn(&op->optinsn);
416 }
417
418 return 0;
419}
420
6274de49
MH
421/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
422static inline int kprobe_disarmed(struct kprobe *p)
423{
424 struct optimized_kprobe *op;
425
426 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
427 if (!kprobe_aggrprobe(p))
428 return kprobe_disabled(p);
429
430 op = container_of(p, struct optimized_kprobe, kp);
431
432 return kprobe_disabled(p) && list_empty(&op->list);
433}
434
435/* Return true(!0) if the probe is queued on (un)optimizing lists */
55479f64 436static int kprobe_queued(struct kprobe *p)
6274de49
MH
437{
438 struct optimized_kprobe *op;
439
440 if (kprobe_aggrprobe(p)) {
441 op = container_of(p, struct optimized_kprobe, kp);
442 if (!list_empty(&op->list))
443 return 1;
444 }
445 return 0;
446}
447
afd66255
MH
448/*
449 * Return an optimized kprobe whose optimizing code replaces
450 * instructions including addr (exclude breakpoint).
451 */
55479f64 452static struct kprobe *get_optimized_kprobe(unsigned long addr)
afd66255
MH
453{
454 int i;
455 struct kprobe *p = NULL;
456 struct optimized_kprobe *op;
457
458 /* Don't check i == 0, since that is a breakpoint case. */
459 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
460 p = get_kprobe((void *)(addr - i));
461
462 if (p && kprobe_optready(p)) {
463 op = container_of(p, struct optimized_kprobe, kp);
464 if (arch_within_optimized_kprobe(op, addr))
465 return p;
466 }
467
468 return NULL;
469}
470
471/* Optimization staging list, protected by kprobe_mutex */
472static LIST_HEAD(optimizing_list);
6274de49 473static LIST_HEAD(unoptimizing_list);
7b959fc5 474static LIST_HEAD(freeing_list);
afd66255
MH
475
476static void kprobe_optimizer(struct work_struct *work);
477static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
478#define OPTIMIZE_DELAY 5
479
61f4e13f
MH
480/*
481 * Optimize (replace a breakpoint with a jump) kprobes listed on
482 * optimizing_list.
483 */
55479f64 484static void do_optimize_kprobes(void)
afd66255 485{
afd66255
MH
486 /*
487 * The optimization/unoptimization refers online_cpus via
488 * stop_machine() and cpu-hotplug modifies online_cpus.
489 * And same time, text_mutex will be held in cpu-hotplug and here.
490 * This combination can cause a deadlock (cpu-hotplug try to lock
491 * text_mutex but stop_machine can not be done because online_cpus
492 * has been changed)
2d1e38f5 493 * To avoid this deadlock, caller must have locked cpu hotplug
afd66255
MH
494 * for preventing cpu-hotplug outside of text_mutex locking.
495 */
2d1e38f5
TG
496 lockdep_assert_cpus_held();
497
498 /* Optimization never be done when disarmed */
499 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
500 list_empty(&optimizing_list))
501 return;
502
afd66255 503 mutex_lock(&text_mutex);
cd7ebe22 504 arch_optimize_kprobes(&optimizing_list);
afd66255 505 mutex_unlock(&text_mutex);
61f4e13f
MH
506}
507
6274de49
MH
508/*
509 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
510 * if need) kprobes listed on unoptimizing_list.
511 */
55479f64 512static void do_unoptimize_kprobes(void)
6274de49
MH
513{
514 struct optimized_kprobe *op, *tmp;
515
2d1e38f5
TG
516 /* See comment in do_optimize_kprobes() */
517 lockdep_assert_cpus_held();
518
6274de49
MH
519 /* Unoptimization must be done anytime */
520 if (list_empty(&unoptimizing_list))
521 return;
522
6274de49 523 mutex_lock(&text_mutex);
7b959fc5 524 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
f984ba4e 525 /* Loop free_list for disarming */
7b959fc5 526 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
6274de49
MH
527 /* Disarm probes if marked disabled */
528 if (kprobe_disabled(&op->kp))
529 arch_disarm_kprobe(&op->kp);
530 if (kprobe_unused(&op->kp)) {
531 /*
532 * Remove unused probes from hash list. After waiting
533 * for synchronization, these probes are reclaimed.
534 * (reclaiming is done by do_free_cleaned_kprobes.)
535 */
536 hlist_del_rcu(&op->kp.hlist);
6274de49
MH
537 } else
538 list_del_init(&op->list);
539 }
540 mutex_unlock(&text_mutex);
6274de49
MH
541}
542
543/* Reclaim all kprobes on the free_list */
55479f64 544static void do_free_cleaned_kprobes(void)
6274de49
MH
545{
546 struct optimized_kprobe *op, *tmp;
547
7b959fc5 548 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
6274de49
MH
549 BUG_ON(!kprobe_unused(&op->kp));
550 list_del_init(&op->list);
551 free_aggr_kprobe(&op->kp);
552 }
553}
554
555/* Start optimizer after OPTIMIZE_DELAY passed */
55479f64 556static void kick_kprobe_optimizer(void)
6274de49 557{
ad72b3be 558 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
6274de49
MH
559}
560
61f4e13f 561/* Kprobe jump optimizer */
55479f64 562static void kprobe_optimizer(struct work_struct *work)
61f4e13f 563{
72ef3794 564 mutex_lock(&kprobe_mutex);
2d1e38f5 565 cpus_read_lock();
61f4e13f
MH
566 /* Lock modules while optimizing kprobes */
567 mutex_lock(&module_mutex);
61f4e13f
MH
568
569 /*
6274de49
MH
570 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
571 * kprobes before waiting for quiesence period.
572 */
7b959fc5 573 do_unoptimize_kprobes();
6274de49
MH
574
575 /*
a30b85df
MH
576 * Step 2: Wait for quiesence period to ensure all potentially
577 * preempted tasks to have normally scheduled. Because optprobe
578 * may modify multiple instructions, there is a chance that Nth
579 * instruction is preempted. In that case, such tasks can return
580 * to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
581 * Note that on non-preemptive kernel, this is transparently converted
582 * to synchronoze_sched() to wait for all interrupts to have completed.
61f4e13f 583 */
a30b85df 584 synchronize_rcu_tasks();
61f4e13f 585
6274de49 586 /* Step 3: Optimize kprobes after quiesence period */
61f4e13f 587 do_optimize_kprobes();
6274de49
MH
588
589 /* Step 4: Free cleaned kprobes after quiesence period */
7b959fc5 590 do_free_cleaned_kprobes();
6274de49 591
afd66255 592 mutex_unlock(&module_mutex);
2d1e38f5 593 cpus_read_unlock();
72ef3794 594 mutex_unlock(&kprobe_mutex);
6274de49 595
cd7ebe22 596 /* Step 5: Kick optimizer again if needed */
f984ba4e 597 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
cd7ebe22 598 kick_kprobe_optimizer();
6274de49
MH
599}
600
601/* Wait for completing optimization and unoptimization */
30e7d894 602void wait_for_kprobe_optimizer(void)
6274de49 603{
ad72b3be
TH
604 mutex_lock(&kprobe_mutex);
605
606 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
607 mutex_unlock(&kprobe_mutex);
608
609 /* this will also make optimizing_work execute immmediately */
610 flush_delayed_work(&optimizing_work);
611 /* @optimizing_work might not have been queued yet, relax */
612 cpu_relax();
613
614 mutex_lock(&kprobe_mutex);
615 }
616
617 mutex_unlock(&kprobe_mutex);
afd66255
MH
618}
619
620/* Optimize kprobe if p is ready to be optimized */
55479f64 621static void optimize_kprobe(struct kprobe *p)
afd66255
MH
622{
623 struct optimized_kprobe *op;
624
625 /* Check if the kprobe is disabled or not ready for optimization. */
b2be84df 626 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
afd66255
MH
627 (kprobe_disabled(p) || kprobes_all_disarmed))
628 return;
629
630 /* Both of break_handler and post_handler are not supported. */
631 if (p->break_handler || p->post_handler)
632 return;
633
634 op = container_of(p, struct optimized_kprobe, kp);
635
636 /* Check there is no other kprobes at the optimized instructions */
637 if (arch_check_optimized_kprobe(op) < 0)
638 return;
639
640 /* Check if it is already optimized. */
641 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
642 return;
afd66255 643 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
6274de49
MH
644
645 if (!list_empty(&op->list))
646 /* This is under unoptimizing. Just dequeue the probe */
647 list_del_init(&op->list);
648 else {
649 list_add(&op->list, &optimizing_list);
650 kick_kprobe_optimizer();
651 }
652}
653
654/* Short cut to direct unoptimizing */
55479f64 655static void force_unoptimize_kprobe(struct optimized_kprobe *op)
6274de49 656{
2d1e38f5 657 lockdep_assert_cpus_held();
6274de49 658 arch_unoptimize_kprobe(op);
6274de49
MH
659 if (kprobe_disabled(&op->kp))
660 arch_disarm_kprobe(&op->kp);
afd66255
MH
661}
662
663/* Unoptimize a kprobe if p is optimized */
55479f64 664static void unoptimize_kprobe(struct kprobe *p, bool force)
afd66255
MH
665{
666 struct optimized_kprobe *op;
667
6274de49
MH
668 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
669 return; /* This is not an optprobe nor optimized */
670
671 op = container_of(p, struct optimized_kprobe, kp);
672 if (!kprobe_optimized(p)) {
673 /* Unoptimized or unoptimizing case */
674 if (force && !list_empty(&op->list)) {
675 /*
676 * Only if this is unoptimizing kprobe and forced,
677 * forcibly unoptimize it. (No need to unoptimize
678 * unoptimized kprobe again :)
679 */
afd66255 680 list_del_init(&op->list);
6274de49
MH
681 force_unoptimize_kprobe(op);
682 }
683 return;
684 }
685
686 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
687 if (!list_empty(&op->list)) {
688 /* Dequeue from the optimization queue */
689 list_del_init(&op->list);
690 return;
691 }
692 /* Optimized kprobe case */
693 if (force)
694 /* Forcibly update the code: this is a special case */
695 force_unoptimize_kprobe(op);
696 else {
697 list_add(&op->list, &unoptimizing_list);
698 kick_kprobe_optimizer();
afd66255
MH
699 }
700}
701
0490cd1f
MH
702/* Cancel unoptimizing for reusing */
703static void reuse_unused_kprobe(struct kprobe *ap)
704{
705 struct optimized_kprobe *op;
706
707 BUG_ON(!kprobe_unused(ap));
708 /*
709 * Unused kprobe MUST be on the way of delayed unoptimizing (means
710 * there is still a relative jump) and disabled.
711 */
712 op = container_of(ap, struct optimized_kprobe, kp);
713 if (unlikely(list_empty(&op->list)))
714 printk(KERN_WARNING "Warning: found a stray unused "
715 "aggrprobe@%p\n", ap->addr);
716 /* Enable the probe again */
717 ap->flags &= ~KPROBE_FLAG_DISABLED;
718 /* Optimize it again (remove from op->list) */
719 BUG_ON(!kprobe_optready(ap));
720 optimize_kprobe(ap);
721}
722
afd66255 723/* Remove optimized instructions */
55479f64 724static void kill_optimized_kprobe(struct kprobe *p)
afd66255
MH
725{
726 struct optimized_kprobe *op;
727
728 op = container_of(p, struct optimized_kprobe, kp);
6274de49
MH
729 if (!list_empty(&op->list))
730 /* Dequeue from the (un)optimization queue */
afd66255 731 list_del_init(&op->list);
6274de49 732 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
7b959fc5
MH
733
734 if (kprobe_unused(p)) {
735 /* Enqueue if it is unused */
736 list_add(&op->list, &freeing_list);
737 /*
738 * Remove unused probes from the hash list. After waiting
739 * for synchronization, this probe is reclaimed.
740 * (reclaiming is done by do_free_cleaned_kprobes().)
741 */
742 hlist_del_rcu(&op->kp.hlist);
743 }
744
6274de49 745 /* Don't touch the code, because it is already freed. */
afd66255
MH
746 arch_remove_optimized_kprobe(op);
747}
748
a460246c
MH
749static inline
750void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
751{
752 if (!kprobe_ftrace(p))
753 arch_prepare_optimized_kprobe(op, p);
754}
755
afd66255 756/* Try to prepare optimized instructions */
55479f64 757static void prepare_optimized_kprobe(struct kprobe *p)
afd66255
MH
758{
759 struct optimized_kprobe *op;
760
761 op = container_of(p, struct optimized_kprobe, kp);
a460246c 762 __prepare_optimized_kprobe(op, p);
afd66255
MH
763}
764
afd66255 765/* Allocate new optimized_kprobe and try to prepare optimized instructions */
55479f64 766static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
afd66255
MH
767{
768 struct optimized_kprobe *op;
769
770 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
771 if (!op)
772 return NULL;
773
774 INIT_LIST_HEAD(&op->list);
775 op->kp.addr = p->addr;
a460246c 776 __prepare_optimized_kprobe(op, p);
afd66255
MH
777
778 return &op->kp;
779}
780
55479f64 781static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
afd66255
MH
782
783/*
784 * Prepare an optimized_kprobe and optimize it
785 * NOTE: p must be a normal registered kprobe
786 */
55479f64 787static void try_to_optimize_kprobe(struct kprobe *p)
afd66255
MH
788{
789 struct kprobe *ap;
790 struct optimized_kprobe *op;
791
ae6aa16f
MH
792 /* Impossible to optimize ftrace-based kprobe */
793 if (kprobe_ftrace(p))
794 return;
795
25764288 796 /* For preparing optimization, jump_label_text_reserved() is called */
2d1e38f5 797 cpus_read_lock();
25764288
MH
798 jump_label_lock();
799 mutex_lock(&text_mutex);
800
afd66255
MH
801 ap = alloc_aggr_kprobe(p);
802 if (!ap)
25764288 803 goto out;
afd66255
MH
804
805 op = container_of(ap, struct optimized_kprobe, kp);
806 if (!arch_prepared_optinsn(&op->optinsn)) {
807 /* If failed to setup optimizing, fallback to kprobe */
6274de49
MH
808 arch_remove_optimized_kprobe(op);
809 kfree(op);
25764288 810 goto out;
afd66255
MH
811 }
812
813 init_aggr_kprobe(ap, p);
25764288
MH
814 optimize_kprobe(ap); /* This just kicks optimizer thread */
815
816out:
817 mutex_unlock(&text_mutex);
818 jump_label_unlock();
2d1e38f5 819 cpus_read_unlock();
afd66255
MH
820}
821
b2be84df 822#ifdef CONFIG_SYSCTL
55479f64 823static void optimize_all_kprobes(void)
b2be84df
MH
824{
825 struct hlist_head *head;
b2be84df
MH
826 struct kprobe *p;
827 unsigned int i;
828
5c51543b 829 mutex_lock(&kprobe_mutex);
b2be84df
MH
830 /* If optimization is already allowed, just return */
831 if (kprobes_allow_optimization)
5c51543b 832 goto out;
b2be84df 833
2d1e38f5 834 cpus_read_lock();
b2be84df 835 kprobes_allow_optimization = true;
b2be84df
MH
836 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
837 head = &kprobe_table[i];
b67bfe0d 838 hlist_for_each_entry_rcu(p, head, hlist)
b2be84df
MH
839 if (!kprobe_disabled(p))
840 optimize_kprobe(p);
841 }
2d1e38f5 842 cpus_read_unlock();
b2be84df 843 printk(KERN_INFO "Kprobes globally optimized\n");
5c51543b
MH
844out:
845 mutex_unlock(&kprobe_mutex);
b2be84df
MH
846}
847
55479f64 848static void unoptimize_all_kprobes(void)
b2be84df
MH
849{
850 struct hlist_head *head;
b2be84df
MH
851 struct kprobe *p;
852 unsigned int i;
853
5c51543b 854 mutex_lock(&kprobe_mutex);
b2be84df 855 /* If optimization is already prohibited, just return */
5c51543b
MH
856 if (!kprobes_allow_optimization) {
857 mutex_unlock(&kprobe_mutex);
b2be84df 858 return;
5c51543b 859 }
b2be84df 860
2d1e38f5 861 cpus_read_lock();
b2be84df 862 kprobes_allow_optimization = false;
b2be84df
MH
863 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
864 head = &kprobe_table[i];
b67bfe0d 865 hlist_for_each_entry_rcu(p, head, hlist) {
b2be84df 866 if (!kprobe_disabled(p))
6274de49 867 unoptimize_kprobe(p, false);
b2be84df
MH
868 }
869 }
2d1e38f5 870 cpus_read_unlock();
5c51543b
MH
871 mutex_unlock(&kprobe_mutex);
872
6274de49
MH
873 /* Wait for unoptimizing completion */
874 wait_for_kprobe_optimizer();
875 printk(KERN_INFO "Kprobes globally unoptimized\n");
b2be84df
MH
876}
877
5c51543b 878static DEFINE_MUTEX(kprobe_sysctl_mutex);
b2be84df
MH
879int sysctl_kprobes_optimization;
880int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
881 void __user *buffer, size_t *length,
882 loff_t *ppos)
883{
884 int ret;
885
5c51543b 886 mutex_lock(&kprobe_sysctl_mutex);
b2be84df
MH
887 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
888 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
889
890 if (sysctl_kprobes_optimization)
891 optimize_all_kprobes();
892 else
893 unoptimize_all_kprobes();
5c51543b 894 mutex_unlock(&kprobe_sysctl_mutex);
b2be84df
MH
895
896 return ret;
897}
898#endif /* CONFIG_SYSCTL */
899
6274de49 900/* Put a breakpoint for a probe. Must be called with text_mutex locked */
55479f64 901static void __arm_kprobe(struct kprobe *p)
afd66255 902{
6d8e40a8 903 struct kprobe *_p;
afd66255
MH
904
905 /* Check collision with other optimized kprobes */
6d8e40a8
MH
906 _p = get_optimized_kprobe((unsigned long)p->addr);
907 if (unlikely(_p))
6274de49
MH
908 /* Fallback to unoptimized kprobe */
909 unoptimize_kprobe(_p, true);
afd66255
MH
910
911 arch_arm_kprobe(p);
912 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
913}
914
6274de49 915/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
55479f64 916static void __disarm_kprobe(struct kprobe *p, bool reopt)
afd66255 917{
6d8e40a8 918 struct kprobe *_p;
afd66255 919
69d54b91
WN
920 /* Try to unoptimize */
921 unoptimize_kprobe(p, kprobes_all_disarmed);
afd66255 922
6274de49
MH
923 if (!kprobe_queued(p)) {
924 arch_disarm_kprobe(p);
925 /* If another kprobe was blocked, optimize it. */
926 _p = get_optimized_kprobe((unsigned long)p->addr);
927 if (unlikely(_p) && reopt)
928 optimize_kprobe(_p);
929 }
930 /* TODO: reoptimize others after unoptimized this probe */
afd66255
MH
931}
932
933#else /* !CONFIG_OPTPROBES */
934
935#define optimize_kprobe(p) do {} while (0)
6274de49 936#define unoptimize_kprobe(p, f) do {} while (0)
afd66255
MH
937#define kill_optimized_kprobe(p) do {} while (0)
938#define prepare_optimized_kprobe(p) do {} while (0)
939#define try_to_optimize_kprobe(p) do {} while (0)
940#define __arm_kprobe(p) arch_arm_kprobe(p)
6274de49
MH
941#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
942#define kprobe_disarmed(p) kprobe_disabled(p)
943#define wait_for_kprobe_optimizer() do {} while (0)
afd66255 944
0490cd1f
MH
945/* There should be no unused kprobes can be reused without optimization */
946static void reuse_unused_kprobe(struct kprobe *ap)
947{
948 printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
949 BUG_ON(kprobe_unused(ap));
950}
951
55479f64 952static void free_aggr_kprobe(struct kprobe *p)
afd66255 953{
6274de49 954 arch_remove_kprobe(p);
afd66255
MH
955 kfree(p);
956}
957
55479f64 958static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
afd66255
MH
959{
960 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
961}
962#endif /* CONFIG_OPTPROBES */
963
e7dbfe34 964#ifdef CONFIG_KPROBES_ON_FTRACE
ae6aa16f 965static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
e5253896 966 .func = kprobe_ftrace_handler,
1d70be34 967 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
ae6aa16f
MH
968};
969static int kprobe_ftrace_enabled;
970
971/* Must ensure p->addr is really on ftrace */
55479f64 972static int prepare_kprobe(struct kprobe *p)
ae6aa16f
MH
973{
974 if (!kprobe_ftrace(p))
975 return arch_prepare_kprobe(p);
976
977 return arch_prepare_kprobe_ftrace(p);
978}
979
980/* Caller must lock kprobe_mutex */
55479f64 981static void arm_kprobe_ftrace(struct kprobe *p)
ae6aa16f
MH
982{
983 int ret;
984
985 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
986 (unsigned long)p->addr, 0, 0);
987 WARN(ret < 0, "Failed to arm kprobe-ftrace at %p (%d)\n", p->addr, ret);
988 kprobe_ftrace_enabled++;
989 if (kprobe_ftrace_enabled == 1) {
990 ret = register_ftrace_function(&kprobe_ftrace_ops);
991 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret);
992 }
993}
994
995/* Caller must lock kprobe_mutex */
55479f64 996static void disarm_kprobe_ftrace(struct kprobe *p)
ae6aa16f
MH
997{
998 int ret;
999
1000 kprobe_ftrace_enabled--;
1001 if (kprobe_ftrace_enabled == 0) {
1002 ret = unregister_ftrace_function(&kprobe_ftrace_ops);
1003 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret);
1004 }
1005 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
1006 (unsigned long)p->addr, 1, 0);
1007 WARN(ret < 0, "Failed to disarm kprobe-ftrace at %p (%d)\n", p->addr, ret);
1008}
e7dbfe34 1009#else /* !CONFIG_KPROBES_ON_FTRACE */
ae6aa16f
MH
1010#define prepare_kprobe(p) arch_prepare_kprobe(p)
1011#define arm_kprobe_ftrace(p) do {} while (0)
1012#define disarm_kprobe_ftrace(p) do {} while (0)
1013#endif
1014
201517a7 1015/* Arm a kprobe with text_mutex */
55479f64 1016static void arm_kprobe(struct kprobe *kp)
201517a7 1017{
ae6aa16f
MH
1018 if (unlikely(kprobe_ftrace(kp))) {
1019 arm_kprobe_ftrace(kp);
1020 return;
1021 }
2d1e38f5 1022 cpus_read_lock();
201517a7 1023 mutex_lock(&text_mutex);
afd66255 1024 __arm_kprobe(kp);
201517a7 1025 mutex_unlock(&text_mutex);
2d1e38f5 1026 cpus_read_unlock();
201517a7
MH
1027}
1028
1029/* Disarm a kprobe with text_mutex */
55479f64 1030static void disarm_kprobe(struct kprobe *kp, bool reopt)
201517a7 1031{
ae6aa16f
MH
1032 if (unlikely(kprobe_ftrace(kp))) {
1033 disarm_kprobe_ftrace(kp);
1034 return;
1035 }
2d1e38f5
TG
1036
1037 cpus_read_lock();
201517a7 1038 mutex_lock(&text_mutex);
ae6aa16f 1039 __disarm_kprobe(kp, reopt);
201517a7 1040 mutex_unlock(&text_mutex);
2d1e38f5 1041 cpus_read_unlock();
201517a7
MH
1042}
1043
64f562c6
AM
1044/*
1045 * Aggregate handlers for multiple kprobes support - these handlers
1046 * take care of invoking the individual kprobe handlers on p->list
1047 */
820aede0 1048static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
64f562c6
AM
1049{
1050 struct kprobe *kp;
1051
3516a460 1052 list_for_each_entry_rcu(kp, &p->list, list) {
de5bd88d 1053 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
e6584523 1054 set_kprobe_instance(kp);
8b0914ea
PP
1055 if (kp->pre_handler(kp, regs))
1056 return 1;
64f562c6 1057 }
e6584523 1058 reset_kprobe_instance();
64f562c6
AM
1059 }
1060 return 0;
1061}
820aede0 1062NOKPROBE_SYMBOL(aggr_pre_handler);
64f562c6 1063
820aede0
MH
1064static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1065 unsigned long flags)
64f562c6
AM
1066{
1067 struct kprobe *kp;
1068
3516a460 1069 list_for_each_entry_rcu(kp, &p->list, list) {
de5bd88d 1070 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
e6584523 1071 set_kprobe_instance(kp);
64f562c6 1072 kp->post_handler(kp, regs, flags);
e6584523 1073 reset_kprobe_instance();
64f562c6
AM
1074 }
1075 }
64f562c6 1076}
820aede0 1077NOKPROBE_SYMBOL(aggr_post_handler);
64f562c6 1078
820aede0
MH
1079static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
1080 int trapnr)
64f562c6 1081{
b76834bc 1082 struct kprobe *cur = __this_cpu_read(kprobe_instance);
e6584523 1083
64f562c6
AM
1084 /*
1085 * if we faulted "during" the execution of a user specified
1086 * probe handler, invoke just that probe's fault handler
1087 */
e6584523
AM
1088 if (cur && cur->fault_handler) {
1089 if (cur->fault_handler(cur, regs, trapnr))
64f562c6
AM
1090 return 1;
1091 }
1092 return 0;
1093}
820aede0 1094NOKPROBE_SYMBOL(aggr_fault_handler);
64f562c6 1095
820aede0 1096static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
8b0914ea 1097{
b76834bc 1098 struct kprobe *cur = __this_cpu_read(kprobe_instance);
e6584523
AM
1099 int ret = 0;
1100
1101 if (cur && cur->break_handler) {
1102 if (cur->break_handler(cur, regs))
1103 ret = 1;
8b0914ea 1104 }
e6584523
AM
1105 reset_kprobe_instance();
1106 return ret;
8b0914ea 1107}
820aede0 1108NOKPROBE_SYMBOL(aggr_break_handler);
8b0914ea 1109
bf8d5c52 1110/* Walks the list and increments nmissed count for multiprobe case */
820aede0 1111void kprobes_inc_nmissed_count(struct kprobe *p)
bf8d5c52
KA
1112{
1113 struct kprobe *kp;
afd66255 1114 if (!kprobe_aggrprobe(p)) {
bf8d5c52
KA
1115 p->nmissed++;
1116 } else {
1117 list_for_each_entry_rcu(kp, &p->list, list)
1118 kp->nmissed++;
1119 }
1120 return;
1121}
820aede0 1122NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
bf8d5c52 1123
820aede0
MH
1124void recycle_rp_inst(struct kretprobe_instance *ri,
1125 struct hlist_head *head)
b94cce92 1126{
ef53d9c5
S
1127 struct kretprobe *rp = ri->rp;
1128
b94cce92
HN
1129 /* remove rp inst off the rprobe_inst_table */
1130 hlist_del(&ri->hlist);
ef53d9c5
S
1131 INIT_HLIST_NODE(&ri->hlist);
1132 if (likely(rp)) {
ec484608 1133 raw_spin_lock(&rp->lock);
ef53d9c5 1134 hlist_add_head(&ri->hlist, &rp->free_instances);
ec484608 1135 raw_spin_unlock(&rp->lock);
b94cce92
HN
1136 } else
1137 /* Unregistering */
99219a3f 1138 hlist_add_head(&ri->hlist, head);
b94cce92 1139}
820aede0 1140NOKPROBE_SYMBOL(recycle_rp_inst);
b94cce92 1141
820aede0 1142void kretprobe_hash_lock(struct task_struct *tsk,
ef53d9c5 1143 struct hlist_head **head, unsigned long *flags)
635c17c2 1144__acquires(hlist_lock)
ef53d9c5
S
1145{
1146 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
ec484608 1147 raw_spinlock_t *hlist_lock;
ef53d9c5
S
1148
1149 *head = &kretprobe_inst_table[hash];
1150 hlist_lock = kretprobe_table_lock_ptr(hash);
ec484608 1151 raw_spin_lock_irqsave(hlist_lock, *flags);
ef53d9c5 1152}
820aede0 1153NOKPROBE_SYMBOL(kretprobe_hash_lock);
ef53d9c5 1154
820aede0
MH
1155static void kretprobe_table_lock(unsigned long hash,
1156 unsigned long *flags)
635c17c2 1157__acquires(hlist_lock)
b94cce92 1158{
ec484608
TG
1159 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1160 raw_spin_lock_irqsave(hlist_lock, *flags);
ef53d9c5 1161}
820aede0 1162NOKPROBE_SYMBOL(kretprobe_table_lock);
ef53d9c5 1163
820aede0
MH
1164void kretprobe_hash_unlock(struct task_struct *tsk,
1165 unsigned long *flags)
635c17c2 1166__releases(hlist_lock)
ef53d9c5
S
1167{
1168 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
ec484608 1169 raw_spinlock_t *hlist_lock;
ef53d9c5
S
1170
1171 hlist_lock = kretprobe_table_lock_ptr(hash);
ec484608 1172 raw_spin_unlock_irqrestore(hlist_lock, *flags);
ef53d9c5 1173}
820aede0 1174NOKPROBE_SYMBOL(kretprobe_hash_unlock);
ef53d9c5 1175
820aede0
MH
1176static void kretprobe_table_unlock(unsigned long hash,
1177 unsigned long *flags)
635c17c2 1178__releases(hlist_lock)
ef53d9c5 1179{
ec484608
TG
1180 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1181 raw_spin_unlock_irqrestore(hlist_lock, *flags);
b94cce92 1182}
820aede0 1183NOKPROBE_SYMBOL(kretprobe_table_unlock);
b94cce92 1184
b94cce92 1185/*
c6fd91f0 1186 * This function is called from finish_task_switch when task tk becomes dead,
1187 * so that we can recycle any function-return probe instances associated
1188 * with this task. These left over instances represent probed functions
1189 * that have been called but will never return.
b94cce92 1190 */
820aede0 1191void kprobe_flush_task(struct task_struct *tk)
b94cce92 1192{
62c27be0 1193 struct kretprobe_instance *ri;
99219a3f 1194 struct hlist_head *head, empty_rp;
b67bfe0d 1195 struct hlist_node *tmp;
ef53d9c5 1196 unsigned long hash, flags = 0;
802eae7c 1197
ef53d9c5
S
1198 if (unlikely(!kprobes_initialized))
1199 /* Early boot. kretprobe_table_locks not yet initialized. */
1200 return;
1201
d496aab5 1202 INIT_HLIST_HEAD(&empty_rp);
ef53d9c5
S
1203 hash = hash_ptr(tk, KPROBE_HASH_BITS);
1204 head = &kretprobe_inst_table[hash];
1205 kretprobe_table_lock(hash, &flags);
b67bfe0d 1206 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
62c27be0 1207 if (ri->task == tk)
99219a3f 1208 recycle_rp_inst(ri, &empty_rp);
62c27be0 1209 }
ef53d9c5 1210 kretprobe_table_unlock(hash, &flags);
b67bfe0d 1211 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
99219a3f 1212 hlist_del(&ri->hlist);
1213 kfree(ri);
1214 }
b94cce92 1215}
820aede0 1216NOKPROBE_SYMBOL(kprobe_flush_task);
b94cce92 1217
b94cce92
HN
1218static inline void free_rp_inst(struct kretprobe *rp)
1219{
1220 struct kretprobe_instance *ri;
b67bfe0d 1221 struct hlist_node *next;
4c4308cb 1222
b67bfe0d 1223 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
ef53d9c5 1224 hlist_del(&ri->hlist);
b94cce92
HN
1225 kfree(ri);
1226 }
1227}
1228
820aede0 1229static void cleanup_rp_inst(struct kretprobe *rp)
4a296e07 1230{
ef53d9c5 1231 unsigned long flags, hash;
4a296e07 1232 struct kretprobe_instance *ri;
b67bfe0d 1233 struct hlist_node *next;
ef53d9c5
S
1234 struct hlist_head *head;
1235
4a296e07 1236 /* No race here */
ef53d9c5
S
1237 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1238 kretprobe_table_lock(hash, &flags);
1239 head = &kretprobe_inst_table[hash];
b67bfe0d 1240 hlist_for_each_entry_safe(ri, next, head, hlist) {
ef53d9c5
S
1241 if (ri->rp == rp)
1242 ri->rp = NULL;
1243 }
1244 kretprobe_table_unlock(hash, &flags);
4a296e07 1245 }
4a296e07
MH
1246 free_rp_inst(rp);
1247}
820aede0 1248NOKPROBE_SYMBOL(cleanup_rp_inst);
4a296e07 1249
8b0914ea 1250/*
b918e5e6 1251* Add the new probe to ap->list. Fail if this is the
8b0914ea
PP
1252* second jprobe at the address - two jprobes can't coexist
1253*/
55479f64 1254static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
8b0914ea 1255{
de5bd88d 1256 BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
afd66255
MH
1257
1258 if (p->break_handler || p->post_handler)
6274de49 1259 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
afd66255 1260
8b0914ea 1261 if (p->break_handler) {
b918e5e6 1262 if (ap->break_handler)
36721656 1263 return -EEXIST;
b918e5e6
MH
1264 list_add_tail_rcu(&p->list, &ap->list);
1265 ap->break_handler = aggr_break_handler;
8b0914ea 1266 } else
b918e5e6
MH
1267 list_add_rcu(&p->list, &ap->list);
1268 if (p->post_handler && !ap->post_handler)
1269 ap->post_handler = aggr_post_handler;
de5bd88d 1270
8b0914ea
PP
1271 return 0;
1272}
1273
64f562c6
AM
1274/*
1275 * Fill in the required fields of the "manager kprobe". Replace the
1276 * earlier kprobe in the hlist with the manager kprobe
1277 */
55479f64 1278static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
64f562c6 1279{
afd66255 1280 /* Copy p's insn slot to ap */
8b0914ea 1281 copy_kprobe(p, ap);
a9ad965e 1282 flush_insn_slot(ap);
64f562c6 1283 ap->addr = p->addr;
afd66255 1284 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
64f562c6 1285 ap->pre_handler = aggr_pre_handler;
64f562c6 1286 ap->fault_handler = aggr_fault_handler;
e8386a0c
MH
1287 /* We don't care the kprobe which has gone. */
1288 if (p->post_handler && !kprobe_gone(p))
36721656 1289 ap->post_handler = aggr_post_handler;
e8386a0c 1290 if (p->break_handler && !kprobe_gone(p))
36721656 1291 ap->break_handler = aggr_break_handler;
64f562c6
AM
1292
1293 INIT_LIST_HEAD(&ap->list);
afd66255 1294 INIT_HLIST_NODE(&ap->hlist);
64f562c6 1295
afd66255 1296 list_add_rcu(&p->list, &ap->list);
adad0f33 1297 hlist_replace_rcu(&p->hlist, &ap->hlist);
64f562c6
AM
1298}
1299
1300/*
1301 * This is the second or subsequent kprobe at the address - handle
1302 * the intricacies
64f562c6 1303 */
55479f64 1304static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
64f562c6
AM
1305{
1306 int ret = 0;
6d8e40a8 1307 struct kprobe *ap = orig_p;
64f562c6 1308
2d1e38f5
TG
1309 cpus_read_lock();
1310
25764288
MH
1311 /* For preparing optimization, jump_label_text_reserved() is called */
1312 jump_label_lock();
25764288
MH
1313 mutex_lock(&text_mutex);
1314
6d8e40a8
MH
1315 if (!kprobe_aggrprobe(orig_p)) {
1316 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1317 ap = alloc_aggr_kprobe(orig_p);
25764288
MH
1318 if (!ap) {
1319 ret = -ENOMEM;
1320 goto out;
1321 }
6d8e40a8 1322 init_aggr_kprobe(ap, orig_p);
6274de49 1323 } else if (kprobe_unused(ap))
0490cd1f
MH
1324 /* This probe is going to die. Rescue it */
1325 reuse_unused_kprobe(ap);
b918e5e6
MH
1326
1327 if (kprobe_gone(ap)) {
e8386a0c
MH
1328 /*
1329 * Attempting to insert new probe at the same location that
1330 * had a probe in the module vaddr area which already
1331 * freed. So, the instruction slot has already been
1332 * released. We need a new slot for the new probe.
1333 */
b918e5e6 1334 ret = arch_prepare_kprobe(ap);
e8386a0c 1335 if (ret)
b918e5e6
MH
1336 /*
1337 * Even if fail to allocate new slot, don't need to
1338 * free aggr_probe. It will be used next time, or
1339 * freed by unregister_kprobe.
1340 */
25764288 1341 goto out;
de5bd88d 1342
afd66255
MH
1343 /* Prepare optimized instructions if possible. */
1344 prepare_optimized_kprobe(ap);
1345
e8386a0c 1346 /*
de5bd88d
MH
1347 * Clear gone flag to prevent allocating new slot again, and
1348 * set disabled flag because it is not armed yet.
e8386a0c 1349 */
de5bd88d
MH
1350 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1351 | KPROBE_FLAG_DISABLED;
e8386a0c 1352 }
b918e5e6 1353
afd66255 1354 /* Copy ap's insn slot to p */
b918e5e6 1355 copy_kprobe(ap, p);
25764288
MH
1356 ret = add_new_kprobe(ap, p);
1357
1358out:
1359 mutex_unlock(&text_mutex);
25764288 1360 jump_label_unlock();
2d1e38f5 1361 cpus_read_unlock();
25764288
MH
1362
1363 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1364 ap->flags &= ~KPROBE_FLAG_DISABLED;
1365 if (!kprobes_all_disarmed)
1366 /* Arm the breakpoint again. */
1367 arm_kprobe(ap);
1368 }
1369 return ret;
64f562c6
AM
1370}
1371
be8f2743
MH
1372bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1373{
1374 /* The __kprobes marked functions and entry code must not be probed */
1375 return addr >= (unsigned long)__kprobes_text_start &&
1376 addr < (unsigned long)__kprobes_text_end;
1377}
1378
e5779e8e 1379bool within_kprobe_blacklist(unsigned long addr)
d0aaff97 1380{
376e2424 1381 struct kprobe_blacklist_entry *ent;
3d8d996e 1382
be8f2743 1383 if (arch_within_kprobe_blacklist(addr))
376e2424 1384 return true;
3d8d996e
SD
1385 /*
1386 * If there exists a kprobe_blacklist, verify and
1387 * fail any probe registration in the prohibited area
1388 */
376e2424
MH
1389 list_for_each_entry(ent, &kprobe_blacklist, list) {
1390 if (addr >= ent->start_addr && addr < ent->end_addr)
1391 return true;
3d8d996e 1392 }
376e2424
MH
1393
1394 return false;
d0aaff97
PP
1395}
1396
b2a5cd69
MH
1397/*
1398 * If we have a symbol_name argument, look it up and add the offset field
1399 * to it. This way, we can specify a relative address to a symbol.
bc81d48d
MH
1400 * This returns encoded errors if it fails to look up symbol or invalid
1401 * combination of parameters.
b2a5cd69 1402 */
1d585e70
NR
1403static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1404 const char *symbol_name, unsigned int offset)
b2a5cd69 1405{
1d585e70 1406 if ((symbol_name && addr) || (!symbol_name && !addr))
bc81d48d
MH
1407 goto invalid;
1408
1d585e70 1409 if (symbol_name) {
7246f600 1410 addr = kprobe_lookup_name(symbol_name, offset);
bc81d48d
MH
1411 if (!addr)
1412 return ERR_PTR(-ENOENT);
b2a5cd69
MH
1413 }
1414
1d585e70 1415 addr = (kprobe_opcode_t *)(((char *)addr) + offset);
bc81d48d
MH
1416 if (addr)
1417 return addr;
1418
1419invalid:
1420 return ERR_PTR(-EINVAL);
b2a5cd69
MH
1421}
1422
1d585e70
NR
1423static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1424{
1425 return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1426}
1427
1f0ab409 1428/* Check passed kprobe is valid and return kprobe in kprobe_table. */
55479f64 1429static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1f0ab409 1430{
6d8e40a8 1431 struct kprobe *ap, *list_p;
1f0ab409 1432
6d8e40a8
MH
1433 ap = get_kprobe(p->addr);
1434 if (unlikely(!ap))
1f0ab409
AM
1435 return NULL;
1436
6d8e40a8
MH
1437 if (p != ap) {
1438 list_for_each_entry_rcu(list_p, &ap->list, list)
1f0ab409
AM
1439 if (list_p == p)
1440 /* kprobe p is a valid probe */
1441 goto valid;
1442 return NULL;
1443 }
1444valid:
6d8e40a8 1445 return ap;
1f0ab409
AM
1446}
1447
1448/* Return error if the kprobe is being re-registered */
1449static inline int check_kprobe_rereg(struct kprobe *p)
1450{
1451 int ret = 0;
1f0ab409
AM
1452
1453 mutex_lock(&kprobe_mutex);
6d8e40a8 1454 if (__get_valid_kprobe(p))
1f0ab409
AM
1455 ret = -EINVAL;
1456 mutex_unlock(&kprobe_mutex);
6d8e40a8 1457
1f0ab409
AM
1458 return ret;
1459}
1460
f7f242ff 1461int __weak arch_check_ftrace_location(struct kprobe *p)
1da177e4 1462{
ae6aa16f
MH
1463 unsigned long ftrace_addr;
1464
ae6aa16f
MH
1465 ftrace_addr = ftrace_location((unsigned long)p->addr);
1466 if (ftrace_addr) {
e7dbfe34 1467#ifdef CONFIG_KPROBES_ON_FTRACE
ae6aa16f
MH
1468 /* Given address is not on the instruction boundary */
1469 if ((unsigned long)p->addr != ftrace_addr)
1470 return -EILSEQ;
ae6aa16f 1471 p->flags |= KPROBE_FLAG_FTRACE;
e7dbfe34 1472#else /* !CONFIG_KPROBES_ON_FTRACE */
ae6aa16f
MH
1473 return -EINVAL;
1474#endif
1475 }
f7f242ff
HC
1476 return 0;
1477}
1478
1479static int check_kprobe_address_safe(struct kprobe *p,
1480 struct module **probed_mod)
1481{
1482 int ret;
1f0ab409 1483
f7f242ff
HC
1484 ret = arch_check_ftrace_location(p);
1485 if (ret)
1486 return ret;
91bad2f8 1487 jump_label_lock();
de31c3ca 1488 preempt_disable();
f7fa6ef0
MH
1489
1490 /* Ensure it is not in reserved area nor out of text */
ec30c5f3 1491 if (!kernel_text_address((unsigned long) p->addr) ||
376e2424 1492 within_kprobe_blacklist((unsigned long) p->addr) ||
f986a499
PN
1493 jump_label_text_reserved(p->addr, p->addr)) {
1494 ret = -EINVAL;
f7fa6ef0 1495 goto out;
f986a499 1496 }
b3e55c72 1497
f7fa6ef0
MH
1498 /* Check if are we probing a module */
1499 *probed_mod = __module_text_address((unsigned long) p->addr);
1500 if (*probed_mod) {
6f716acd 1501 /*
e8386a0c
MH
1502 * We must hold a refcount of the probed module while updating
1503 * its code to prohibit unexpected unloading.
df019b1d 1504 */
f7fa6ef0
MH
1505 if (unlikely(!try_module_get(*probed_mod))) {
1506 ret = -ENOENT;
1507 goto out;
1508 }
de31c3ca 1509
f24659d9
MH
1510 /*
1511 * If the module freed .init.text, we couldn't insert
1512 * kprobes in there.
1513 */
f7fa6ef0
MH
1514 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1515 (*probed_mod)->state != MODULE_STATE_COMING) {
1516 module_put(*probed_mod);
1517 *probed_mod = NULL;
1518 ret = -ENOENT;
f24659d9 1519 }
df019b1d 1520 }
f7fa6ef0 1521out:
a189d035 1522 preempt_enable();
de31c3ca 1523 jump_label_unlock();
1da177e4 1524
f7fa6ef0
MH
1525 return ret;
1526}
1527
55479f64 1528int register_kprobe(struct kprobe *p)
f7fa6ef0
MH
1529{
1530 int ret;
1531 struct kprobe *old_p;
1532 struct module *probed_mod;
1533 kprobe_opcode_t *addr;
1534
1535 /* Adjust probe address from symbol */
1536 addr = kprobe_addr(p);
1537 if (IS_ERR(addr))
1538 return PTR_ERR(addr);
1539 p->addr = addr;
1540
1541 ret = check_kprobe_rereg(p);
1542 if (ret)
1543 return ret;
1544
1545 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1546 p->flags &= KPROBE_FLAG_DISABLED;
3516a460 1547 p->nmissed = 0;
9861668f 1548 INIT_LIST_HEAD(&p->list);
afd66255 1549
f7fa6ef0
MH
1550 ret = check_kprobe_address_safe(p, &probed_mod);
1551 if (ret)
1552 return ret;
1553
1554 mutex_lock(&kprobe_mutex);
afd66255 1555
64f562c6
AM
1556 old_p = get_kprobe(p->addr);
1557 if (old_p) {
afd66255 1558 /* Since this may unoptimize old_p, locking text_mutex. */
64f562c6 1559 ret = register_aggr_kprobe(old_p, p);
1da177e4
LT
1560 goto out;
1561 }
1da177e4 1562
2d1e38f5
TG
1563 cpus_read_lock();
1564 /* Prevent text modification */
1565 mutex_lock(&text_mutex);
ae6aa16f 1566 ret = prepare_kprobe(p);
25764288 1567 mutex_unlock(&text_mutex);
2d1e38f5 1568 cpus_read_unlock();
6f716acd 1569 if (ret)
afd66255 1570 goto out;
49a2a1b8 1571
64f562c6 1572 INIT_HLIST_NODE(&p->hlist);
3516a460 1573 hlist_add_head_rcu(&p->hlist,
1da177e4
LT
1574 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1575
de5bd88d 1576 if (!kprobes_all_disarmed && !kprobe_disabled(p))
25764288 1577 arm_kprobe(p);
afd66255
MH
1578
1579 /* Try to optimize kprobe */
1580 try_to_optimize_kprobe(p);
1da177e4 1581out:
7a7d1cf9 1582 mutex_unlock(&kprobe_mutex);
49a2a1b8 1583
e8386a0c 1584 if (probed_mod)
df019b1d 1585 module_put(probed_mod);
e8386a0c 1586
1da177e4
LT
1587 return ret;
1588}
99081ab5 1589EXPORT_SYMBOL_GPL(register_kprobe);
1da177e4 1590
6f0f1dd7 1591/* Check if all probes on the aggrprobe are disabled */
55479f64 1592static int aggr_kprobe_disabled(struct kprobe *ap)
6f0f1dd7
MH
1593{
1594 struct kprobe *kp;
1595
1596 list_for_each_entry_rcu(kp, &ap->list, list)
1597 if (!kprobe_disabled(kp))
1598 /*
1599 * There is an active probe on the list.
1600 * We can't disable this ap.
1601 */
1602 return 0;
1603
1604 return 1;
1605}
1606
1607/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
55479f64 1608static struct kprobe *__disable_kprobe(struct kprobe *p)
6f0f1dd7
MH
1609{
1610 struct kprobe *orig_p;
1611
1612 /* Get an original kprobe for return */
1613 orig_p = __get_valid_kprobe(p);
1614 if (unlikely(orig_p == NULL))
1615 return NULL;
1616
1617 if (!kprobe_disabled(p)) {
1618 /* Disable probe if it is a child probe */
1619 if (p != orig_p)
1620 p->flags |= KPROBE_FLAG_DISABLED;
1621
1622 /* Try to disarm and disable this/parent probe */
1623 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
69d54b91
WN
1624 /*
1625 * If kprobes_all_disarmed is set, orig_p
1626 * should have already been disarmed, so
1627 * skip unneed disarming process.
1628 */
1629 if (!kprobes_all_disarmed)
1630 disarm_kprobe(orig_p, true);
6f0f1dd7
MH
1631 orig_p->flags |= KPROBE_FLAG_DISABLED;
1632 }
1633 }
1634
1635 return orig_p;
1636}
1637
de5bd88d
MH
1638/*
1639 * Unregister a kprobe without a scheduler synchronization.
1640 */
55479f64 1641static int __unregister_kprobe_top(struct kprobe *p)
de5bd88d 1642{
6d8e40a8 1643 struct kprobe *ap, *list_p;
de5bd88d 1644
6f0f1dd7
MH
1645 /* Disable kprobe. This will disarm it if needed. */
1646 ap = __disable_kprobe(p);
6d8e40a8 1647 if (ap == NULL)
de5bd88d
MH
1648 return -EINVAL;
1649
6f0f1dd7 1650 if (ap == p)
bf8f6e5b 1651 /*
6f0f1dd7
MH
1652 * This probe is an independent(and non-optimized) kprobe
1653 * (not an aggrprobe). Remove from the hash list.
bf8f6e5b 1654 */
6f0f1dd7
MH
1655 goto disarmed;
1656
1657 /* Following process expects this probe is an aggrprobe */
1658 WARN_ON(!kprobe_aggrprobe(ap));
1659
6274de49
MH
1660 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1661 /*
1662 * !disarmed could be happen if the probe is under delayed
1663 * unoptimizing.
1664 */
6f0f1dd7
MH
1665 goto disarmed;
1666 else {
1667 /* If disabling probe has special handlers, update aggrprobe */
e8386a0c 1668 if (p->break_handler && !kprobe_gone(p))
6d8e40a8 1669 ap->break_handler = NULL;
e8386a0c 1670 if (p->post_handler && !kprobe_gone(p)) {
6d8e40a8 1671 list_for_each_entry_rcu(list_p, &ap->list, list) {
9861668f
MH
1672 if ((list_p != p) && (list_p->post_handler))
1673 goto noclean;
1674 }
6d8e40a8 1675 ap->post_handler = NULL;
9861668f
MH
1676 }
1677noclean:
6f0f1dd7
MH
1678 /*
1679 * Remove from the aggrprobe: this path will do nothing in
1680 * __unregister_kprobe_bottom().
1681 */
49a2a1b8 1682 list_del_rcu(&p->list);
6f0f1dd7
MH
1683 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1684 /*
1685 * Try to optimize this probe again, because post
1686 * handler may have been changed.
1687 */
1688 optimize_kprobe(ap);
49a2a1b8 1689 }
9861668f 1690 return 0;
6f0f1dd7
MH
1691
1692disarmed:
6274de49 1693 BUG_ON(!kprobe_disarmed(ap));
6f0f1dd7
MH
1694 hlist_del_rcu(&ap->hlist);
1695 return 0;
9861668f 1696}
3516a460 1697
55479f64 1698static void __unregister_kprobe_bottom(struct kprobe *p)
9861668f 1699{
6d8e40a8 1700 struct kprobe *ap;
b3e55c72 1701
e8386a0c 1702 if (list_empty(&p->list))
6274de49 1703 /* This is an independent kprobe */
0498b635 1704 arch_remove_kprobe(p);
e8386a0c 1705 else if (list_is_singular(&p->list)) {
6274de49 1706 /* This is the last child of an aggrprobe */
6d8e40a8 1707 ap = list_entry(p->list.next, struct kprobe, list);
e8386a0c 1708 list_del(&p->list);
6d8e40a8 1709 free_aggr_kprobe(ap);
9861668f 1710 }
6274de49 1711 /* Otherwise, do nothing. */
9861668f
MH
1712}
1713
55479f64 1714int register_kprobes(struct kprobe **kps, int num)
9861668f
MH
1715{
1716 int i, ret = 0;
1717
1718 if (num <= 0)
1719 return -EINVAL;
1720 for (i = 0; i < num; i++) {
49ad2fd7 1721 ret = register_kprobe(kps[i]);
67dddaad
MH
1722 if (ret < 0) {
1723 if (i > 0)
1724 unregister_kprobes(kps, i);
9861668f 1725 break;
36721656 1726 }
49a2a1b8 1727 }
9861668f
MH
1728 return ret;
1729}
99081ab5 1730EXPORT_SYMBOL_GPL(register_kprobes);
9861668f 1731
55479f64 1732void unregister_kprobe(struct kprobe *p)
9861668f
MH
1733{
1734 unregister_kprobes(&p, 1);
1735}
99081ab5 1736EXPORT_SYMBOL_GPL(unregister_kprobe);
9861668f 1737
55479f64 1738void unregister_kprobes(struct kprobe **kps, int num)
9861668f
MH
1739{
1740 int i;
1741
1742 if (num <= 0)
1743 return;
1744 mutex_lock(&kprobe_mutex);
1745 for (i = 0; i < num; i++)
1746 if (__unregister_kprobe_top(kps[i]) < 0)
1747 kps[i]->addr = NULL;
1748 mutex_unlock(&kprobe_mutex);
1749
1750 synchronize_sched();
1751 for (i = 0; i < num; i++)
1752 if (kps[i]->addr)
1753 __unregister_kprobe_bottom(kps[i]);
1da177e4 1754}
99081ab5 1755EXPORT_SYMBOL_GPL(unregister_kprobes);
1da177e4 1756
5f6bee34
NR
1757int __weak kprobe_exceptions_notify(struct notifier_block *self,
1758 unsigned long val, void *data)
fc62d020
NR
1759{
1760 return NOTIFY_DONE;
1761}
5f6bee34 1762NOKPROBE_SYMBOL(kprobe_exceptions_notify);
fc62d020 1763
1da177e4 1764static struct notifier_block kprobe_exceptions_nb = {
3d5631e0
AK
1765 .notifier_call = kprobe_exceptions_notify,
1766 .priority = 0x7fffffff /* we need to be notified first */
1767};
1768
3d7e3382
ME
1769unsigned long __weak arch_deref_entry_point(void *entry)
1770{
1771 return (unsigned long)entry;
1772}
1da177e4 1773
590c8459 1774#if 0
55479f64 1775int register_jprobes(struct jprobe **jps, int num)
1da177e4 1776{
26b31c19 1777 int ret = 0, i;
3d7e3382 1778
26b31c19 1779 if (num <= 0)
3d7e3382 1780 return -EINVAL;
0f73ff80 1781
26b31c19 1782 for (i = 0; i < num; i++) {
0f73ff80 1783 ret = register_jprobe(jps[i]);
edbaadbe 1784
67dddaad
MH
1785 if (ret < 0) {
1786 if (i > 0)
1787 unregister_jprobes(jps, i);
26b31c19
MH
1788 break;
1789 }
1790 }
0f73ff80 1791
26b31c19
MH
1792 return ret;
1793}
99081ab5 1794EXPORT_SYMBOL_GPL(register_jprobes);
3d7e3382 1795
55479f64 1796int register_jprobe(struct jprobe *jp)
26b31c19 1797{
0f73ff80
NR
1798 unsigned long addr, offset;
1799 struct kprobe *kp = &jp->kp;
1800
dbf58062
NR
1801 /*
1802 * Verify probepoint as well as the jprobe handler are
1803 * valid function entry points.
1804 */
0f73ff80
NR
1805 addr = arch_deref_entry_point(jp->entry);
1806
dbf58062
NR
1807 if (kallsyms_lookup_size_offset(addr, NULL, &offset) && offset == 0 &&
1808 kprobe_on_func_entry(kp->addr, kp->symbol_name, kp->offset)) {
0f73ff80
NR
1809 kp->pre_handler = setjmp_pre_handler;
1810 kp->break_handler = longjmp_break_handler;
1811 return register_kprobe(kp);
1812 }
1813
1814 return -EINVAL;
1da177e4 1815}
99081ab5 1816EXPORT_SYMBOL_GPL(register_jprobe);
1da177e4 1817
55479f64 1818void unregister_jprobe(struct jprobe *jp)
1da177e4 1819{
26b31c19
MH
1820 unregister_jprobes(&jp, 1);
1821}
99081ab5 1822EXPORT_SYMBOL_GPL(unregister_jprobe);
26b31c19 1823
55479f64 1824void unregister_jprobes(struct jprobe **jps, int num)
26b31c19
MH
1825{
1826 int i;
1827
1828 if (num <= 0)
1829 return;
1830 mutex_lock(&kprobe_mutex);
1831 for (i = 0; i < num; i++)
1832 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1833 jps[i]->kp.addr = NULL;
1834 mutex_unlock(&kprobe_mutex);
1835
1836 synchronize_sched();
1837 for (i = 0; i < num; i++) {
1838 if (jps[i]->kp.addr)
1839 __unregister_kprobe_bottom(&jps[i]->kp);
1840 }
1da177e4 1841}
99081ab5 1842EXPORT_SYMBOL_GPL(unregister_jprobes);
590c8459 1843#endif
1da177e4 1844
9edddaa2 1845#ifdef CONFIG_KRETPROBES
e65cefe8
AB
1846/*
1847 * This kprobe pre_handler is registered with every kretprobe. When probe
1848 * hits it will set up the return probe.
1849 */
820aede0 1850static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
e65cefe8
AB
1851{
1852 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
ef53d9c5
S
1853 unsigned long hash, flags = 0;
1854 struct kretprobe_instance *ri;
e65cefe8 1855
f96f5678
MH
1856 /*
1857 * To avoid deadlocks, prohibit return probing in NMI contexts,
1858 * just skip the probe and increase the (inexact) 'nmissed'
1859 * statistical counter, so that the user is informed that
1860 * something happened:
1861 */
1862 if (unlikely(in_nmi())) {
1863 rp->nmissed++;
1864 return 0;
1865 }
1866
1867 /* TODO: consider to only swap the RA after the last pre_handler fired */
ef53d9c5 1868 hash = hash_ptr(current, KPROBE_HASH_BITS);
ec484608 1869 raw_spin_lock_irqsave(&rp->lock, flags);
4c4308cb 1870 if (!hlist_empty(&rp->free_instances)) {
4c4308cb 1871 ri = hlist_entry(rp->free_instances.first,
ef53d9c5
S
1872 struct kretprobe_instance, hlist);
1873 hlist_del(&ri->hlist);
ec484608 1874 raw_spin_unlock_irqrestore(&rp->lock, flags);
ef53d9c5 1875
4c4308cb
CH
1876 ri->rp = rp;
1877 ri->task = current;
f47cd9b5 1878
55ca6140
JL
1879 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1880 raw_spin_lock_irqsave(&rp->lock, flags);
1881 hlist_add_head(&ri->hlist, &rp->free_instances);
1882 raw_spin_unlock_irqrestore(&rp->lock, flags);
f47cd9b5 1883 return 0;
55ca6140 1884 }
f47cd9b5 1885
4c4308cb
CH
1886 arch_prepare_kretprobe(ri, regs);
1887
1888 /* XXX(hch): why is there no hlist_move_head? */
ef53d9c5
S
1889 INIT_HLIST_NODE(&ri->hlist);
1890 kretprobe_table_lock(hash, &flags);
1891 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1892 kretprobe_table_unlock(hash, &flags);
1893 } else {
4c4308cb 1894 rp->nmissed++;
ec484608 1895 raw_spin_unlock_irqrestore(&rp->lock, flags);
ef53d9c5 1896 }
e65cefe8
AB
1897 return 0;
1898}
820aede0 1899NOKPROBE_SYMBOL(pre_handler_kretprobe);
e65cefe8 1900
659b957f 1901bool __weak arch_kprobe_on_func_entry(unsigned long offset)
90ec5e89
NR
1902{
1903 return !offset;
1904}
1905
659b957f 1906bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
1d585e70
NR
1907{
1908 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
1909
1910 if (IS_ERR(kp_addr))
1911 return false;
1912
1913 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
659b957f 1914 !arch_kprobe_on_func_entry(offset))
1d585e70
NR
1915 return false;
1916
1917 return true;
1918}
1919
55479f64 1920int register_kretprobe(struct kretprobe *rp)
b94cce92
HN
1921{
1922 int ret = 0;
1923 struct kretprobe_instance *inst;
1924 int i;
b2a5cd69 1925 void *addr;
90ec5e89 1926
659b957f 1927 if (!kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset))
90ec5e89 1928 return -EINVAL;
f438d914
MH
1929
1930 if (kretprobe_blacklist_size) {
b2a5cd69 1931 addr = kprobe_addr(&rp->kp);
bc81d48d
MH
1932 if (IS_ERR(addr))
1933 return PTR_ERR(addr);
f438d914
MH
1934
1935 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1936 if (kretprobe_blacklist[i].addr == addr)
1937 return -EINVAL;
1938 }
1939 }
b94cce92
HN
1940
1941 rp->kp.pre_handler = pre_handler_kretprobe;
7522a842
AM
1942 rp->kp.post_handler = NULL;
1943 rp->kp.fault_handler = NULL;
1944 rp->kp.break_handler = NULL;
b94cce92
HN
1945
1946 /* Pre-allocate memory for max kretprobe instances */
1947 if (rp->maxactive <= 0) {
1948#ifdef CONFIG_PREEMPT
c2ef6661 1949 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
b94cce92 1950#else
4dae560f 1951 rp->maxactive = num_possible_cpus();
b94cce92
HN
1952#endif
1953 }
ec484608 1954 raw_spin_lock_init(&rp->lock);
b94cce92
HN
1955 INIT_HLIST_HEAD(&rp->free_instances);
1956 for (i = 0; i < rp->maxactive; i++) {
f47cd9b5
AS
1957 inst = kmalloc(sizeof(struct kretprobe_instance) +
1958 rp->data_size, GFP_KERNEL);
b94cce92
HN
1959 if (inst == NULL) {
1960 free_rp_inst(rp);
1961 return -ENOMEM;
1962 }
ef53d9c5
S
1963 INIT_HLIST_NODE(&inst->hlist);
1964 hlist_add_head(&inst->hlist, &rp->free_instances);
b94cce92
HN
1965 }
1966
1967 rp->nmissed = 0;
1968 /* Establish function entry probe point */
49ad2fd7 1969 ret = register_kprobe(&rp->kp);
4a296e07 1970 if (ret != 0)
b94cce92
HN
1971 free_rp_inst(rp);
1972 return ret;
1973}
99081ab5 1974EXPORT_SYMBOL_GPL(register_kretprobe);
b94cce92 1975
55479f64 1976int register_kretprobes(struct kretprobe **rps, int num)
4a296e07
MH
1977{
1978 int ret = 0, i;
1979
1980 if (num <= 0)
1981 return -EINVAL;
1982 for (i = 0; i < num; i++) {
49ad2fd7 1983 ret = register_kretprobe(rps[i]);
67dddaad
MH
1984 if (ret < 0) {
1985 if (i > 0)
1986 unregister_kretprobes(rps, i);
4a296e07
MH
1987 break;
1988 }
1989 }
1990 return ret;
1991}
99081ab5 1992EXPORT_SYMBOL_GPL(register_kretprobes);
4a296e07 1993
55479f64 1994void unregister_kretprobe(struct kretprobe *rp)
4a296e07
MH
1995{
1996 unregister_kretprobes(&rp, 1);
1997}
99081ab5 1998EXPORT_SYMBOL_GPL(unregister_kretprobe);
4a296e07 1999
55479f64 2000void unregister_kretprobes(struct kretprobe **rps, int num)
4a296e07
MH
2001{
2002 int i;
2003
2004 if (num <= 0)
2005 return;
2006 mutex_lock(&kprobe_mutex);
2007 for (i = 0; i < num; i++)
2008 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
2009 rps[i]->kp.addr = NULL;
2010 mutex_unlock(&kprobe_mutex);
2011
2012 synchronize_sched();
2013 for (i = 0; i < num; i++) {
2014 if (rps[i]->kp.addr) {
2015 __unregister_kprobe_bottom(&rps[i]->kp);
2016 cleanup_rp_inst(rps[i]);
2017 }
2018 }
2019}
99081ab5 2020EXPORT_SYMBOL_GPL(unregister_kretprobes);
4a296e07 2021
9edddaa2 2022#else /* CONFIG_KRETPROBES */
55479f64 2023int register_kretprobe(struct kretprobe *rp)
b94cce92
HN
2024{
2025 return -ENOSYS;
2026}
99081ab5 2027EXPORT_SYMBOL_GPL(register_kretprobe);
b94cce92 2028
55479f64 2029int register_kretprobes(struct kretprobe **rps, int num)
346fd59b 2030{
4a296e07 2031 return -ENOSYS;
346fd59b 2032}
99081ab5
MH
2033EXPORT_SYMBOL_GPL(register_kretprobes);
2034
55479f64 2035void unregister_kretprobe(struct kretprobe *rp)
b94cce92 2036{
4a296e07 2037}
99081ab5 2038EXPORT_SYMBOL_GPL(unregister_kretprobe);
b94cce92 2039
55479f64 2040void unregister_kretprobes(struct kretprobe **rps, int num)
4a296e07
MH
2041{
2042}
99081ab5 2043EXPORT_SYMBOL_GPL(unregister_kretprobes);
4c4308cb 2044
820aede0 2045static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
4a296e07
MH
2046{
2047 return 0;
b94cce92 2048}
820aede0 2049NOKPROBE_SYMBOL(pre_handler_kretprobe);
b94cce92 2050
4a296e07
MH
2051#endif /* CONFIG_KRETPROBES */
2052
e8386a0c 2053/* Set the kprobe gone and remove its instruction buffer. */
55479f64 2054static void kill_kprobe(struct kprobe *p)
e8386a0c
MH
2055{
2056 struct kprobe *kp;
de5bd88d 2057
e8386a0c 2058 p->flags |= KPROBE_FLAG_GONE;
afd66255 2059 if (kprobe_aggrprobe(p)) {
e8386a0c
MH
2060 /*
2061 * If this is an aggr_kprobe, we have to list all the
2062 * chained probes and mark them GONE.
2063 */
2064 list_for_each_entry_rcu(kp, &p->list, list)
2065 kp->flags |= KPROBE_FLAG_GONE;
2066 p->post_handler = NULL;
2067 p->break_handler = NULL;
afd66255 2068 kill_optimized_kprobe(p);
e8386a0c
MH
2069 }
2070 /*
2071 * Here, we can remove insn_slot safely, because no thread calls
2072 * the original probed function (which will be freed soon) any more.
2073 */
2074 arch_remove_kprobe(p);
2075}
2076
c0614829 2077/* Disable one kprobe */
55479f64 2078int disable_kprobe(struct kprobe *kp)
c0614829
MH
2079{
2080 int ret = 0;
c0614829
MH
2081
2082 mutex_lock(&kprobe_mutex);
2083
6f0f1dd7
MH
2084 /* Disable this kprobe */
2085 if (__disable_kprobe(kp) == NULL)
c0614829 2086 ret = -EINVAL;
c0614829 2087
c0614829
MH
2088 mutex_unlock(&kprobe_mutex);
2089 return ret;
2090}
2091EXPORT_SYMBOL_GPL(disable_kprobe);
2092
2093/* Enable one kprobe */
55479f64 2094int enable_kprobe(struct kprobe *kp)
c0614829
MH
2095{
2096 int ret = 0;
2097 struct kprobe *p;
2098
2099 mutex_lock(&kprobe_mutex);
2100
2101 /* Check whether specified probe is valid. */
2102 p = __get_valid_kprobe(kp);
2103 if (unlikely(p == NULL)) {
2104 ret = -EINVAL;
2105 goto out;
2106 }
2107
2108 if (kprobe_gone(kp)) {
2109 /* This kprobe has gone, we couldn't enable it. */
2110 ret = -EINVAL;
2111 goto out;
2112 }
2113
2114 if (p != kp)
2115 kp->flags &= ~KPROBE_FLAG_DISABLED;
2116
2117 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2118 p->flags &= ~KPROBE_FLAG_DISABLED;
2119 arm_kprobe(p);
2120 }
2121out:
2122 mutex_unlock(&kprobe_mutex);
2123 return ret;
2124}
2125EXPORT_SYMBOL_GPL(enable_kprobe);
2126
820aede0 2127void dump_kprobe(struct kprobe *kp)
24851d24
FW
2128{
2129 printk(KERN_WARNING "Dumping kprobe:\n");
2130 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
2131 kp->symbol_name, kp->addr, kp->offset);
2132}
820aede0 2133NOKPROBE_SYMBOL(dump_kprobe);
24851d24 2134
376e2424
MH
2135/*
2136 * Lookup and populate the kprobe_blacklist.
2137 *
2138 * Unlike the kretprobe blacklist, we'll need to determine
2139 * the range of addresses that belong to the said functions,
2140 * since a kprobe need not necessarily be at the beginning
2141 * of a function.
2142 */
2143static int __init populate_kprobe_blacklist(unsigned long *start,
2144 unsigned long *end)
2145{
2146 unsigned long *iter;
2147 struct kprobe_blacklist_entry *ent;
d81b4253 2148 unsigned long entry, offset = 0, size = 0;
376e2424
MH
2149
2150 for (iter = start; iter < end; iter++) {
d81b4253
MH
2151 entry = arch_deref_entry_point((void *)*iter);
2152
2153 if (!kernel_text_address(entry) ||
2154 !kallsyms_lookup_size_offset(entry, &size, &offset)) {
2155 pr_err("Failed to find blacklist at %p\n",
2156 (void *)entry);
376e2424
MH
2157 continue;
2158 }
2159
2160 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2161 if (!ent)
2162 return -ENOMEM;
d81b4253
MH
2163 ent->start_addr = entry;
2164 ent->end_addr = entry + size;
376e2424
MH
2165 INIT_LIST_HEAD(&ent->list);
2166 list_add_tail(&ent->list, &kprobe_blacklist);
2167 }
2168 return 0;
2169}
2170
e8386a0c 2171/* Module notifier call back, checking kprobes on the module */
55479f64
MH
2172static int kprobes_module_callback(struct notifier_block *nb,
2173 unsigned long val, void *data)
e8386a0c
MH
2174{
2175 struct module *mod = data;
2176 struct hlist_head *head;
e8386a0c
MH
2177 struct kprobe *p;
2178 unsigned int i;
f24659d9 2179 int checkcore = (val == MODULE_STATE_GOING);
e8386a0c 2180
f24659d9 2181 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
e8386a0c
MH
2182 return NOTIFY_DONE;
2183
2184 /*
f24659d9
MH
2185 * When MODULE_STATE_GOING was notified, both of module .text and
2186 * .init.text sections would be freed. When MODULE_STATE_LIVE was
2187 * notified, only .init.text section would be freed. We need to
2188 * disable kprobes which have been inserted in the sections.
e8386a0c
MH
2189 */
2190 mutex_lock(&kprobe_mutex);
2191 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2192 head = &kprobe_table[i];
b67bfe0d 2193 hlist_for_each_entry_rcu(p, head, hlist)
f24659d9
MH
2194 if (within_module_init((unsigned long)p->addr, mod) ||
2195 (checkcore &&
2196 within_module_core((unsigned long)p->addr, mod))) {
e8386a0c
MH
2197 /*
2198 * The vaddr this probe is installed will soon
2199 * be vfreed buy not synced to disk. Hence,
2200 * disarming the breakpoint isn't needed.
545a0281
SRV
2201 *
2202 * Note, this will also move any optimized probes
2203 * that are pending to be removed from their
2204 * corresponding lists to the freeing_list and
2205 * will not be touched by the delayed
2206 * kprobe_optimizer work handler.
e8386a0c
MH
2207 */
2208 kill_kprobe(p);
2209 }
2210 }
2211 mutex_unlock(&kprobe_mutex);
2212 return NOTIFY_DONE;
2213}
2214
2215static struct notifier_block kprobe_module_nb = {
2216 .notifier_call = kprobes_module_callback,
2217 .priority = 0
2218};
2219
376e2424
MH
2220/* Markers of _kprobe_blacklist section */
2221extern unsigned long __start_kprobe_blacklist[];
2222extern unsigned long __stop_kprobe_blacklist[];
2223
1da177e4
LT
2224static int __init init_kprobes(void)
2225{
2226 int i, err = 0;
2227
2228 /* FIXME allocate the probe table, currently defined statically */
2229 /* initialize all list heads */
b94cce92 2230 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1da177e4 2231 INIT_HLIST_HEAD(&kprobe_table[i]);
b94cce92 2232 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
ec484608 2233 raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
b94cce92 2234 }
1da177e4 2235
376e2424
MH
2236 err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2237 __stop_kprobe_blacklist);
2238 if (err) {
2239 pr_err("kprobes: failed to populate blacklist: %d\n", err);
2240 pr_err("Please take care of using kprobes.\n");
3d8d996e
SD
2241 }
2242
f438d914
MH
2243 if (kretprobe_blacklist_size) {
2244 /* lookup the function address from its name */
2245 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
49e0b465 2246 kretprobe_blacklist[i].addr =
290e3070 2247 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
f438d914
MH
2248 if (!kretprobe_blacklist[i].addr)
2249 printk("kretprobe: lookup failed: %s\n",
2250 kretprobe_blacklist[i].name);
2251 }
2252 }
2253
b2be84df
MH
2254#if defined(CONFIG_OPTPROBES)
2255#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
afd66255
MH
2256 /* Init kprobe_optinsn_slots */
2257 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2258#endif
b2be84df
MH
2259 /* By default, kprobes can be optimized */
2260 kprobes_allow_optimization = true;
2261#endif
afd66255 2262
e579abeb
MH
2263 /* By default, kprobes are armed */
2264 kprobes_all_disarmed = false;
bf8f6e5b 2265
6772926b 2266 err = arch_init_kprobes();
802eae7c
RL
2267 if (!err)
2268 err = register_die_notifier(&kprobe_exceptions_nb);
e8386a0c
MH
2269 if (!err)
2270 err = register_module_notifier(&kprobe_module_nb);
2271
ef53d9c5 2272 kprobes_initialized = (err == 0);
802eae7c 2273
8c1c9356
AM
2274 if (!err)
2275 init_test_probes();
1da177e4
LT
2276 return err;
2277}
2278
346fd59b 2279#ifdef CONFIG_DEBUG_FS
55479f64 2280static void report_probe(struct seq_file *pi, struct kprobe *p,
afd66255 2281 const char *sym, int offset, char *modname, struct kprobe *pp)
346fd59b
SD
2282{
2283 char *kprobe_type;
2284
2285 if (p->pre_handler == pre_handler_kretprobe)
2286 kprobe_type = "r";
2287 else if (p->pre_handler == setjmp_pre_handler)
2288 kprobe_type = "j";
2289 else
2290 kprobe_type = "k";
afd66255 2291
346fd59b 2292 if (sym)
afd66255 2293 seq_printf(pi, "%p %s %s+0x%x %s ",
de5bd88d 2294 p->addr, kprobe_type, sym, offset,
afd66255 2295 (modname ? modname : " "));
346fd59b 2296 else
afd66255
MH
2297 seq_printf(pi, "%p %s %p ",
2298 p->addr, kprobe_type, p->addr);
2299
2300 if (!pp)
2301 pp = p;
ae6aa16f 2302 seq_printf(pi, "%s%s%s%s\n",
afd66255
MH
2303 (kprobe_gone(p) ? "[GONE]" : ""),
2304 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
ae6aa16f
MH
2305 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2306 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
346fd59b
SD
2307}
2308
55479f64 2309static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
346fd59b
SD
2310{
2311 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2312}
2313
55479f64 2314static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
346fd59b
SD
2315{
2316 (*pos)++;
2317 if (*pos >= KPROBE_TABLE_SIZE)
2318 return NULL;
2319 return pos;
2320}
2321
55479f64 2322static void kprobe_seq_stop(struct seq_file *f, void *v)
346fd59b
SD
2323{
2324 /* Nothing to do */
2325}
2326
55479f64 2327static int show_kprobe_addr(struct seq_file *pi, void *v)
346fd59b
SD
2328{
2329 struct hlist_head *head;
346fd59b
SD
2330 struct kprobe *p, *kp;
2331 const char *sym = NULL;
2332 unsigned int i = *(loff_t *) v;
ffb45122 2333 unsigned long offset = 0;
ab767865 2334 char *modname, namebuf[KSYM_NAME_LEN];
346fd59b
SD
2335
2336 head = &kprobe_table[i];
2337 preempt_disable();
b67bfe0d 2338 hlist_for_each_entry_rcu(p, head, hlist) {
ffb45122 2339 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
346fd59b 2340 &offset, &modname, namebuf);
afd66255 2341 if (kprobe_aggrprobe(p)) {
346fd59b 2342 list_for_each_entry_rcu(kp, &p->list, list)
afd66255 2343 report_probe(pi, kp, sym, offset, modname, p);
346fd59b 2344 } else
afd66255 2345 report_probe(pi, p, sym, offset, modname, NULL);
346fd59b
SD
2346 }
2347 preempt_enable();
2348 return 0;
2349}
2350
88e9d34c 2351static const struct seq_operations kprobes_seq_ops = {
346fd59b
SD
2352 .start = kprobe_seq_start,
2353 .next = kprobe_seq_next,
2354 .stop = kprobe_seq_stop,
2355 .show = show_kprobe_addr
2356};
2357
55479f64 2358static int kprobes_open(struct inode *inode, struct file *filp)
346fd59b
SD
2359{
2360 return seq_open(filp, &kprobes_seq_ops);
2361}
2362
828c0950 2363static const struct file_operations debugfs_kprobes_operations = {
346fd59b
SD
2364 .open = kprobes_open,
2365 .read = seq_read,
2366 .llseek = seq_lseek,
2367 .release = seq_release,
2368};
2369
63724740
MH
2370/* kprobes/blacklist -- shows which functions can not be probed */
2371static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2372{
2373 return seq_list_start(&kprobe_blacklist, *pos);
2374}
2375
2376static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2377{
2378 return seq_list_next(v, &kprobe_blacklist, pos);
2379}
2380
2381static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2382{
2383 struct kprobe_blacklist_entry *ent =
2384 list_entry(v, struct kprobe_blacklist_entry, list);
2385
2386 seq_printf(m, "0x%p-0x%p\t%ps\n", (void *)ent->start_addr,
2387 (void *)ent->end_addr, (void *)ent->start_addr);
2388 return 0;
2389}
2390
2391static const struct seq_operations kprobe_blacklist_seq_ops = {
2392 .start = kprobe_blacklist_seq_start,
2393 .next = kprobe_blacklist_seq_next,
2394 .stop = kprobe_seq_stop, /* Reuse void function */
2395 .show = kprobe_blacklist_seq_show,
2396};
2397
2398static int kprobe_blacklist_open(struct inode *inode, struct file *filp)
2399{
2400 return seq_open(filp, &kprobe_blacklist_seq_ops);
2401}
2402
2403static const struct file_operations debugfs_kprobe_blacklist_ops = {
2404 .open = kprobe_blacklist_open,
2405 .read = seq_read,
2406 .llseek = seq_lseek,
2407 .release = seq_release,
2408};
2409
55479f64 2410static void arm_all_kprobes(void)
bf8f6e5b
AM
2411{
2412 struct hlist_head *head;
bf8f6e5b
AM
2413 struct kprobe *p;
2414 unsigned int i;
2415
2416 mutex_lock(&kprobe_mutex);
2417
e579abeb
MH
2418 /* If kprobes are armed, just return */
2419 if (!kprobes_all_disarmed)
bf8f6e5b
AM
2420 goto already_enabled;
2421
977ad481
WN
2422 /*
2423 * optimize_kprobe() called by arm_kprobe() checks
2424 * kprobes_all_disarmed, so set kprobes_all_disarmed before
2425 * arm_kprobe.
2426 */
2427 kprobes_all_disarmed = false;
afd66255 2428 /* Arming kprobes doesn't optimize kprobe itself */
bf8f6e5b
AM
2429 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2430 head = &kprobe_table[i];
b67bfe0d 2431 hlist_for_each_entry_rcu(p, head, hlist)
de5bd88d 2432 if (!kprobe_disabled(p))
ae6aa16f 2433 arm_kprobe(p);
bf8f6e5b
AM
2434 }
2435
bf8f6e5b
AM
2436 printk(KERN_INFO "Kprobes globally enabled\n");
2437
2438already_enabled:
2439 mutex_unlock(&kprobe_mutex);
2440 return;
2441}
2442
55479f64 2443static void disarm_all_kprobes(void)
bf8f6e5b
AM
2444{
2445 struct hlist_head *head;
bf8f6e5b
AM
2446 struct kprobe *p;
2447 unsigned int i;
2448
2449 mutex_lock(&kprobe_mutex);
2450
e579abeb 2451 /* If kprobes are already disarmed, just return */
6274de49
MH
2452 if (kprobes_all_disarmed) {
2453 mutex_unlock(&kprobe_mutex);
2454 return;
2455 }
bf8f6e5b 2456
e579abeb 2457 kprobes_all_disarmed = true;
bf8f6e5b 2458 printk(KERN_INFO "Kprobes globally disabled\n");
afd66255 2459
bf8f6e5b
AM
2460 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2461 head = &kprobe_table[i];
b67bfe0d 2462 hlist_for_each_entry_rcu(p, head, hlist) {
de5bd88d 2463 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
ae6aa16f 2464 disarm_kprobe(p, false);
bf8f6e5b
AM
2465 }
2466 }
bf8f6e5b 2467 mutex_unlock(&kprobe_mutex);
bf8f6e5b 2468
6274de49
MH
2469 /* Wait for disarming all kprobes by optimizer */
2470 wait_for_kprobe_optimizer();
bf8f6e5b
AM
2471}
2472
2473/*
2474 * XXX: The debugfs bool file interface doesn't allow for callbacks
2475 * when the bool state is switched. We can reuse that facility when
2476 * available
2477 */
2478static ssize_t read_enabled_file_bool(struct file *file,
2479 char __user *user_buf, size_t count, loff_t *ppos)
2480{
2481 char buf[3];
2482
e579abeb 2483 if (!kprobes_all_disarmed)
bf8f6e5b
AM
2484 buf[0] = '1';
2485 else
2486 buf[0] = '0';
2487 buf[1] = '\n';
2488 buf[2] = 0x00;
2489 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2490}
2491
2492static ssize_t write_enabled_file_bool(struct file *file,
2493 const char __user *user_buf, size_t count, loff_t *ppos)
2494{
2495 char buf[32];
efeb156e 2496 size_t buf_size;
bf8f6e5b
AM
2497
2498 buf_size = min(count, (sizeof(buf)-1));
2499 if (copy_from_user(buf, user_buf, buf_size))
2500 return -EFAULT;
2501
10fb46d5 2502 buf[buf_size] = '\0';
bf8f6e5b
AM
2503 switch (buf[0]) {
2504 case 'y':
2505 case 'Y':
2506 case '1':
e579abeb 2507 arm_all_kprobes();
bf8f6e5b
AM
2508 break;
2509 case 'n':
2510 case 'N':
2511 case '0':
e579abeb 2512 disarm_all_kprobes();
bf8f6e5b 2513 break;
10fb46d5
MK
2514 default:
2515 return -EINVAL;
bf8f6e5b
AM
2516 }
2517
2518 return count;
2519}
2520
828c0950 2521static const struct file_operations fops_kp = {
bf8f6e5b
AM
2522 .read = read_enabled_file_bool,
2523 .write = write_enabled_file_bool,
6038f373 2524 .llseek = default_llseek,
bf8f6e5b
AM
2525};
2526
55479f64 2527static int __init debugfs_kprobe_init(void)
346fd59b
SD
2528{
2529 struct dentry *dir, *file;
bf8f6e5b 2530 unsigned int value = 1;
346fd59b
SD
2531
2532 dir = debugfs_create_dir("kprobes", NULL);
2533 if (!dir)
2534 return -ENOMEM;
2535
e3869792 2536 file = debugfs_create_file("list", 0444, dir, NULL,
346fd59b 2537 &debugfs_kprobes_operations);
63724740
MH
2538 if (!file)
2539 goto error;
346fd59b 2540
bf8f6e5b
AM
2541 file = debugfs_create_file("enabled", 0600, dir,
2542 &value, &fops_kp);
63724740
MH
2543 if (!file)
2544 goto error;
2545
2546 file = debugfs_create_file("blacklist", 0444, dir, NULL,
2547 &debugfs_kprobe_blacklist_ops);
2548 if (!file)
2549 goto error;
bf8f6e5b 2550
346fd59b 2551 return 0;
63724740
MH
2552
2553error:
2554 debugfs_remove(dir);
2555 return -ENOMEM;
346fd59b
SD
2556}
2557
2558late_initcall(debugfs_kprobe_init);
2559#endif /* CONFIG_DEBUG_FS */
2560
2561module_init(init_kprobes);
1da177e4 2562
99081ab5 2563/* defined in arch/.../kernel/kprobes.c */
1da177e4 2564EXPORT_SYMBOL_GPL(jprobe_return);