]>
Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
fd7b231f AK |
2 | /* |
3 | * Kernel Probes (KProbes) | |
4 | * arch/ia64/kernel/kprobes.c | |
5 | * | |
fd7b231f AK |
6 | * Copyright (C) IBM Corporation, 2002, 2004 |
7 | * Copyright (C) Intel Corporation, 2005 | |
8 | * | |
9 | * 2005-Apr Rusty Lynch <rusty.lynch@intel.com> and Anil S Keshavamurthy | |
10 | * <anil.s.keshavamurthy@intel.com> adapted from i386 | |
11 | */ | |
12 | ||
fd7b231f AK |
13 | #include <linux/kprobes.h> |
14 | #include <linux/ptrace.h> | |
fd7b231f AK |
15 | #include <linux/string.h> |
16 | #include <linux/slab.h> | |
17 | #include <linux/preempt.h> | |
e7088170 | 18 | #include <linux/extable.h> |
1eeb66a1 | 19 | #include <linux/kdebug.h> |
ca5999fd | 20 | #include <linux/pgtable.h> |
65fddcfc | 21 | |
c7b645f9 | 22 | #include <asm/sections.h> |
82ed1ac9 | 23 | #include <asm/exception.h> |
fd7b231f | 24 | |
8a5c4dc5 AM |
25 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
26 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); | |
fd7b231f | 27 | |
f438d914 MH |
28 | struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; |
29 | ||
fd7b231f AK |
30 | enum instruction_type {A, I, M, F, B, L, X, u}; |
31 | static enum instruction_type bundle_encoding[32][3] = { | |
32 | { M, I, I }, /* 00 */ | |
33 | { M, I, I }, /* 01 */ | |
34 | { M, I, I }, /* 02 */ | |
35 | { M, I, I }, /* 03 */ | |
36 | { M, L, X }, /* 04 */ | |
37 | { M, L, X }, /* 05 */ | |
38 | { u, u, u }, /* 06 */ | |
39 | { u, u, u }, /* 07 */ | |
40 | { M, M, I }, /* 08 */ | |
41 | { M, M, I }, /* 09 */ | |
42 | { M, M, I }, /* 0A */ | |
43 | { M, M, I }, /* 0B */ | |
44 | { M, F, I }, /* 0C */ | |
45 | { M, F, I }, /* 0D */ | |
46 | { M, M, F }, /* 0E */ | |
47 | { M, M, F }, /* 0F */ | |
48 | { M, I, B }, /* 10 */ | |
49 | { M, I, B }, /* 11 */ | |
50 | { M, B, B }, /* 12 */ | |
51 | { M, B, B }, /* 13 */ | |
52 | { u, u, u }, /* 14 */ | |
53 | { u, u, u }, /* 15 */ | |
54 | { B, B, B }, /* 16 */ | |
55 | { B, B, B }, /* 17 */ | |
56 | { M, M, B }, /* 18 */ | |
57 | { M, M, B }, /* 19 */ | |
58 | { u, u, u }, /* 1A */ | |
59 | { u, u, u }, /* 1B */ | |
60 | { M, F, B }, /* 1C */ | |
61 | { M, F, B }, /* 1D */ | |
62 | { u, u, u }, /* 1E */ | |
63 | { u, u, u }, /* 1F */ | |
64 | }; | |
65 | ||
34e1ceb1 MH |
66 | /* Insert a long branch code */ |
67 | static void __kprobes set_brl_inst(void *from, void *to) | |
68 | { | |
69 | s64 rel = ((s64) to - (s64) from) >> 4; | |
70 | bundle_t *brl; | |
71 | brl = (bundle_t *) ((u64) from & ~0xf); | |
72 | brl->quad0.template = 0x05; /* [MLX](stop) */ | |
73 | brl->quad0.slot0 = NOP_M_INST; /* nop.m 0x0 */ | |
74 | brl->quad0.slot1_p0 = ((rel >> 20) & 0x7fffffffff) << 2; | |
75 | brl->quad1.slot1_p1 = (((rel >> 20) & 0x7fffffffff) << 2) >> (64 - 46); | |
76 | /* brl.cond.sptk.many.clr rel<<4 (qp=0) */ | |
77 | brl->quad1.slot2 = BRL_INST(rel >> 59, rel & 0xfffff); | |
78 | } | |
79 | ||
a5403183 AK |
80 | /* |
81 | * In this function we check to see if the instruction | |
82 | * is IP relative instruction and update the kprobe | |
83 | * inst flag accordingly | |
84 | */ | |
1f7ad57b PP |
85 | static void __kprobes update_kprobe_inst_flag(uint template, uint slot, |
86 | uint major_opcode, | |
87 | unsigned long kprobe_inst, | |
88 | struct kprobe *p) | |
fd7b231f | 89 | { |
8bc76772 RL |
90 | p->ainsn.inst_flag = 0; |
91 | p->ainsn.target_br_reg = 0; | |
08ed38b6 | 92 | p->ainsn.slot = slot; |
fd7b231f | 93 | |
deac66ae | 94 | /* Check for Break instruction |
62c27be0 | 95 | * Bits 37:40 Major opcode to be zero |
deac66ae KA |
96 | * Bits 27:32 X6 to be zero |
97 | * Bits 32:35 X3 to be zero | |
98 | */ | |
99 | if ((!major_opcode) && (!((kprobe_inst >> 27) & 0x1FF)) ) { | |
100 | /* is a break instruction */ | |
101 | p->ainsn.inst_flag |= INST_FLAG_BREAK_INST; | |
102 | return; | |
103 | } | |
104 | ||
a5403183 AK |
105 | if (bundle_encoding[template][slot] == B) { |
106 | switch (major_opcode) { | |
107 | case INDIRECT_CALL_OPCODE: | |
108 | p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; | |
62c27be0 | 109 | p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); |
110 | break; | |
a5403183 AK |
111 | case IP_RELATIVE_PREDICT_OPCODE: |
112 | case IP_RELATIVE_BRANCH_OPCODE: | |
113 | p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR; | |
62c27be0 | 114 | break; |
a5403183 | 115 | case IP_RELATIVE_CALL_OPCODE: |
62c27be0 | 116 | p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR; |
117 | p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; | |
118 | p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); | |
119 | break; | |
a5403183 | 120 | } |
62c27be0 | 121 | } else if (bundle_encoding[template][slot] == X) { |
a5403183 AK |
122 | switch (major_opcode) { |
123 | case LONG_CALL_OPCODE: | |
124 | p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG; | |
125 | p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7); | |
126 | break; | |
127 | } | |
128 | } | |
129 | return; | |
130 | } | |
fd7b231f | 131 | |
1674eafc AK |
132 | /* |
133 | * In this function we check to see if the instruction | |
134 | * (qp) cmpx.crel.ctype p1,p2=r2,r3 | |
135 | * on which we are inserting kprobe is cmp instruction | |
136 | * with ctype as unc. | |
137 | */ | |
1f7ad57b PP |
138 | static uint __kprobes is_cmp_ctype_unc_inst(uint template, uint slot, |
139 | uint major_opcode, | |
140 | unsigned long kprobe_inst) | |
1674eafc AK |
141 | { |
142 | cmp_inst_t cmp_inst; | |
143 | uint ctype_unc = 0; | |
144 | ||
145 | if (!((bundle_encoding[template][slot] == I) || | |
146 | (bundle_encoding[template][slot] == M))) | |
147 | goto out; | |
148 | ||
149 | if (!((major_opcode == 0xC) || (major_opcode == 0xD) || | |
150 | (major_opcode == 0xE))) | |
151 | goto out; | |
152 | ||
153 | cmp_inst.l = kprobe_inst; | |
154 | if ((cmp_inst.f.x2 == 0) || (cmp_inst.f.x2 == 1)) { | |
72fdbdce | 155 | /* Integer compare - Register Register (A6 type)*/ |
1674eafc AK |
156 | if ((cmp_inst.f.tb == 0) && (cmp_inst.f.ta == 0) |
157 | &&(cmp_inst.f.c == 1)) | |
158 | ctype_unc = 1; | |
159 | } else if ((cmp_inst.f.x2 == 2)||(cmp_inst.f.x2 == 3)) { | |
72fdbdce | 160 | /* Integer compare - Immediate Register (A8 type)*/ |
1674eafc AK |
161 | if ((cmp_inst.f.ta == 0) &&(cmp_inst.f.c == 1)) |
162 | ctype_unc = 1; | |
163 | } | |
164 | out: | |
165 | return ctype_unc; | |
166 | } | |
167 | ||
df3e0d1c | 168 | /* |
169 | * In this function we check to see if the instruction | |
170 | * on which we are inserting kprobe is supported. | |
171 | * Returns qp value if supported | |
172 | * Returns -EINVAL if unsupported | |
173 | */ | |
174 | static int __kprobes unsupported_inst(uint template, uint slot, | |
175 | uint major_opcode, | |
176 | unsigned long kprobe_inst, | |
177 | unsigned long addr) | |
178 | { | |
179 | int qp; | |
180 | ||
181 | qp = kprobe_inst & 0x3f; | |
182 | if (is_cmp_ctype_unc_inst(template, slot, major_opcode, kprobe_inst)) { | |
183 | if (slot == 1 && qp) { | |
c2eeb321 JP |
184 | printk(KERN_WARNING "Kprobes on cmp unc " |
185 | "instruction on slot 1 at <0x%lx> " | |
df3e0d1c | 186 | "is not supported\n", addr); |
187 | return -EINVAL; | |
188 | ||
189 | } | |
190 | qp = 0; | |
191 | } | |
192 | else if (bundle_encoding[template][slot] == I) { | |
193 | if (major_opcode == 0) { | |
194 | /* | |
195 | * Check for Integer speculation instruction | |
196 | * - Bit 33-35 to be equal to 0x1 | |
197 | */ | |
198 | if (((kprobe_inst >> 33) & 0x7) == 1) { | |
199 | printk(KERN_WARNING | |
200 | "Kprobes on speculation inst at <0x%lx> not supported\n", | |
201 | addr); | |
202 | return -EINVAL; | |
203 | } | |
204 | /* | |
205 | * IP relative mov instruction | |
206 | * - Bit 27-35 to be equal to 0x30 | |
207 | */ | |
208 | if (((kprobe_inst >> 27) & 0x1FF) == 0x30) { | |
209 | printk(KERN_WARNING | |
210 | "Kprobes on \"mov r1=ip\" at <0x%lx> not supported\n", | |
211 | addr); | |
212 | return -EINVAL; | |
213 | ||
214 | } | |
215 | } | |
216 | else if ((major_opcode == 5) && !(kprobe_inst & (0xFUl << 33)) && | |
217 | (kprobe_inst & (0x1UL << 12))) { | |
218 | /* test bit instructions, tbit,tnat,tf | |
219 | * bit 33-36 to be equal to 0 | |
220 | * bit 12 to be equal to 1 | |
221 | */ | |
222 | if (slot == 1 && qp) { | |
c2eeb321 JP |
223 | printk(KERN_WARNING "Kprobes on test bit " |
224 | "instruction on slot at <0x%lx> " | |
df3e0d1c | 225 | "is not supported\n", addr); |
226 | return -EINVAL; | |
227 | } | |
228 | qp = 0; | |
229 | } | |
230 | } | |
231 | else if (bundle_encoding[template][slot] == B) { | |
232 | if (major_opcode == 7) { | |
233 | /* IP-Relative Predict major code is 7 */ | |
234 | printk(KERN_WARNING "Kprobes on IP-Relative" | |
235 | "Predict is not supported\n"); | |
236 | return -EINVAL; | |
237 | } | |
238 | else if (major_opcode == 2) { | |
239 | /* Indirect Predict, major code is 2 | |
240 | * bit 27-32 to be equal to 10 or 11 | |
241 | */ | |
242 | int x6=(kprobe_inst >> 27) & 0x3F; | |
243 | if ((x6 == 0x10) || (x6 == 0x11)) { | |
c2eeb321 | 244 | printk(KERN_WARNING "Kprobes on " |
df3e0d1c | 245 | "Indirect Predict is not supported\n"); |
246 | return -EINVAL; | |
247 | } | |
248 | } | |
249 | } | |
250 | /* kernel does not use float instruction, here for safety kprobe | |
251 | * will judge whether it is fcmp/flass/float approximation instruction | |
252 | */ | |
253 | else if (unlikely(bundle_encoding[template][slot] == F)) { | |
254 | if ((major_opcode == 4 || major_opcode == 5) && | |
255 | (kprobe_inst & (0x1 << 12))) { | |
256 | /* fcmp/fclass unc instruction */ | |
257 | if (slot == 1 && qp) { | |
258 | printk(KERN_WARNING "Kprobes on fcmp/fclass " | |
259 | "instruction on slot at <0x%lx> " | |
260 | "is not supported\n", addr); | |
261 | return -EINVAL; | |
262 | ||
263 | } | |
264 | qp = 0; | |
265 | } | |
266 | if ((major_opcode == 0 || major_opcode == 1) && | |
267 | (kprobe_inst & (0x1UL << 33))) { | |
268 | /* float Approximation instruction */ | |
269 | if (slot == 1 && qp) { | |
270 | printk(KERN_WARNING "Kprobes on float Approx " | |
271 | "instr at <0x%lx> is not supported\n", | |
272 | addr); | |
273 | return -EINVAL; | |
274 | } | |
275 | qp = 0; | |
276 | } | |
277 | } | |
278 | return qp; | |
279 | } | |
280 | ||
a5403183 AK |
281 | /* |
282 | * In this function we override the bundle with | |
283 | * the break instruction at the given slot. | |
284 | */ | |
1f7ad57b PP |
285 | static void __kprobes prepare_break_inst(uint template, uint slot, |
286 | uint major_opcode, | |
287 | unsigned long kprobe_inst, | |
df3e0d1c | 288 | struct kprobe *p, |
289 | int qp) | |
a5403183 AK |
290 | { |
291 | unsigned long break_inst = BREAK_INST; | |
214ddde2 | 292 | bundle_t *bundle = &p->opcode.bundle; |
a5403183 AK |
293 | |
294 | /* | |
295 | * Copy the original kprobe_inst qualifying predicate(qp) | |
df3e0d1c | 296 | * to the break instruction |
a5403183 | 297 | */ |
df3e0d1c | 298 | break_inst |= qp; |
a5403183 AK |
299 | |
300 | switch (slot) { | |
301 | case 0: | |
302 | bundle->quad0.slot0 = break_inst; | |
303 | break; | |
304 | case 1: | |
305 | bundle->quad0.slot1_p0 = break_inst; | |
306 | bundle->quad1.slot1_p1 = break_inst >> (64-46); | |
307 | break; | |
308 | case 2: | |
309 | bundle->quad1.slot2 = break_inst; | |
310 | break; | |
8bc76772 | 311 | } |
cd2675bf | 312 | |
a5403183 AK |
313 | /* |
314 | * Update the instruction flag, so that we can | |
315 | * emulate the instruction properly after we | |
316 | * single step on original instruction | |
317 | */ | |
318 | update_kprobe_inst_flag(template, slot, major_opcode, kprobe_inst, p); | |
319 | } | |
320 | ||
3ca269d8 | 321 | static void __kprobes get_kprobe_inst(bundle_t *bundle, uint slot, |
a5403183 AK |
322 | unsigned long *kprobe_inst, uint *major_opcode) |
323 | { | |
324 | unsigned long kprobe_inst_p0, kprobe_inst_p1; | |
325 | unsigned int template; | |
326 | ||
327 | template = bundle->quad0.template; | |
fd7b231f | 328 | |
fd7b231f | 329 | switch (slot) { |
a5403183 | 330 | case 0: |
62c27be0 | 331 | *major_opcode = (bundle->quad0.slot0 >> SLOT0_OPCODE_SHIFT); |
332 | *kprobe_inst = bundle->quad0.slot0; | |
333 | break; | |
a5403183 | 334 | case 1: |
62c27be0 | 335 | *major_opcode = (bundle->quad1.slot1_p1 >> SLOT1_p1_OPCODE_SHIFT); |
336 | kprobe_inst_p0 = bundle->quad0.slot1_p0; | |
337 | kprobe_inst_p1 = bundle->quad1.slot1_p1; | |
338 | *kprobe_inst = kprobe_inst_p0 | (kprobe_inst_p1 << (64-46)); | |
fd7b231f | 339 | break; |
a5403183 | 340 | case 2: |
62c27be0 | 341 | *major_opcode = (bundle->quad1.slot2 >> SLOT2_OPCODE_SHIFT); |
342 | *kprobe_inst = bundle->quad1.slot2; | |
fd7b231f AK |
343 | break; |
344 | } | |
a5403183 | 345 | } |
fd7b231f | 346 | |
c7b645f9 | 347 | /* Returns non-zero if the addr is in the Interrupt Vector Table */ |
3ca269d8 | 348 | static int __kprobes in_ivt_functions(unsigned long addr) |
c7b645f9 KA |
349 | { |
350 | return (addr >= (unsigned long)__start_ivt_text | |
351 | && addr < (unsigned long)__end_ivt_text); | |
352 | } | |
353 | ||
1f7ad57b PP |
354 | static int __kprobes valid_kprobe_addr(int template, int slot, |
355 | unsigned long addr) | |
a5403183 AK |
356 | { |
357 | if ((slot > 2) || ((bundle_encoding[template][1] == L) && slot > 1)) { | |
c7b645f9 KA |
358 | printk(KERN_WARNING "Attempting to insert unaligned kprobe " |
359 | "at 0x%lx\n", addr); | |
a5403183 | 360 | return -EINVAL; |
8bc76772 | 361 | } |
a528e21c | 362 | |
62c27be0 | 363 | if (in_ivt_functions(addr)) { |
364 | printk(KERN_WARNING "Kprobes can't be inserted inside " | |
c7b645f9 | 365 | "IVT functions at 0x%lx\n", addr); |
62c27be0 | 366 | return -EINVAL; |
367 | } | |
c7b645f9 | 368 | |
a5403183 AK |
369 | return 0; |
370 | } | |
371 | ||
3ca269d8 | 372 | static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) |
852caccc | 373 | { |
cdc7dbdf AK |
374 | unsigned int i; |
375 | i = atomic_add_return(1, &kcb->prev_kprobe_index); | |
376 | kcb->prev_kprobe[i-1].kp = kprobe_running(); | |
377 | kcb->prev_kprobe[i-1].status = kcb->kprobe_status; | |
852caccc AK |
378 | } |
379 | ||
3ca269d8 | 380 | static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
852caccc | 381 | { |
cdc7dbdf | 382 | unsigned int i; |
97075c4b | 383 | i = atomic_read(&kcb->prev_kprobe_index); |
6065a244 | 384 | __this_cpu_write(current_kprobe, kcb->prev_kprobe[i-1].kp); |
97075c4b MH |
385 | kcb->kprobe_status = kcb->prev_kprobe[i-1].status; |
386 | atomic_sub(1, &kcb->prev_kprobe_index); | |
852caccc AK |
387 | } |
388 | ||
3ca269d8 | 389 | static void __kprobes set_current_kprobe(struct kprobe *p, |
8a5c4dc5 | 390 | struct kprobe_ctlblk *kcb) |
852caccc | 391 | { |
6065a244 | 392 | __this_cpu_write(current_kprobe, p); |
852caccc AK |
393 | } |
394 | ||
9508dbfe RL |
395 | static void kretprobe_trampoline(void) |
396 | { | |
397 | } | |
398 | ||
1f7ad57b | 399 | int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
9508dbfe | 400 | { |
e792ff80 | 401 | regs->cr_iip = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL); |
d217d545 AM |
402 | /* |
403 | * By returning a non-zero value, we are telling | |
404 | * kprobe_handler() that we don't want the post_handler | |
405 | * to run (and have re-enabled preemption) | |
406 | */ | |
9138d581 | 407 | return 1; |
9508dbfe RL |
408 | } |
409 | ||
4c4308cb | 410 | void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, |
1f7ad57b | 411 | struct pt_regs *regs) |
9508dbfe | 412 | { |
4c4308cb | 413 | ri->ret_addr = (kprobe_opcode_t *)regs->b0; |
e792ff80 | 414 | ri->fp = NULL; |
9508dbfe | 415 | |
4c4308cb CH |
416 | /* Replace the return addr with trampoline addr */ |
417 | regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip; | |
9508dbfe RL |
418 | } |
419 | ||
34e1ceb1 MH |
420 | /* Check the instruction in the slot is break */ |
421 | static int __kprobes __is_ia64_break_inst(bundle_t *bundle, uint slot) | |
422 | { | |
423 | unsigned int major_opcode; | |
424 | unsigned int template = bundle->quad0.template; | |
425 | unsigned long kprobe_inst; | |
426 | ||
427 | /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */ | |
428 | if (slot == 1 && bundle_encoding[template][1] == L) | |
429 | slot++; | |
430 | ||
431 | /* Get Kprobe probe instruction at given slot*/ | |
432 | get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode); | |
433 | ||
434 | /* For break instruction, | |
435 | * Bits 37:40 Major opcode to be zero | |
436 | * Bits 27:32 X6 to be zero | |
437 | * Bits 32:35 X3 to be zero | |
438 | */ | |
439 | if (major_opcode || ((kprobe_inst >> 27) & 0x1FF)) { | |
440 | /* Not a break instruction */ | |
441 | return 0; | |
442 | } | |
443 | ||
444 | /* Is a break instruction */ | |
445 | return 1; | |
446 | } | |
447 | ||
448 | /* | |
449 | * In this function, we check whether the target bundle modifies IP or | |
450 | * it triggers an exception. If so, it cannot be boostable. | |
451 | */ | |
452 | static int __kprobes can_boost(bundle_t *bundle, uint slot, | |
453 | unsigned long bundle_addr) | |
454 | { | |
455 | unsigned int template = bundle->quad0.template; | |
456 | ||
457 | do { | |
458 | if (search_exception_tables(bundle_addr + slot) || | |
459 | __is_ia64_break_inst(bundle, slot)) | |
460 | return 0; /* exception may occur in this bundle*/ | |
461 | } while ((++slot) < 3); | |
462 | template &= 0x1e; | |
463 | if (template >= 0x10 /* including B unit */ || | |
464 | template == 0x04 /* including X unit */ || | |
465 | template == 0x06) /* undefined */ | |
466 | return 0; | |
467 | ||
468 | return 1; | |
469 | } | |
470 | ||
471 | /* Prepare long jump bundle and disables other boosters if need */ | |
472 | static void __kprobes prepare_booster(struct kprobe *p) | |
473 | { | |
474 | unsigned long addr = (unsigned long)p->addr & ~0xFULL; | |
475 | unsigned int slot = (unsigned long)p->addr & 0xf; | |
476 | struct kprobe *other_kp; | |
477 | ||
478 | if (can_boost(&p->ainsn.insn[0].bundle, slot, addr)) { | |
479 | set_brl_inst(&p->ainsn.insn[1].bundle, (bundle_t *)addr + 1); | |
480 | p->ainsn.inst_flag |= INST_FLAG_BOOSTABLE; | |
481 | } | |
482 | ||
483 | /* disables boosters in previous slots */ | |
484 | for (; addr < (unsigned long)p->addr; addr++) { | |
485 | other_kp = get_kprobe((void *)addr); | |
486 | if (other_kp) | |
487 | other_kp->ainsn.inst_flag &= ~INST_FLAG_BOOSTABLE; | |
488 | } | |
489 | } | |
490 | ||
1f7ad57b | 491 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
a5403183 AK |
492 | { |
493 | unsigned long addr = (unsigned long) p->addr; | |
494 | unsigned long *kprobe_addr = (unsigned long *)(addr & ~0xFULL); | |
495 | unsigned long kprobe_inst=0; | |
496 | unsigned int slot = addr & 0xf, template, major_opcode = 0; | |
214ddde2 | 497 | bundle_t *bundle; |
df3e0d1c | 498 | int qp; |
a5403183 | 499 | |
214ddde2 | 500 | bundle = &((kprobe_opcode_t *)kprobe_addr)->bundle; |
62c27be0 | 501 | template = bundle->quad0.template; |
a5403183 AK |
502 | |
503 | if(valid_kprobe_addr(template, slot, addr)) | |
504 | return -EINVAL; | |
505 | ||
506 | /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */ | |
62c27be0 | 507 | if (slot == 1 && bundle_encoding[template][1] == L) |
508 | slot++; | |
a5403183 AK |
509 | |
510 | /* Get kprobe_inst and major_opcode from the bundle */ | |
511 | get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode); | |
512 | ||
df3e0d1c | 513 | qp = unsupported_inst(template, slot, major_opcode, kprobe_inst, addr); |
514 | if (qp < 0) | |
515 | return -EINVAL; | |
8bc76772 | 516 | |
214ddde2 | 517 | p->ainsn.insn = get_insn_slot(); |
518 | if (!p->ainsn.insn) | |
519 | return -ENOMEM; | |
520 | memcpy(&p->opcode, kprobe_addr, sizeof(kprobe_opcode_t)); | |
521 | memcpy(p->ainsn.insn, kprobe_addr, sizeof(kprobe_opcode_t)); | |
8bc76772 | 522 | |
df3e0d1c | 523 | prepare_break_inst(template, slot, major_opcode, kprobe_inst, p, qp); |
a9ad965e | 524 | |
34e1ceb1 MH |
525 | prepare_booster(p); |
526 | ||
214ddde2 | 527 | return 0; |
a9ad965e | 528 | } |
529 | ||
1f7ad57b | 530 | void __kprobes arch_arm_kprobe(struct kprobe *p) |
8bc76772 | 531 | { |
08ed38b6 TL |
532 | unsigned long arm_addr; |
533 | bundle_t *src, *dest; | |
534 | ||
535 | arm_addr = ((unsigned long)p->addr) & ~0xFUL; | |
536 | dest = &((kprobe_opcode_t *)arm_addr)->bundle; | |
537 | src = &p->opcode.bundle; | |
8bc76772 | 538 | |
214ddde2 | 539 | flush_icache_range((unsigned long)p->ainsn.insn, |
34e1ceb1 MH |
540 | (unsigned long)p->ainsn.insn + |
541 | sizeof(kprobe_opcode_t) * MAX_INSN_SIZE); | |
542 | ||
08ed38b6 TL |
543 | switch (p->ainsn.slot) { |
544 | case 0: | |
545 | dest->quad0.slot0 = src->quad0.slot0; | |
546 | break; | |
547 | case 1: | |
548 | dest->quad1.slot1_p1 = src->quad1.slot1_p1; | |
549 | break; | |
550 | case 2: | |
551 | dest->quad1.slot2 = src->quad1.slot2; | |
552 | break; | |
553 | } | |
214ddde2 | 554 | flush_icache_range(arm_addr, arm_addr + sizeof(kprobe_opcode_t)); |
fd7b231f AK |
555 | } |
556 | ||
1f7ad57b | 557 | void __kprobes arch_disarm_kprobe(struct kprobe *p) |
fd7b231f | 558 | { |
08ed38b6 TL |
559 | unsigned long arm_addr; |
560 | bundle_t *src, *dest; | |
fd7b231f | 561 | |
08ed38b6 TL |
562 | arm_addr = ((unsigned long)p->addr) & ~0xFUL; |
563 | dest = &((kprobe_opcode_t *)arm_addr)->bundle; | |
214ddde2 | 564 | /* p->ainsn.insn contains the original unaltered kprobe_opcode_t */ |
08ed38b6 TL |
565 | src = &p->ainsn.insn->bundle; |
566 | switch (p->ainsn.slot) { | |
567 | case 0: | |
568 | dest->quad0.slot0 = src->quad0.slot0; | |
569 | break; | |
570 | case 1: | |
571 | dest->quad1.slot1_p1 = src->quad1.slot1_p1; | |
572 | break; | |
573 | case 2: | |
574 | dest->quad1.slot2 = src->quad1.slot2; | |
575 | break; | |
576 | } | |
214ddde2 | 577 | flush_icache_range(arm_addr, arm_addr + sizeof(kprobe_opcode_t)); |
fd7b231f AK |
578 | } |
579 | ||
214ddde2 | 580 | void __kprobes arch_remove_kprobe(struct kprobe *p) |
581 | { | |
12941560 MH |
582 | if (p->ainsn.insn) { |
583 | free_insn_slot(p->ainsn.insn, | |
584 | p->ainsn.inst_flag & INST_FLAG_BOOSTABLE); | |
585 | p->ainsn.insn = NULL; | |
586 | } | |
214ddde2 | 587 | } |
fd7b231f AK |
588 | /* |
589 | * We are resuming execution after a single step fault, so the pt_regs | |
590 | * structure reflects the register state after we executed the instruction | |
34e1ceb1 | 591 | * located in the kprobe (p->ainsn.insn->bundle). We still need to adjust |
cd2675bf AK |
592 | * the ip to point back to the original stack address. To set the IP address |
593 | * to original stack address, handle the case where we need to fixup the | |
594 | * relative IP address and/or fixup branch register. | |
fd7b231f | 595 | */ |
1f7ad57b | 596 | static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs) |
fd7b231f | 597 | { |
62c27be0 | 598 | unsigned long bundle_addr = (unsigned long) (&p->ainsn.insn->bundle); |
599 | unsigned long resume_addr = (unsigned long)p->addr & ~0xFULL; | |
600 | unsigned long template; | |
601 | int slot = ((unsigned long)p->addr & 0xf); | |
fd7b231f | 602 | |
214ddde2 | 603 | template = p->ainsn.insn->bundle.quad0.template; |
cd2675bf | 604 | |
62c27be0 | 605 | if (slot == 1 && bundle_encoding[template][1] == L) |
606 | slot = 2; | |
cd2675bf | 607 | |
34e1ceb1 | 608 | if (p->ainsn.inst_flag & ~INST_FLAG_BOOSTABLE) { |
cd2675bf AK |
609 | |
610 | if (p->ainsn.inst_flag & INST_FLAG_FIX_RELATIVE_IP_ADDR) { | |
611 | /* Fix relative IP address */ | |
62c27be0 | 612 | regs->cr_iip = (regs->cr_iip - bundle_addr) + |
613 | resume_addr; | |
cd2675bf AK |
614 | } |
615 | ||
616 | if (p->ainsn.inst_flag & INST_FLAG_FIX_BRANCH_REG) { | |
617 | /* | |
618 | * Fix target branch register, software convention is | |
619 | * to use either b0 or b6 or b7, so just checking | |
620 | * only those registers | |
621 | */ | |
622 | switch (p->ainsn.target_br_reg) { | |
623 | case 0: | |
624 | if ((regs->b0 == bundle_addr) || | |
625 | (regs->b0 == bundle_addr + 0x10)) { | |
626 | regs->b0 = (regs->b0 - bundle_addr) + | |
627 | resume_addr; | |
628 | } | |
629 | break; | |
630 | case 6: | |
631 | if ((regs->b6 == bundle_addr) || | |
632 | (regs->b6 == bundle_addr + 0x10)) { | |
633 | regs->b6 = (regs->b6 - bundle_addr) + | |
634 | resume_addr; | |
635 | } | |
636 | break; | |
637 | case 7: | |
638 | if ((regs->b7 == bundle_addr) || | |
639 | (regs->b7 == bundle_addr + 0x10)) { | |
640 | regs->b7 = (regs->b7 - bundle_addr) + | |
641 | resume_addr; | |
642 | } | |
643 | break; | |
644 | } /* end switch */ | |
645 | } | |
646 | goto turn_ss_off; | |
647 | } | |
fd7b231f | 648 | |
cd2675bf | 649 | if (slot == 2) { |
62c27be0 | 650 | if (regs->cr_iip == bundle_addr + 0x10) { |
651 | regs->cr_iip = resume_addr + 0x10; | |
652 | } | |
653 | } else { | |
654 | if (regs->cr_iip == bundle_addr) { | |
655 | regs->cr_iip = resume_addr; | |
656 | } | |
a5403183 | 657 | } |
fd7b231f | 658 | |
cd2675bf | 659 | turn_ss_off: |
62c27be0 | 660 | /* Turn off Single Step bit */ |
661 | ia64_psr(regs)->ss = 0; | |
fd7b231f AK |
662 | } |
663 | ||
1f7ad57b | 664 | static void __kprobes prepare_ss(struct kprobe *p, struct pt_regs *regs) |
fd7b231f | 665 | { |
214ddde2 | 666 | unsigned long bundle_addr = (unsigned long) &p->ainsn.insn->bundle; |
fd7b231f AK |
667 | unsigned long slot = (unsigned long)p->addr & 0xf; |
668 | ||
deac66ae KA |
669 | /* single step inline if break instruction */ |
670 | if (p->ainsn.inst_flag == INST_FLAG_BREAK_INST) | |
671 | regs->cr_iip = (unsigned long)p->addr & ~0xFULL; | |
672 | else | |
673 | regs->cr_iip = bundle_addr & ~0xFULL; | |
fd7b231f AK |
674 | |
675 | if (slot > 2) | |
676 | slot = 0; | |
677 | ||
678 | ia64_psr(regs)->ri = slot; | |
679 | ||
680 | /* turn on single stepping */ | |
681 | ia64_psr(regs)->ss = 1; | |
682 | } | |
683 | ||
661e5a3d KA |
684 | static int __kprobes is_ia64_break_inst(struct pt_regs *regs) |
685 | { | |
686 | unsigned int slot = ia64_psr(regs)->ri; | |
661e5a3d KA |
687 | unsigned long *kprobe_addr = (unsigned long *)regs->cr_iip; |
688 | bundle_t bundle; | |
689 | ||
690 | memcpy(&bundle, kprobe_addr, sizeof(bundle_t)); | |
661e5a3d | 691 | |
34e1ceb1 | 692 | return __is_ia64_break_inst(&bundle, slot); |
661e5a3d KA |
693 | } |
694 | ||
1f7ad57b | 695 | static int __kprobes pre_kprobes_handler(struct die_args *args) |
fd7b231f AK |
696 | { |
697 | struct kprobe *p; | |
698 | int ret = 0; | |
89cb14c0 | 699 | struct pt_regs *regs = args->regs; |
fd7b231f | 700 | kprobe_opcode_t *addr = (kprobe_opcode_t *)instruction_pointer(regs); |
d217d545 AM |
701 | struct kprobe_ctlblk *kcb; |
702 | ||
703 | /* | |
704 | * We don't want to be preempted for the entire | |
705 | * duration of kprobe processing | |
706 | */ | |
707 | preempt_disable(); | |
708 | kcb = get_kprobe_ctlblk(); | |
fd7b231f | 709 | |
fd7b231f AK |
710 | /* Handle recursion cases */ |
711 | if (kprobe_running()) { | |
712 | p = get_kprobe(addr); | |
713 | if (p) { | |
8a5c4dc5 | 714 | if ((kcb->kprobe_status == KPROBE_HIT_SS) && |
deac66ae | 715 | (p->ainsn.inst_flag == INST_FLAG_BREAK_INST)) { |
62c27be0 | 716 | ia64_psr(regs)->ss = 0; |
fd7b231f AK |
717 | goto no_kprobe; |
718 | } | |
852caccc AK |
719 | /* We have reentered the pre_kprobe_handler(), since |
720 | * another probe was hit while within the handler. | |
721 | * We here save the original kprobes variables and | |
722 | * just single step on the instruction of the new probe | |
723 | * without calling any user handlers. | |
724 | */ | |
8a5c4dc5 AM |
725 | save_previous_kprobe(kcb); |
726 | set_current_kprobe(p, kcb); | |
bf8d5c52 | 727 | kprobes_inc_nmissed_count(p); |
852caccc | 728 | prepare_ss(p, regs); |
8a5c4dc5 | 729 | kcb->kprobe_status = KPROBE_REENTER; |
852caccc | 730 | return 1; |
eb3a7292 KA |
731 | } else if (!is_ia64_break_inst(regs)) { |
732 | /* The breakpoint instruction was removed by | |
733 | * another cpu right after we hit, no further | |
734 | * handling of this interrupt is appropriate | |
735 | */ | |
736 | ret = 1; | |
737 | goto no_kprobe; | |
89cb14c0 KA |
738 | } else { |
739 | /* Not our break */ | |
740 | goto no_kprobe; | |
fd7b231f AK |
741 | } |
742 | } | |
743 | ||
fd7b231f AK |
744 | p = get_kprobe(addr); |
745 | if (!p) { | |
661e5a3d KA |
746 | if (!is_ia64_break_inst(regs)) { |
747 | /* | |
748 | * The breakpoint instruction was removed right | |
749 | * after we hit it. Another cpu has removed | |
750 | * either a probepoint or a debugger breakpoint | |
751 | * at this address. In either case, no further | |
752 | * handling of this interrupt is appropriate. | |
753 | */ | |
754 | ret = 1; | |
755 | ||
756 | } | |
757 | ||
758 | /* Not one of our break, let kernel handle it */ | |
fd7b231f AK |
759 | goto no_kprobe; |
760 | } | |
761 | ||
8a5c4dc5 AM |
762 | set_current_kprobe(p, kcb); |
763 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; | |
fd7b231f | 764 | |
cce188bd MH |
765 | if (p->pre_handler && p->pre_handler(p, regs)) { |
766 | reset_current_kprobe(); | |
767 | preempt_enable_no_resched(); | |
fd7b231f | 768 | return 1; |
cce188bd | 769 | } |
fd7b231f | 770 | |
b9b75e53 | 771 | #if !defined(CONFIG_PREEMPTION) |
34e1ceb1 MH |
772 | if (p->ainsn.inst_flag == INST_FLAG_BOOSTABLE && !p->post_handler) { |
773 | /* Boost up -- we can execute copied instructions directly */ | |
774 | ia64_psr(regs)->ri = p->ainsn.slot; | |
775 | regs->cr_iip = (unsigned long)&p->ainsn.insn->bundle & ~0xFULL; | |
776 | /* turn single stepping off */ | |
777 | ia64_psr(regs)->ss = 0; | |
778 | ||
779 | reset_current_kprobe(); | |
780 | preempt_enable_no_resched(); | |
781 | return 1; | |
782 | } | |
783 | #endif | |
fd7b231f | 784 | prepare_ss(p, regs); |
8a5c4dc5 | 785 | kcb->kprobe_status = KPROBE_HIT_SS; |
fd7b231f AK |
786 | return 1; |
787 | ||
788 | no_kprobe: | |
d217d545 | 789 | preempt_enable_no_resched(); |
fd7b231f AK |
790 | return ret; |
791 | } | |
792 | ||
1f7ad57b | 793 | static int __kprobes post_kprobes_handler(struct pt_regs *regs) |
fd7b231f | 794 | { |
8a5c4dc5 AM |
795 | struct kprobe *cur = kprobe_running(); |
796 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
797 | ||
798 | if (!cur) | |
fd7b231f AK |
799 | return 0; |
800 | ||
8a5c4dc5 AM |
801 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
802 | kcb->kprobe_status = KPROBE_HIT_SSDONE; | |
803 | cur->post_handler(cur, regs, 0); | |
852caccc | 804 | } |
fd7b231f | 805 | |
8a5c4dc5 | 806 | resume_execution(cur, regs); |
fd7b231f | 807 | |
852caccc | 808 | /*Restore back the original saved kprobes variables and continue. */ |
8a5c4dc5 AM |
809 | if (kcb->kprobe_status == KPROBE_REENTER) { |
810 | restore_previous_kprobe(kcb); | |
852caccc AK |
811 | goto out; |
812 | } | |
8a5c4dc5 | 813 | reset_current_kprobe(); |
852caccc AK |
814 | |
815 | out: | |
fd7b231f AK |
816 | preempt_enable_no_resched(); |
817 | return 1; | |
818 | } | |
819 | ||
45e18c22 | 820 | int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
fd7b231f | 821 | { |
8a5c4dc5 AM |
822 | struct kprobe *cur = kprobe_running(); |
823 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); | |
824 | ||
fd7b231f | 825 | |
c04c1c81 PP |
826 | switch(kcb->kprobe_status) { |
827 | case KPROBE_HIT_SS: | |
828 | case KPROBE_REENTER: | |
829 | /* | |
830 | * We are here because the instruction being single | |
831 | * stepped caused a page fault. We reset the current | |
832 | * kprobe and the instruction pointer points back to | |
833 | * the probe address and allow the page fault handler | |
834 | * to continue as a normal page fault. | |
835 | */ | |
836 | regs->cr_iip = ((unsigned long)cur->addr) & ~0xFULL; | |
837 | ia64_psr(regs)->ri = ((unsigned long)cur->addr) & 0xf; | |
838 | if (kcb->kprobe_status == KPROBE_REENTER) | |
839 | restore_previous_kprobe(kcb); | |
840 | else | |
841 | reset_current_kprobe(); | |
fd7b231f | 842 | preempt_enable_no_resched(); |
c04c1c81 PP |
843 | break; |
844 | case KPROBE_HIT_ACTIVE: | |
845 | case KPROBE_HIT_SSDONE: | |
846 | /* | |
847 | * We increment the nmissed count for accounting, | |
23d6d3db | 848 | * we can also use npre/npostfault count for accounting |
c04c1c81 PP |
849 | * these specific fault cases. |
850 | */ | |
851 | kprobes_inc_nmissed_count(cur); | |
852 | ||
853 | /* | |
854 | * We come here because instructions in the pre/post | |
855 | * handler caused the page_fault, this could happen | |
856 | * if handler tries to access user space by | |
857 | * copy_from_user(), get_user() etc. Let the | |
858 | * user-specified handler try to fix it first. | |
859 | */ | |
860 | if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) | |
861 | return 1; | |
fd32cb3a KA |
862 | /* |
863 | * In case the user-specified fault handler returned | |
864 | * zero, try to fix up. | |
865 | */ | |
866 | if (ia64_done_with_exception(regs)) | |
867 | return 1; | |
c04c1c81 PP |
868 | |
869 | /* | |
870 | * Let ia64_do_page_fault() fix it. | |
871 | */ | |
872 | break; | |
873 | default: | |
874 | break; | |
fd7b231f AK |
875 | } |
876 | ||
877 | return 0; | |
878 | } | |
879 | ||
1f7ad57b PP |
880 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, |
881 | unsigned long val, void *data) | |
fd7b231f AK |
882 | { |
883 | struct die_args *args = (struct die_args *)data; | |
66ff2d06 AM |
884 | int ret = NOTIFY_DONE; |
885 | ||
2326c770 | 886 | if (args->regs && user_mode(args->regs)) |
887 | return ret; | |
888 | ||
fd7b231f AK |
889 | switch(val) { |
890 | case DIE_BREAK: | |
9138d581 | 891 | /* err is break number from ia64_bad_break() */ |
08ed38b6 | 892 | if ((args->err >> 12) == (__IA64_BREAK_KPROBE >> 12) |
08ed38b6 | 893 | || args->err == 0) |
9138d581 KO |
894 | if (pre_kprobes_handler(args)) |
895 | ret = NOTIFY_STOP; | |
fd7b231f | 896 | break; |
9138d581 KO |
897 | case DIE_FAULT: |
898 | /* err is vector number from ia64_fault() */ | |
899 | if (args->err == 36) | |
900 | if (post_kprobes_handler(args->regs)) | |
901 | ret = NOTIFY_STOP; | |
fd7b231f | 902 | break; |
fd7b231f AK |
903 | default: |
904 | break; | |
905 | } | |
66ff2d06 | 906 | return ret; |
fd7b231f AK |
907 | } |
908 | ||
3d7e3382 ME |
909 | unsigned long arch_deref_entry_point(void *entry) |
910 | { | |
911 | return ((struct fnptr *)entry)->ip; | |
912 | } | |
913 | ||
9508dbfe RL |
914 | static struct kprobe trampoline_p = { |
915 | .pre_handler = trampoline_probe_handler | |
916 | }; | |
917 | ||
6772926b | 918 | int __init arch_init_kprobes(void) |
9508dbfe RL |
919 | { |
920 | trampoline_p.addr = | |
921 | (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip; | |
922 | return register_kprobe(&trampoline_p); | |
923 | } | |
bf8f6e5b AM |
924 | |
925 | int __kprobes arch_trampoline_kprobe(struct kprobe *p) | |
926 | { | |
927 | if (p->addr == | |
928 | (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip) | |
929 | return 1; | |
930 | ||
931 | return 0; | |
932 | } |