]>
Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
ee819aed | 2 | #include <linux/objtool.h> |
ee9f8fce JP |
3 | #include <linux/module.h> |
4 | #include <linux/sort.h> | |
5 | #include <asm/ptrace.h> | |
6 | #include <asm/stacktrace.h> | |
7 | #include <asm/unwind.h> | |
8 | #include <asm/orc_types.h> | |
9 | #include <asm/orc_lookup.h> | |
ee9f8fce JP |
10 | |
11 | #define orc_warn(fmt, ...) \ | |
b08418b5 JP |
12 | printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) |
13 | ||
14 | #define orc_warn_current(args...) \ | |
15 | ({ \ | |
b59cc976 | 16 | if (state->task == current && !state->error) \ |
b08418b5 JP |
17 | orc_warn(args); \ |
18 | }) | |
ee9f8fce JP |
19 | |
20 | extern int __start_orc_unwind_ip[]; | |
21 | extern int __stop_orc_unwind_ip[]; | |
22 | extern struct orc_entry __start_orc_unwind[]; | |
23 | extern struct orc_entry __stop_orc_unwind[]; | |
24 | ||
153eb222 JP |
25 | static bool orc_init __ro_after_init; |
26 | static unsigned int lookup_num_blocks __ro_after_init; | |
ee9f8fce | 27 | |
ee9f8fce JP |
28 | static inline unsigned long orc_ip(const int *ip) |
29 | { | |
30 | return (unsigned long)ip + *ip; | |
31 | } | |
32 | ||
33 | static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, | |
34 | unsigned int num_entries, unsigned long ip) | |
35 | { | |
36 | int *first = ip_table; | |
37 | int *last = ip_table + num_entries - 1; | |
38 | int *mid = first, *found = first; | |
39 | ||
40 | if (!num_entries) | |
41 | return NULL; | |
42 | ||
43 | /* | |
44 | * Do a binary range search to find the rightmost duplicate of a given | |
45 | * starting address. Some entries are section terminators which are | |
46 | * "weak" entries for ensuring there are no gaps. They should be | |
47 | * ignored when they conflict with a real entry. | |
48 | */ | |
49 | while (first <= last) { | |
50 | mid = first + ((last - first) / 2); | |
51 | ||
52 | if (orc_ip(mid) <= ip) { | |
53 | found = mid; | |
54 | first = mid + 1; | |
55 | } else | |
56 | last = mid - 1; | |
57 | } | |
58 | ||
59 | return u_table + (found - ip_table); | |
60 | } | |
61 | ||
62 | #ifdef CONFIG_MODULES | |
63 | static struct orc_entry *orc_module_find(unsigned long ip) | |
64 | { | |
65 | struct module *mod; | |
66 | ||
67 | mod = __module_address(ip); | |
68 | if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) | |
69 | return NULL; | |
70 | return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, | |
71 | mod->arch.num_orcs, ip); | |
72 | } | |
73 | #else | |
74 | static struct orc_entry *orc_module_find(unsigned long ip) | |
75 | { | |
76 | return NULL; | |
77 | } | |
78 | #endif | |
79 | ||
6be7fa3c SRV |
80 | #ifdef CONFIG_DYNAMIC_FTRACE |
81 | static struct orc_entry *orc_find(unsigned long ip); | |
82 | ||
83 | /* | |
84 | * Ftrace dynamic trampolines do not have orc entries of their own. | |
85 | * But they are copies of the ftrace entries that are static and | |
86 | * defined in ftrace_*.S, which do have orc entries. | |
87 | * | |
ae6a45a0 | 88 | * If the unwinder comes across a ftrace trampoline, then find the |
6be7fa3c | 89 | * ftrace function that was used to create it, and use that ftrace |
ae6a45a0 | 90 | * function's orc entry, as the placement of the return code in |
6be7fa3c SRV |
91 | * the stack will be identical. |
92 | */ | |
93 | static struct orc_entry *orc_ftrace_find(unsigned long ip) | |
94 | { | |
95 | struct ftrace_ops *ops; | |
96 | unsigned long caller; | |
97 | ||
98 | ops = ftrace_ops_trampoline(ip); | |
99 | if (!ops) | |
100 | return NULL; | |
101 | ||
102 | if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) | |
103 | caller = (unsigned long)ftrace_regs_call; | |
104 | else | |
105 | caller = (unsigned long)ftrace_call; | |
106 | ||
107 | /* Prevent unlikely recursion */ | |
108 | if (ip == caller) | |
109 | return NULL; | |
110 | ||
111 | return orc_find(caller); | |
112 | } | |
113 | #else | |
114 | static struct orc_entry *orc_ftrace_find(unsigned long ip) | |
115 | { | |
116 | return NULL; | |
117 | } | |
118 | #endif | |
119 | ||
ac5ceccc JH |
120 | /* |
121 | * If we crash with IP==0, the last successfully executed instruction | |
122 | * was probably an indirect function call with a NULL function pointer, | |
123 | * and we don't have unwind information for NULL. | |
124 | * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function | |
125 | * pointer into its parent and then continue normally from there. | |
126 | */ | |
127 | static struct orc_entry null_orc_entry = { | |
128 | .sp_offset = sizeof(long), | |
129 | .sp_reg = ORC_REG_SP, | |
130 | .bp_reg = ORC_REG_UNDEFINED, | |
ee819aed | 131 | .type = UNWIND_HINT_TYPE_CALL |
ac5ceccc JH |
132 | }; |
133 | ||
ae6a45a0 JP |
134 | /* Fake frame pointer entry -- used as a fallback for generated code */ |
135 | static struct orc_entry orc_fp_entry = { | |
ee819aed | 136 | .type = UNWIND_HINT_TYPE_CALL, |
ae6a45a0 JP |
137 | .sp_reg = ORC_REG_BP, |
138 | .sp_offset = 16, | |
139 | .bp_reg = ORC_REG_PREV_SP, | |
140 | .bp_offset = -16, | |
141 | .end = 0, | |
142 | }; | |
143 | ||
ee9f8fce JP |
144 | static struct orc_entry *orc_find(unsigned long ip) |
145 | { | |
6be7fa3c SRV |
146 | static struct orc_entry *orc; |
147 | ||
ac5ceccc JH |
148 | if (ip == 0) |
149 | return &null_orc_entry; | |
150 | ||
ee9f8fce JP |
151 | /* For non-init vmlinux addresses, use the fast lookup table: */ |
152 | if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { | |
153 | unsigned int idx, start, stop; | |
154 | ||
155 | idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; | |
156 | ||
157 | if (unlikely((idx >= lookup_num_blocks-1))) { | |
58c3862b JP |
158 | orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n", |
159 | idx, lookup_num_blocks, (void *)ip); | |
ee9f8fce JP |
160 | return NULL; |
161 | } | |
162 | ||
163 | start = orc_lookup[idx]; | |
164 | stop = orc_lookup[idx + 1] + 1; | |
165 | ||
166 | if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || | |
167 | (__start_orc_unwind + stop > __stop_orc_unwind))) { | |
58c3862b JP |
168 | orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n", |
169 | idx, lookup_num_blocks, start, stop, (void *)ip); | |
ee9f8fce JP |
170 | return NULL; |
171 | } | |
172 | ||
173 | return __orc_find(__start_orc_unwind_ip + start, | |
174 | __start_orc_unwind + start, stop - start, ip); | |
175 | } | |
176 | ||
177 | /* vmlinux .init slow lookup: */ | |
9fbcc57a | 178 | if (init_kernel_text(ip)) |
ee9f8fce JP |
179 | return __orc_find(__start_orc_unwind_ip, __start_orc_unwind, |
180 | __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); | |
181 | ||
182 | /* Module lookup: */ | |
6be7fa3c SRV |
183 | orc = orc_module_find(ip); |
184 | if (orc) | |
185 | return orc; | |
186 | ||
187 | return orc_ftrace_find(ip); | |
ee9f8fce JP |
188 | } |
189 | ||
22a7fa88 SZ |
190 | #ifdef CONFIG_MODULES |
191 | ||
fb9cbbc8 JP |
192 | static DEFINE_MUTEX(sort_mutex); |
193 | static int *cur_orc_ip_table = __start_orc_unwind_ip; | |
194 | static struct orc_entry *cur_orc_table = __start_orc_unwind; | |
195 | ||
ee9f8fce JP |
196 | static void orc_sort_swap(void *_a, void *_b, int size) |
197 | { | |
198 | struct orc_entry *orc_a, *orc_b; | |
199 | struct orc_entry orc_tmp; | |
200 | int *a = _a, *b = _b, tmp; | |
201 | int delta = _b - _a; | |
202 | ||
203 | /* Swap the .orc_unwind_ip entries: */ | |
204 | tmp = *a; | |
205 | *a = *b + delta; | |
206 | *b = tmp - delta; | |
207 | ||
208 | /* Swap the corresponding .orc_unwind entries: */ | |
209 | orc_a = cur_orc_table + (a - cur_orc_ip_table); | |
210 | orc_b = cur_orc_table + (b - cur_orc_ip_table); | |
211 | orc_tmp = *orc_a; | |
212 | *orc_a = *orc_b; | |
213 | *orc_b = orc_tmp; | |
214 | } | |
215 | ||
216 | static int orc_sort_cmp(const void *_a, const void *_b) | |
217 | { | |
218 | struct orc_entry *orc_a; | |
219 | const int *a = _a, *b = _b; | |
220 | unsigned long a_val = orc_ip(a); | |
221 | unsigned long b_val = orc_ip(b); | |
222 | ||
223 | if (a_val > b_val) | |
224 | return 1; | |
225 | if (a_val < b_val) | |
226 | return -1; | |
227 | ||
228 | /* | |
229 | * The "weak" section terminator entries need to always be on the left | |
230 | * to ensure the lookup code skips them in favor of real entries. | |
231 | * These terminator entries exist to handle any gaps created by | |
232 | * whitelisted .o files which didn't get objtool generation. | |
233 | */ | |
234 | orc_a = cur_orc_table + (a - cur_orc_ip_table); | |
d31a5802 | 235 | return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1; |
ee9f8fce JP |
236 | } |
237 | ||
ee9f8fce JP |
238 | void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, |
239 | void *_orc, size_t orc_size) | |
240 | { | |
241 | int *orc_ip = _orc_ip; | |
242 | struct orc_entry *orc = _orc; | |
243 | unsigned int num_entries = orc_ip_size / sizeof(int); | |
244 | ||
245 | WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || | |
246 | orc_size % sizeof(*orc) != 0 || | |
247 | num_entries != orc_size / sizeof(*orc)); | |
248 | ||
249 | /* | |
250 | * The 'cur_orc_*' globals allow the orc_sort_swap() callback to | |
251 | * associate an .orc_unwind_ip table entry with its corresponding | |
252 | * .orc_unwind entry so they can both be swapped. | |
253 | */ | |
254 | mutex_lock(&sort_mutex); | |
255 | cur_orc_ip_table = orc_ip; | |
256 | cur_orc_table = orc; | |
257 | sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap); | |
258 | mutex_unlock(&sort_mutex); | |
259 | ||
260 | mod->arch.orc_unwind_ip = orc_ip; | |
261 | mod->arch.orc_unwind = orc; | |
262 | mod->arch.num_orcs = num_entries; | |
263 | } | |
264 | #endif | |
265 | ||
266 | void __init unwind_init(void) | |
267 | { | |
268 | size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; | |
269 | size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; | |
270 | size_t num_entries = orc_ip_size / sizeof(int); | |
271 | struct orc_entry *orc; | |
272 | int i; | |
273 | ||
274 | if (!num_entries || orc_ip_size % sizeof(int) != 0 || | |
275 | orc_size % sizeof(struct orc_entry) != 0 || | |
276 | num_entries != orc_size / sizeof(struct orc_entry)) { | |
277 | orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n"); | |
278 | return; | |
279 | } | |
280 | ||
f14bf6a3 SZ |
281 | /* |
282 | * Note, the orc_unwind and orc_unwind_ip tables were already | |
283 | * sorted at build time via the 'sorttable' tool. | |
284 | * It's ready for binary search straight away, no need to sort it. | |
285 | */ | |
ee9f8fce JP |
286 | |
287 | /* Initialize the fast lookup table: */ | |
288 | lookup_num_blocks = orc_lookup_end - orc_lookup; | |
289 | for (i = 0; i < lookup_num_blocks-1; i++) { | |
290 | orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, | |
291 | num_entries, | |
292 | LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); | |
293 | if (!orc) { | |
294 | orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); | |
295 | return; | |
296 | } | |
297 | ||
298 | orc_lookup[i] = orc - __start_orc_unwind; | |
299 | } | |
300 | ||
301 | /* Initialize the ending block: */ | |
302 | orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, | |
303 | LOOKUP_STOP_IP); | |
304 | if (!orc) { | |
305 | orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); | |
306 | return; | |
307 | } | |
308 | orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; | |
309 | ||
310 | orc_init = true; | |
311 | } | |
312 | ||
313 | unsigned long unwind_get_return_address(struct unwind_state *state) | |
314 | { | |
315 | if (unwind_done(state)) | |
316 | return 0; | |
317 | ||
318 | return __kernel_text_address(state->ip) ? state->ip : 0; | |
319 | } | |
320 | EXPORT_SYMBOL_GPL(unwind_get_return_address); | |
321 | ||
322 | unsigned long *unwind_get_return_address_ptr(struct unwind_state *state) | |
323 | { | |
324 | if (unwind_done(state)) | |
325 | return NULL; | |
326 | ||
327 | if (state->regs) | |
328 | return &state->regs->ip; | |
329 | ||
330 | if (state->sp) | |
331 | return (unsigned long *)state->sp - 1; | |
332 | ||
333 | return NULL; | |
334 | } | |
335 | ||
b02fcf9b | 336 | static bool stack_access_ok(struct unwind_state *state, unsigned long _addr, |
ee9f8fce JP |
337 | size_t len) |
338 | { | |
339 | struct stack_info *info = &state->stack_info; | |
b02fcf9b | 340 | void *addr = (void *)_addr; |
ee9f8fce | 341 | |
b02fcf9b JP |
342 | if (!on_stack(info, addr, len) && |
343 | (get_stack_info(addr, state->task, info, &state->stack_mask))) | |
344 | return false; | |
ee9f8fce JP |
345 | |
346 | return true; | |
347 | } | |
348 | ||
349 | static bool deref_stack_reg(struct unwind_state *state, unsigned long addr, | |
350 | unsigned long *val) | |
351 | { | |
352 | if (!stack_access_ok(state, addr, sizeof(long))) | |
353 | return false; | |
354 | ||
881125bf | 355 | *val = READ_ONCE_NOCHECK(*(unsigned long *)addr); |
ee9f8fce JP |
356 | return true; |
357 | } | |
358 | ||
ee9f8fce | 359 | static bool deref_stack_regs(struct unwind_state *state, unsigned long addr, |
b02fcf9b | 360 | unsigned long *ip, unsigned long *sp) |
ee9f8fce | 361 | { |
b02fcf9b | 362 | struct pt_regs *regs = (struct pt_regs *)addr; |
ee9f8fce | 363 | |
b02fcf9b JP |
364 | /* x86-32 support will be more complicated due to the ®s->sp hack */ |
365 | BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32)); | |
ee9f8fce | 366 | |
b02fcf9b | 367 | if (!stack_access_ok(state, addr, sizeof(struct pt_regs))) |
ee9f8fce JP |
368 | return false; |
369 | ||
e504e74c JP |
370 | *ip = READ_ONCE_NOCHECK(regs->ip); |
371 | *sp = READ_ONCE_NOCHECK(regs->sp); | |
b02fcf9b JP |
372 | return true; |
373 | } | |
ee9f8fce | 374 | |
b02fcf9b JP |
375 | static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr, |
376 | unsigned long *ip, unsigned long *sp) | |
377 | { | |
378 | struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET; | |
ee9f8fce | 379 | |
b02fcf9b JP |
380 | if (!stack_access_ok(state, addr, IRET_FRAME_SIZE)) |
381 | return false; | |
ee9f8fce | 382 | |
e504e74c JP |
383 | *ip = READ_ONCE_NOCHECK(regs->ip); |
384 | *sp = READ_ONCE_NOCHECK(regs->sp); | |
ee9f8fce JP |
385 | return true; |
386 | } | |
387 | ||
81b67439 JP |
388 | /* |
389 | * If state->regs is non-NULL, and points to a full pt_regs, just get the reg | |
390 | * value from state->regs. | |
391 | * | |
392 | * Otherwise, if state->regs just points to IRET regs, and the previous frame | |
393 | * had full regs, it's safe to get the value from the previous regs. This can | |
394 | * happen when early/late IRQ entry code gets interrupted by an NMI. | |
395 | */ | |
396 | static bool get_reg(struct unwind_state *state, unsigned int reg_off, | |
397 | unsigned long *val) | |
398 | { | |
399 | unsigned int reg = reg_off/8; | |
400 | ||
401 | if (!state->regs) | |
402 | return false; | |
403 | ||
404 | if (state->full_regs) { | |
e504e74c | 405 | *val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]); |
81b67439 JP |
406 | return true; |
407 | } | |
408 | ||
409 | if (state->prev_regs) { | |
e504e74c | 410 | *val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]); |
81b67439 JP |
411 | return true; |
412 | } | |
413 | ||
414 | return false; | |
415 | } | |
416 | ||
ee9f8fce JP |
417 | bool unwind_next_frame(struct unwind_state *state) |
418 | { | |
81b67439 | 419 | unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp; |
ee9f8fce JP |
420 | enum stack_type prev_type = state->stack_info.type; |
421 | struct orc_entry *orc; | |
ee9f8fce JP |
422 | bool indirect = false; |
423 | ||
424 | if (unwind_done(state)) | |
425 | return false; | |
426 | ||
427 | /* Don't let modules unload while we're reading their ORC data. */ | |
428 | preempt_disable(); | |
429 | ||
d31a5802 | 430 | /* End-of-stack check for user tasks: */ |
ee9f8fce | 431 | if (state->regs && user_mode(state->regs)) |
d31a5802 | 432 | goto the_end; |
ee9f8fce JP |
433 | |
434 | /* | |
435 | * Find the orc_entry associated with the text address. | |
436 | * | |
372a8eaa JP |
437 | * For a call frame (as opposed to a signal frame), state->ip points to |
438 | * the instruction after the call. That instruction's stack layout | |
439 | * could be different from the call instruction's layout, for example | |
440 | * if the call was to a noreturn function. So get the ORC data for the | |
441 | * call instruction itself. | |
ee9f8fce JP |
442 | */ |
443 | orc = orc_find(state->signal ? state->ip : state->ip - 1); | |
ae6a45a0 JP |
444 | if (!orc) { |
445 | /* | |
446 | * As a fallback, try to assume this code uses a frame pointer. | |
447 | * This is useful for generated code, like BPF, which ORC | |
448 | * doesn't know about. This is just a guess, so the rest of | |
449 | * the unwind is no longer considered reliable. | |
450 | */ | |
451 | orc = &orc_fp_entry; | |
452 | state->error = true; | |
453 | } | |
d31a5802 JP |
454 | |
455 | /* End-of-stack check for kernel threads: */ | |
456 | if (orc->sp_reg == ORC_REG_UNDEFINED) { | |
457 | if (!orc->end) | |
458 | goto err; | |
459 | ||
460 | goto the_end; | |
461 | } | |
ee9f8fce JP |
462 | |
463 | /* Find the previous frame's stack: */ | |
464 | switch (orc->sp_reg) { | |
465 | case ORC_REG_SP: | |
466 | sp = state->sp + orc->sp_offset; | |
467 | break; | |
468 | ||
469 | case ORC_REG_BP: | |
470 | sp = state->bp + orc->sp_offset; | |
471 | break; | |
472 | ||
473 | case ORC_REG_SP_INDIRECT: | |
87ccc826 | 474 | sp = state->sp; |
ee9f8fce JP |
475 | indirect = true; |
476 | break; | |
477 | ||
478 | case ORC_REG_BP_INDIRECT: | |
479 | sp = state->bp + orc->sp_offset; | |
480 | indirect = true; | |
481 | break; | |
482 | ||
483 | case ORC_REG_R10: | |
81b67439 | 484 | if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) { |
b08418b5 JP |
485 | orc_warn_current("missing R10 value at %pB\n", |
486 | (void *)state->ip); | |
d31a5802 | 487 | goto err; |
ee9f8fce | 488 | } |
ee9f8fce JP |
489 | break; |
490 | ||
491 | case ORC_REG_R13: | |
81b67439 | 492 | if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) { |
b08418b5 JP |
493 | orc_warn_current("missing R13 value at %pB\n", |
494 | (void *)state->ip); | |
d31a5802 | 495 | goto err; |
ee9f8fce | 496 | } |
ee9f8fce JP |
497 | break; |
498 | ||
499 | case ORC_REG_DI: | |
81b67439 | 500 | if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) { |
b08418b5 JP |
501 | orc_warn_current("missing RDI value at %pB\n", |
502 | (void *)state->ip); | |
d31a5802 | 503 | goto err; |
ee9f8fce | 504 | } |
ee9f8fce JP |
505 | break; |
506 | ||
507 | case ORC_REG_DX: | |
81b67439 | 508 | if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) { |
b08418b5 JP |
509 | orc_warn_current("missing DX value at %pB\n", |
510 | (void *)state->ip); | |
d31a5802 | 511 | goto err; |
ee9f8fce | 512 | } |
ee9f8fce JP |
513 | break; |
514 | ||
515 | default: | |
b08418b5 | 516 | orc_warn("unknown SP base reg %d at %pB\n", |
ee9f8fce | 517 | orc->sp_reg, (void *)state->ip); |
d31a5802 | 518 | goto err; |
ee9f8fce JP |
519 | } |
520 | ||
521 | if (indirect) { | |
522 | if (!deref_stack_reg(state, sp, &sp)) | |
d31a5802 | 523 | goto err; |
87ccc826 PZ |
524 | |
525 | if (orc->sp_reg == ORC_REG_SP_INDIRECT) | |
526 | sp += orc->sp_offset; | |
ee9f8fce JP |
527 | } |
528 | ||
529 | /* Find IP, SP and possibly regs: */ | |
530 | switch (orc->type) { | |
ee819aed | 531 | case UNWIND_HINT_TYPE_CALL: |
ee9f8fce JP |
532 | ip_p = sp - sizeof(long); |
533 | ||
534 | if (!deref_stack_reg(state, ip_p, &state->ip)) | |
d31a5802 | 535 | goto err; |
ee9f8fce JP |
536 | |
537 | state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, | |
538 | state->ip, (void *)ip_p); | |
539 | ||
540 | state->sp = sp; | |
541 | state->regs = NULL; | |
81b67439 | 542 | state->prev_regs = NULL; |
ee9f8fce JP |
543 | state->signal = false; |
544 | break; | |
545 | ||
ee819aed | 546 | case UNWIND_HINT_TYPE_REGS: |
b02fcf9b | 547 | if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { |
b08418b5 JP |
548 | orc_warn_current("can't access registers at %pB\n", |
549 | (void *)orig_ip); | |
d31a5802 | 550 | goto err; |
ee9f8fce JP |
551 | } |
552 | ||
553 | state->regs = (struct pt_regs *)sp; | |
81b67439 | 554 | state->prev_regs = NULL; |
ee9f8fce JP |
555 | state->full_regs = true; |
556 | state->signal = true; | |
557 | break; | |
558 | ||
ee819aed | 559 | case UNWIND_HINT_TYPE_REGS_PARTIAL: |
b02fcf9b | 560 | if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { |
b08418b5 JP |
561 | orc_warn_current("can't access iret registers at %pB\n", |
562 | (void *)orig_ip); | |
d31a5802 | 563 | goto err; |
ee9f8fce JP |
564 | } |
565 | ||
81b67439 JP |
566 | if (state->full_regs) |
567 | state->prev_regs = state->regs; | |
b02fcf9b JP |
568 | state->regs = (void *)sp - IRET_FRAME_OFFSET; |
569 | state->full_regs = false; | |
ee9f8fce JP |
570 | state->signal = true; |
571 | break; | |
572 | ||
573 | default: | |
b08418b5 | 574 | orc_warn("unknown .orc_unwind entry type %d at %pB\n", |
58c3862b | 575 | orc->type, (void *)orig_ip); |
a0f81bf2 | 576 | goto err; |
ee9f8fce JP |
577 | } |
578 | ||
579 | /* Find BP: */ | |
580 | switch (orc->bp_reg) { | |
581 | case ORC_REG_UNDEFINED: | |
81b67439 JP |
582 | if (get_reg(state, offsetof(struct pt_regs, bp), &tmp)) |
583 | state->bp = tmp; | |
ee9f8fce JP |
584 | break; |
585 | ||
586 | case ORC_REG_PREV_SP: | |
587 | if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp)) | |
d31a5802 | 588 | goto err; |
ee9f8fce JP |
589 | break; |
590 | ||
591 | case ORC_REG_BP: | |
592 | if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp)) | |
d31a5802 | 593 | goto err; |
ee9f8fce JP |
594 | break; |
595 | ||
596 | default: | |
58c3862b | 597 | orc_warn("unknown BP base reg %d for ip %pB\n", |
ee9f8fce | 598 | orc->bp_reg, (void *)orig_ip); |
d31a5802 | 599 | goto err; |
ee9f8fce JP |
600 | } |
601 | ||
602 | /* Prevent a recursive loop due to bad ORC data: */ | |
603 | if (state->stack_info.type == prev_type && | |
604 | on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) && | |
605 | state->sp <= prev_sp) { | |
b08418b5 JP |
606 | orc_warn_current("stack going in the wrong direction? at %pB\n", |
607 | (void *)orig_ip); | |
d31a5802 | 608 | goto err; |
ee9f8fce JP |
609 | } |
610 | ||
611 | preempt_enable(); | |
612 | return true; | |
613 | ||
d31a5802 JP |
614 | err: |
615 | state->error = true; | |
616 | ||
617 | the_end: | |
ee9f8fce JP |
618 | preempt_enable(); |
619 | state->stack_info.type = STACK_TYPE_UNKNOWN; | |
620 | return false; | |
621 | } | |
622 | EXPORT_SYMBOL_GPL(unwind_next_frame); | |
623 | ||
624 | void __unwind_start(struct unwind_state *state, struct task_struct *task, | |
625 | struct pt_regs *regs, unsigned long *first_frame) | |
626 | { | |
627 | memset(state, 0, sizeof(*state)); | |
628 | state->task = task; | |
629 | ||
71c95825 JP |
630 | if (!orc_init) |
631 | goto err; | |
632 | ||
ee9f8fce JP |
633 | /* |
634 | * Refuse to unwind the stack of a task while it's executing on another | |
635 | * CPU. This check is racy, but that's ok: the unwinder has other | |
636 | * checks to prevent it from going off the rails. | |
637 | */ | |
638 | if (task_on_another_cpu(task)) | |
71c95825 | 639 | goto err; |
ee9f8fce JP |
640 | |
641 | if (regs) { | |
642 | if (user_mode(regs)) | |
71c95825 | 643 | goto the_end; |
ee9f8fce JP |
644 | |
645 | state->ip = regs->ip; | |
3c88c692 | 646 | state->sp = regs->sp; |
ee9f8fce JP |
647 | state->bp = regs->bp; |
648 | state->regs = regs; | |
649 | state->full_regs = true; | |
650 | state->signal = true; | |
651 | ||
652 | } else if (task == current) { | |
653 | asm volatile("lea (%%rip), %0\n\t" | |
654 | "mov %%rsp, %1\n\t" | |
655 | "mov %%rbp, %2\n\t" | |
656 | : "=r" (state->ip), "=r" (state->sp), | |
657 | "=r" (state->bp)); | |
658 | ||
659 | } else { | |
660 | struct inactive_task_frame *frame = (void *)task->thread.sp; | |
661 | ||
f2ac57a4 | 662 | state->sp = task->thread.sp + sizeof(*frame); |
ee9f8fce JP |
663 | state->bp = READ_ONCE_NOCHECK(frame->bp); |
664 | state->ip = READ_ONCE_NOCHECK(frame->ret_addr); | |
372a8eaa | 665 | state->signal = (void *)state->ip == ret_from_fork; |
ee9f8fce JP |
666 | } |
667 | ||
668 | if (get_stack_info((unsigned long *)state->sp, state->task, | |
d3a09104 AL |
669 | &state->stack_info, &state->stack_mask)) { |
670 | /* | |
671 | * We weren't on a valid stack. It's possible that | |
672 | * we overflowed a valid stack into a guard page. | |
673 | * See if the next page up is valid so that we can | |
674 | * generate some kind of backtrace if this happens. | |
675 | */ | |
676 | void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp); | |
71c95825 | 677 | state->error = true; |
d3a09104 AL |
678 | if (get_stack_info(next_page, state->task, &state->stack_info, |
679 | &state->stack_mask)) | |
680 | return; | |
681 | } | |
ee9f8fce JP |
682 | |
683 | /* | |
684 | * The caller can provide the address of the first frame directly | |
685 | * (first_frame) or indirectly (regs->sp) to indicate which stack frame | |
686 | * to start unwinding at. Skip ahead until we reach it. | |
687 | */ | |
688 | ||
689 | /* When starting from regs, skip the regs frame: */ | |
690 | if (regs) { | |
691 | unwind_next_frame(state); | |
692 | return; | |
693 | } | |
694 | ||
695 | /* Otherwise, skip ahead to the user-specified starting frame: */ | |
696 | while (!unwind_done(state) && | |
697 | (!on_stack(&state->stack_info, first_frame, sizeof(long)) || | |
f1d9a2ab | 698 | state->sp < (unsigned long)first_frame)) |
ee9f8fce JP |
699 | unwind_next_frame(state); |
700 | ||
701 | return; | |
702 | ||
71c95825 JP |
703 | err: |
704 | state->error = true; | |
705 | the_end: | |
ee9f8fce | 706 | state->stack_info.type = STACK_TYPE_UNKNOWN; |
ee9f8fce JP |
707 | } |
708 | EXPORT_SYMBOL_GPL(__unwind_start); |