]>
Commit | Line | Data |
---|---|---|
7c7900f8 | 1 | #include <linux/sched.h> |
29930025 | 2 | #include <linux/sched/task.h> |
68db0cf1 | 3 | #include <linux/sched/task_stack.h> |
a8b7a923 JP |
4 | #include <linux/interrupt.h> |
5 | #include <asm/sections.h> | |
7c7900f8 JP |
6 | #include <asm/ptrace.h> |
7 | #include <asm/bitops.h> | |
8 | #include <asm/stacktrace.h> | |
9 | #include <asm/unwind.h> | |
10 | ||
11 | #define FRAME_HEADER_SIZE (sizeof(long) * 2) | |
12 | ||
84936118 JP |
13 | /* |
14 | * This disables KASAN checking when reading a value from another task's stack, | |
15 | * since the other task could be running on another CPU and could have poisoned | |
16 | * the stack in the meantime. | |
17 | */ | |
18 | #define READ_ONCE_TASK_STACK(task, x) \ | |
19 | ({ \ | |
20 | unsigned long val; \ | |
21 | if (task == current) \ | |
22 | val = READ_ONCE(x); \ | |
23 | else \ | |
24 | val = READ_ONCE_NOCHECK(x); \ | |
25 | val; \ | |
26 | }) | |
27 | ||
aa4f8534 | 28 | static void unwind_dump(struct unwind_state *state) |
8b5e99f0 JP |
29 | { |
30 | static bool dumped_before = false; | |
31 | bool prev_zero, zero = false; | |
aa4f8534 | 32 | unsigned long word, *sp; |
262fa734 JP |
33 | struct stack_info stack_info = {0}; |
34 | unsigned long visit_mask = 0; | |
8b5e99f0 JP |
35 | |
36 | if (dumped_before) | |
37 | return; | |
38 | ||
39 | dumped_before = true; | |
40 | ||
4ea3d741 | 41 | printk_deferred("unwind stack type:%d next_sp:%p mask:0x%lx graph_idx:%d\n", |
8b5e99f0 JP |
42 | state->stack_info.type, state->stack_info.next_sp, |
43 | state->stack_mask, state->graph_idx); | |
44 | ||
262fa734 JP |
45 | for (sp = state->orig_sp; sp; sp = PTR_ALIGN(stack_info.next_sp, sizeof(long))) { |
46 | if (get_stack_info(sp, state->task, &stack_info, &visit_mask)) | |
47 | break; | |
8b5e99f0 | 48 | |
262fa734 | 49 | for (; sp < stack_info.end; sp++) { |
8b5e99f0 | 50 | |
262fa734 JP |
51 | word = READ_ONCE_NOCHECK(*sp); |
52 | ||
53 | prev_zero = zero; | |
54 | zero = word == 0; | |
8b5e99f0 | 55 | |
262fa734 JP |
56 | if (zero) { |
57 | if (!prev_zero) | |
58 | printk_deferred("%p: %0*x ...\n", | |
59 | sp, BITS_PER_LONG/4, 0); | |
60 | continue; | |
61 | } | |
62 | ||
63 | printk_deferred("%p: %0*lx (%pB)\n", | |
64 | sp, BITS_PER_LONG/4, word, (void *)word); | |
65 | } | |
8b5e99f0 JP |
66 | } |
67 | } | |
68 | ||
7c7900f8 JP |
69 | unsigned long unwind_get_return_address(struct unwind_state *state) |
70 | { | |
7c7900f8 JP |
71 | if (unwind_done(state)) |
72 | return 0; | |
73 | ||
6bcdf9d5 | 74 | return __kernel_text_address(state->ip) ? state->ip : 0; |
7c7900f8 JP |
75 | } |
76 | EXPORT_SYMBOL_GPL(unwind_get_return_address); | |
77 | ||
24d86f59 JP |
78 | static size_t regs_size(struct pt_regs *regs) |
79 | { | |
80 | /* x86_32 regs from kernel mode are two words shorter: */ | |
81 | if (IS_ENABLED(CONFIG_X86_32) && !user_mode(regs)) | |
82 | return sizeof(*regs) - 2*sizeof(long); | |
83 | ||
84 | return sizeof(*regs); | |
85 | } | |
86 | ||
a8b7a923 JP |
87 | static bool in_entry_code(unsigned long ip) |
88 | { | |
89 | char *addr = (char *)ip; | |
90 | ||
91 | if (addr >= __entry_text_start && addr < __entry_text_end) | |
92 | return true; | |
93 | ||
94 | #if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN) | |
95 | if (addr >= __irqentry_text_start && addr < __irqentry_text_end) | |
96 | return true; | |
97 | #endif | |
98 | ||
99 | return false; | |
100 | } | |
101 | ||
b0d50c7b JP |
102 | static inline unsigned long *last_frame(struct unwind_state *state) |
103 | { | |
104 | return (unsigned long *)task_pt_regs(state->task) - 2; | |
105 | } | |
106 | ||
519fb5c3 JP |
107 | static bool is_last_frame(struct unwind_state *state) |
108 | { | |
109 | return state->bp == last_frame(state); | |
110 | } | |
111 | ||
87a6b297 JP |
112 | #ifdef CONFIG_X86_32 |
113 | #define GCC_REALIGN_WORDS 3 | |
114 | #else | |
115 | #define GCC_REALIGN_WORDS 1 | |
116 | #endif | |
117 | ||
b0d50c7b JP |
118 | static inline unsigned long *last_aligned_frame(struct unwind_state *state) |
119 | { | |
120 | return last_frame(state) - GCC_REALIGN_WORDS; | |
121 | } | |
122 | ||
519fb5c3 | 123 | static bool is_last_aligned_frame(struct unwind_state *state) |
acb4608a | 124 | { |
b0d50c7b JP |
125 | unsigned long *last_bp = last_frame(state); |
126 | unsigned long *aligned_bp = last_aligned_frame(state); | |
acb4608a | 127 | |
8023e0e2 | 128 | /* |
519fb5c3 JP |
129 | * GCC can occasionally decide to realign the stack pointer and change |
130 | * the offset of the stack frame in the prologue of a function called | |
131 | * by head/entry code. Examples: | |
87a6b297 JP |
132 | * |
133 | * <start_secondary>: | |
134 | * push %edi | |
135 | * lea 0x8(%esp),%edi | |
136 | * and $0xfffffff8,%esp | |
137 | * pushl -0x4(%edi) | |
138 | * push %ebp | |
139 | * mov %esp,%ebp | |
140 | * | |
141 | * <x86_64_start_kernel>: | |
142 | * lea 0x8(%rsp),%r10 | |
143 | * and $0xfffffffffffffff0,%rsp | |
144 | * pushq -0x8(%r10) | |
145 | * push %rbp | |
146 | * mov %rsp,%rbp | |
147 | * | |
519fb5c3 JP |
148 | * After aligning the stack, it pushes a duplicate copy of the return |
149 | * address before pushing the frame pointer. | |
150 | */ | |
151 | return (state->bp == aligned_bp && *(aligned_bp + 1) == *(last_bp + 1)); | |
152 | } | |
153 | ||
154 | static bool is_last_ftrace_frame(struct unwind_state *state) | |
155 | { | |
156 | unsigned long *last_bp = last_frame(state); | |
157 | unsigned long *last_ftrace_bp = last_bp - 3; | |
158 | ||
159 | /* | |
160 | * When unwinding from an ftrace handler of a function called by entry | |
161 | * code, the stack layout of the last frame is: | |
162 | * | |
163 | * bp | |
164 | * parent ret addr | |
165 | * bp | |
166 | * function ret addr | |
167 | * parent ret addr | |
168 | * pt_regs | |
169 | * ----------------- | |
8023e0e2 | 170 | */ |
519fb5c3 JP |
171 | return (state->bp == last_ftrace_bp && |
172 | *state->bp == *(state->bp + 2) && | |
173 | *(state->bp + 1) == *(state->bp + 4)); | |
174 | } | |
175 | ||
176 | static bool is_last_task_frame(struct unwind_state *state) | |
177 | { | |
178 | return is_last_frame(state) || is_last_aligned_frame(state) || | |
179 | is_last_ftrace_frame(state); | |
acb4608a JP |
180 | } |
181 | ||
946c1911 JP |
182 | /* |
183 | * This determines if the frame pointer actually contains an encoded pointer to | |
184 | * pt_regs on the stack. See ENCODE_FRAME_POINTER. | |
185 | */ | |
186 | static struct pt_regs *decode_frame_pointer(unsigned long *bp) | |
187 | { | |
188 | unsigned long regs = (unsigned long)bp; | |
189 | ||
190 | if (!(regs & 0x1)) | |
191 | return NULL; | |
192 | ||
193 | return (struct pt_regs *)(regs & ~0x1); | |
194 | } | |
195 | ||
5ed8d8bb JP |
196 | static bool update_stack_state(struct unwind_state *state, |
197 | unsigned long *next_bp) | |
7c7900f8 JP |
198 | { |
199 | struct stack_info *info = &state->stack_info; | |
5ed8d8bb JP |
200 | enum stack_type prev_type = info->type; |
201 | struct pt_regs *regs; | |
6bcdf9d5 | 202 | unsigned long *frame, *prev_frame_end, *addr_p, addr; |
5ed8d8bb JP |
203 | size_t len; |
204 | ||
205 | if (state->regs) | |
206 | prev_frame_end = (void *)state->regs + regs_size(state->regs); | |
207 | else | |
208 | prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE; | |
209 | ||
210 | /* Is the next frame pointer an encoded pointer to pt_regs? */ | |
211 | regs = decode_frame_pointer(next_bp); | |
212 | if (regs) { | |
213 | frame = (unsigned long *)regs; | |
214 | len = regs_size(regs); | |
a8b7a923 | 215 | state->got_irq = true; |
5ed8d8bb JP |
216 | } else { |
217 | frame = next_bp; | |
218 | len = FRAME_HEADER_SIZE; | |
219 | } | |
7c7900f8 JP |
220 | |
221 | /* | |
5ed8d8bb | 222 | * If the next bp isn't on the current stack, switch to the next one. |
7c7900f8 JP |
223 | * |
224 | * We may have to traverse multiple stacks to deal with the possibility | |
5ed8d8bb JP |
225 | * that info->next_sp could point to an empty stack and the next bp |
226 | * could be on a subsequent stack. | |
7c7900f8 | 227 | */ |
5ed8d8bb | 228 | while (!on_stack(info, frame, len)) |
7c7900f8 JP |
229 | if (get_stack_info(info->next_sp, state->task, info, |
230 | &state->stack_mask)) | |
231 | return false; | |
232 | ||
5ed8d8bb JP |
233 | /* Make sure it only unwinds up and doesn't overlap the prev frame: */ |
234 | if (state->orig_sp && state->stack_info.type == prev_type && | |
235 | frame < prev_frame_end) | |
236 | return false; | |
237 | ||
238 | /* Move state to the next frame: */ | |
239 | if (regs) { | |
240 | state->regs = regs; | |
241 | state->bp = NULL; | |
242 | } else { | |
243 | state->bp = next_bp; | |
244 | state->regs = NULL; | |
245 | } | |
246 | ||
6bcdf9d5 JP |
247 | /* Save the return address: */ |
248 | if (state->regs && user_mode(state->regs)) | |
249 | state->ip = 0; | |
250 | else { | |
251 | addr_p = unwind_get_return_address_ptr(state); | |
252 | addr = READ_ONCE_TASK_STACK(state->task, *addr_p); | |
253 | state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, | |
254 | addr, addr_p); | |
255 | } | |
256 | ||
5ed8d8bb | 257 | /* Save the original stack pointer for unwind_dump(): */ |
262fa734 | 258 | if (!state->orig_sp) |
5ed8d8bb | 259 | state->orig_sp = frame; |
8b5e99f0 | 260 | |
7c7900f8 JP |
261 | return true; |
262 | } | |
263 | ||
264 | bool unwind_next_frame(struct unwind_state *state) | |
265 | { | |
946c1911 | 266 | struct pt_regs *regs; |
5ed8d8bb | 267 | unsigned long *next_bp; |
7c7900f8 JP |
268 | |
269 | if (unwind_done(state)) | |
270 | return false; | |
271 | ||
5ed8d8bb | 272 | /* Have we reached the end? */ |
946c1911 JP |
273 | if (state->regs && user_mode(state->regs)) |
274 | goto the_end; | |
275 | ||
acb4608a JP |
276 | if (is_last_task_frame(state)) { |
277 | regs = task_pt_regs(state->task); | |
278 | ||
279 | /* | |
280 | * kthreads (other than the boot CPU's idle thread) have some | |
281 | * partial regs at the end of their stack which were placed | |
282 | * there by copy_thread_tls(). But the regs don't have any | |
283 | * useful information, so we can skip them. | |
284 | * | |
285 | * This user_mode() check is slightly broader than a PF_KTHREAD | |
286 | * check because it also catches the awkward situation where a | |
287 | * newly forked kthread transitions into a user task by calling | |
288 | * do_execve(), which eventually clears PF_KTHREAD. | |
289 | */ | |
290 | if (!user_mode(regs)) | |
291 | goto the_end; | |
292 | ||
293 | /* | |
294 | * We're almost at the end, but not quite: there's still the | |
295 | * syscall regs frame. Entry code doesn't encode the regs | |
296 | * pointer for syscalls, so we have to set it manually. | |
297 | */ | |
298 | state->regs = regs; | |
299 | state->bp = NULL; | |
6bcdf9d5 | 300 | state->ip = 0; |
acb4608a JP |
301 | return true; |
302 | } | |
303 | ||
5ed8d8bb | 304 | /* Get the next frame pointer: */ |
946c1911 JP |
305 | if (state->regs) |
306 | next_bp = (unsigned long *)state->regs->bp; | |
307 | else | |
5ed8d8bb | 308 | next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp); |
946c1911 | 309 | |
5ed8d8bb | 310 | /* Move to the next frame if it's safe: */ |
a8b7a923 | 311 | if (!update_stack_state(state, next_bp)) |
c32c47c6 | 312 | goto bad_address; |
c32c47c6 | 313 | |
7c7900f8 | 314 | return true; |
946c1911 | 315 | |
c32c47c6 | 316 | bad_address: |
af085d90 JP |
317 | state->error = true; |
318 | ||
900742d8 JP |
319 | /* |
320 | * When unwinding a non-current task, the task might actually be | |
321 | * running on another CPU, in which case it could be modifying its | |
322 | * stack while we're reading it. This is generally not a problem and | |
323 | * can be ignored as long as the caller understands that unwinding | |
324 | * another task will not always succeed. | |
325 | */ | |
326 | if (state->task != current) | |
327 | goto the_end; | |
328 | ||
a8b7a923 JP |
329 | /* |
330 | * Don't warn if the unwinder got lost due to an interrupt in entry | |
b0d50c7b | 331 | * code or in the C handler before the first frame pointer got set up: |
a8b7a923 JP |
332 | */ |
333 | if (state->got_irq && in_entry_code(state->ip)) | |
334 | goto the_end; | |
b0d50c7b JP |
335 | if (state->regs && |
336 | state->regs->sp >= (unsigned long)last_aligned_frame(state) && | |
337 | state->regs->sp < (unsigned long)task_pt_regs(state->task)) | |
338 | goto the_end; | |
a8b7a923 | 339 | |
24d86f59 JP |
340 | if (state->regs) { |
341 | printk_deferred_once(KERN_WARNING | |
342 | "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n", | |
343 | state->regs, state->task->comm, | |
5ed8d8bb | 344 | state->task->pid, next_bp); |
aa4f8534 | 345 | unwind_dump(state); |
24d86f59 JP |
346 | } else { |
347 | printk_deferred_once(KERN_WARNING | |
348 | "WARNING: kernel stack frame pointer at %p in %s:%d has bad value %p\n", | |
349 | state->bp, state->task->comm, | |
5ed8d8bb | 350 | state->task->pid, next_bp); |
aa4f8534 | 351 | unwind_dump(state); |
24d86f59 | 352 | } |
946c1911 JP |
353 | the_end: |
354 | state->stack_info.type = STACK_TYPE_UNKNOWN; | |
355 | return false; | |
7c7900f8 JP |
356 | } |
357 | EXPORT_SYMBOL_GPL(unwind_next_frame); | |
358 | ||
359 | void __unwind_start(struct unwind_state *state, struct task_struct *task, | |
360 | struct pt_regs *regs, unsigned long *first_frame) | |
361 | { | |
5ed8d8bb | 362 | unsigned long *bp; |
946c1911 | 363 | |
7c7900f8 JP |
364 | memset(state, 0, sizeof(*state)); |
365 | state->task = task; | |
a8b7a923 | 366 | state->got_irq = (regs); |
7c7900f8 | 367 | |
5ed8d8bb | 368 | /* Don't even attempt to start from user mode regs: */ |
7c7900f8 JP |
369 | if (regs && user_mode(regs)) { |
370 | state->stack_info.type = STACK_TYPE_UNKNOWN; | |
371 | return; | |
372 | } | |
373 | ||
946c1911 | 374 | bp = get_frame_pointer(task, regs); |
7c7900f8 | 375 | |
5ed8d8bb JP |
376 | /* Initialize stack info and make sure the frame data is accessible: */ |
377 | get_stack_info(bp, state->task, &state->stack_info, | |
7c7900f8 | 378 | &state->stack_mask); |
5ed8d8bb | 379 | update_stack_state(state, bp); |
7c7900f8 JP |
380 | |
381 | /* | |
382 | * The caller can provide the address of the first frame directly | |
383 | * (first_frame) or indirectly (regs->sp) to indicate which stack frame | |
384 | * to start unwinding at. Skip ahead until we reach it. | |
385 | */ | |
386 | while (!unwind_done(state) && | |
387 | (!on_stack(&state->stack_info, first_frame, sizeof(long)) || | |
388 | state->bp < first_frame)) | |
389 | unwind_next_frame(state); | |
390 | } | |
391 | EXPORT_SYMBOL_GPL(__unwind_start); |