1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 2020, Google LLC.
10 #include <linux/kernel.h>
11 #include <linux/lockdep.h>
12 #include <linux/printk.h>
13 #include <linux/sched/debug.h>
14 #include <linux/seq_file.h>
15 #include <linux/stacktrace.h>
16 #include <linux/string.h>
17 #include <trace/events/error_report.h>
19 #include <asm/kfence.h>
23 /* May be overridden by <asm/kfence.h>. */
24 #ifndef ARCH_FUNC_PREFIX
25 #define ARCH_FUNC_PREFIX ""
28 extern bool no_hash_pointers
;
30 /* Helper function to either print to a seq_file or to console. */
32 static void seq_con_printf(struct seq_file
*seq
, const char *fmt
, ...)
38 seq_vprintf(seq
, fmt
, args
);
45 * Get the number of stack entries to skip to get out of MM internals. @type is
46 * optional, and if set to NULL, assumes an allocation or free stack.
48 static int get_stack_skipnr(const unsigned long stack_entries
[], int num_entries
,
49 const enum kfence_error_type
*type
)
52 int skipnr
, fallback
= 0;
55 /* Depending on error type, find different stack entries. */
57 case KFENCE_ERROR_UAF
:
58 case KFENCE_ERROR_OOB
:
59 case KFENCE_ERROR_INVALID
:
61 * kfence_handle_page_fault() may be called with pt_regs
62 * set to NULL; in that case we'll simply show the full
66 case KFENCE_ERROR_CORRUPTION
:
67 case KFENCE_ERROR_INVALID_FREE
:
72 for (skipnr
= 0; skipnr
< num_entries
; skipnr
++) {
73 int len
= scnprintf(buf
, sizeof(buf
), "%ps", (void *)stack_entries
[skipnr
]);
75 if (str_has_prefix(buf
, ARCH_FUNC_PREFIX
"kfence_") ||
76 str_has_prefix(buf
, ARCH_FUNC_PREFIX
"__kfence_") ||
77 !strncmp(buf
, ARCH_FUNC_PREFIX
"__slab_free", len
)) {
79 * In case of tail calls from any of the below
80 * to any of the above.
82 fallback
= skipnr
+ 1;
85 /* Also the *_bulk() variants by only checking prefixes. */
86 if (str_has_prefix(buf
, ARCH_FUNC_PREFIX
"kfree") ||
87 str_has_prefix(buf
, ARCH_FUNC_PREFIX
"kmem_cache_free") ||
88 str_has_prefix(buf
, ARCH_FUNC_PREFIX
"__kmalloc") ||
89 str_has_prefix(buf
, ARCH_FUNC_PREFIX
"kmem_cache_alloc"))
92 if (fallback
< num_entries
)
96 return skipnr
< num_entries
? skipnr
: 0;
99 static void kfence_print_stack(struct seq_file
*seq
, const struct kfence_metadata
*meta
,
102 const struct kfence_track
*track
= show_alloc
? &meta
->alloc_track
: &meta
->free_track
;
104 if (track
->num_stack_entries
) {
105 /* Skip allocation/free internals stack. */
106 int i
= get_stack_skipnr(track
->stack_entries
, track
->num_stack_entries
, NULL
);
108 /* stack_trace_seq_print() does not exist; open code our own. */
109 for (; i
< track
->num_stack_entries
; i
++)
110 seq_con_printf(seq
, " %pS\n", (void *)track
->stack_entries
[i
]);
112 seq_con_printf(seq
, " no %s stack\n", show_alloc
? "allocation" : "deallocation");
116 void kfence_print_object(struct seq_file
*seq
, const struct kfence_metadata
*meta
)
118 const int size
= abs(meta
->size
);
119 const unsigned long start
= meta
->addr
;
120 const struct kmem_cache
*const cache
= meta
->cache
;
122 lockdep_assert_held(&meta
->lock
);
124 if (meta
->state
== KFENCE_OBJECT_UNUSED
) {
125 seq_con_printf(seq
, "kfence-#%td unused\n", meta
- kfence_metadata
);
130 "kfence-#%td [0x%p-0x%p"
131 ", size=%d, cache=%s] allocated by task %d:\n",
132 meta
- kfence_metadata
, (void *)start
, (void *)(start
+ size
- 1), size
,
133 (cache
&& cache
->name
) ? cache
->name
: "<destroyed>", meta
->alloc_track
.pid
);
134 kfence_print_stack(seq
, meta
, true);
136 if (meta
->state
== KFENCE_OBJECT_FREED
) {
137 seq_con_printf(seq
, "\nfreed by task %d:\n", meta
->free_track
.pid
);
138 kfence_print_stack(seq
, meta
, false);
143 * Show bytes at @addr that are different from the expected canary values, up to
146 static void print_diff_canary(unsigned long address
, size_t bytes_to_show
,
147 const struct kfence_metadata
*meta
)
149 const unsigned long show_until_addr
= address
+ bytes_to_show
;
152 /* Do not show contents of object nor read into following guard page. */
153 end
= (const u8
*)(address
< meta
->addr
? min(show_until_addr
, meta
->addr
)
154 : min(show_until_addr
, PAGE_ALIGN(address
)));
157 for (cur
= (const u8
*)address
; cur
< end
; cur
++) {
158 if (*cur
== KFENCE_CANARY_PATTERN(cur
))
160 else if (no_hash_pointers
)
161 pr_cont(" 0x%02x", *cur
);
162 else /* Do not leak kernel memory in non-debug builds. */
168 static const char *get_access_type(bool is_write
)
170 return is_write
? "write" : "read";
173 void kfence_report_error(unsigned long address
, bool is_write
, struct pt_regs
*regs
,
174 const struct kfence_metadata
*meta
, enum kfence_error_type type
)
176 unsigned long stack_entries
[KFENCE_STACK_DEPTH
] = { 0 };
177 const ptrdiff_t object_index
= meta
? meta
- kfence_metadata
: -1;
178 int num_stack_entries
;
182 num_stack_entries
= stack_trace_save_regs(regs
, stack_entries
, KFENCE_STACK_DEPTH
, 0);
184 num_stack_entries
= stack_trace_save(stack_entries
, KFENCE_STACK_DEPTH
, 1);
185 skipnr
= get_stack_skipnr(stack_entries
, num_stack_entries
, &type
);
188 /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */
189 if (WARN_ON(type
!= KFENCE_ERROR_INVALID
&& !meta
))
193 lockdep_assert_held(&meta
->lock
);
195 * Because we may generate reports in printk-unfriendly parts of the
196 * kernel, such as scheduler code, the use of printk() could deadlock.
197 * Until such time that all printing code here is safe in all parts of
198 * the kernel, accept the risk, and just get our message out (given the
199 * system might already behave unpredictably due to the memory error).
200 * As such, also disable lockdep to hide warnings, and avoid disabling
201 * lockdep for the rest of the kernel.
205 pr_err("==================================================================\n");
206 /* Print report header. */
208 case KFENCE_ERROR_OOB
: {
209 const bool left_of_object
= address
< meta
->addr
;
211 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write
),
212 (void *)stack_entries
[skipnr
]);
213 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n",
214 get_access_type(is_write
), (void *)address
,
215 left_of_object
? meta
->addr
- address
: address
- meta
->addr
,
216 left_of_object
? "left" : "right", object_index
);
219 case KFENCE_ERROR_UAF
:
220 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write
),
221 (void *)stack_entries
[skipnr
]);
222 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n",
223 get_access_type(is_write
), (void *)address
, object_index
);
225 case KFENCE_ERROR_CORRUPTION
:
226 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries
[skipnr
]);
227 pr_err("Corrupted memory at 0x%p ", (void *)address
);
228 print_diff_canary(address
, 16, meta
);
229 pr_cont(" (in kfence-#%td):\n", object_index
);
231 case KFENCE_ERROR_INVALID
:
232 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write
),
233 (void *)stack_entries
[skipnr
]);
234 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write
),
237 case KFENCE_ERROR_INVALID_FREE
:
238 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries
[skipnr
]);
239 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address
,
244 /* Print stack trace and object info. */
245 stack_trace_print(stack_entries
+ skipnr
, num_stack_entries
- skipnr
, 0);
249 kfence_print_object(NULL
, meta
);
252 /* Print report footer. */
254 if (no_hash_pointers
&& regs
)
257 dump_stack_print_info(KERN_ERR
);
258 trace_error_report_end(ERROR_DETECTOR_KFENCE
, address
);
259 pr_err("==================================================================\n");
264 panic("panic_on_warn set ...\n");
266 /* We encountered a memory safety error, taint the kernel! */
267 add_taint(TAINT_BAD_PAGE
, LOCKDEP_STILL_OK
);