1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/atomic.h>
4 #include <linux/bsearch.h>
6 #include <linux/debugfs.h>
7 #include <linux/init.h>
8 #include <linux/kallsyms.h>
9 #include <linux/sched.h>
10 #include <linux/seq_file.h>
11 #include <linux/slab.h>
12 #include <linux/sort.h>
13 #include <linux/string.h>
14 #include <linux/uaccess.h>
19 * Statistics counters.
21 static atomic_long_t counters
[KCSAN_COUNTER_COUNT
];
24 * Addresses for filtering functions from reporting. This list can be used as a
25 * whitelist or blacklist.
28 unsigned long *addrs
; /* array of addresses */
29 size_t size
; /* current size */
30 int used
; /* number of elements used */
31 bool sorted
; /* if elements are sorted */
32 bool whitelist
; /* if list is a blacklist or whitelist */
33 } report_filterlist
= {
35 .size
= 8, /* small initial size */
38 .whitelist
= false, /* default is blacklist */
40 static DEFINE_SPINLOCK(report_filterlist_lock
);
42 static const char *counter_to_name(enum kcsan_counter_id id
)
45 case KCSAN_COUNTER_USED_WATCHPOINTS
: return "used_watchpoints";
46 case KCSAN_COUNTER_SETUP_WATCHPOINTS
: return "setup_watchpoints";
47 case KCSAN_COUNTER_DATA_RACES
: return "data_races";
48 case KCSAN_COUNTER_ASSERT_FAILURES
: return "assert_failures";
49 case KCSAN_COUNTER_NO_CAPACITY
: return "no_capacity";
50 case KCSAN_COUNTER_REPORT_RACES
: return "report_races";
51 case KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN
: return "races_unknown_origin";
52 case KCSAN_COUNTER_UNENCODABLE_ACCESSES
: return "unencodable_accesses";
53 case KCSAN_COUNTER_ENCODING_FALSE_POSITIVES
: return "encoding_false_positives";
54 case KCSAN_COUNTER_COUNT
:
60 void kcsan_counter_inc(enum kcsan_counter_id id
)
62 atomic_long_inc(&counters
[id
]);
65 void kcsan_counter_dec(enum kcsan_counter_id id
)
67 atomic_long_dec(&counters
[id
]);
71 * The microbenchmark allows benchmarking KCSAN core runtime only. To run
72 * multiple threads, pipe 'microbench=<iters>' from multiple tasks into the
73 * debugfs file. This will not generate any conflicts, and tests fast-path only.
75 static noinline
void microbenchmark(unsigned long iters
)
77 const struct kcsan_ctx ctx_save
= current
->kcsan_ctx
;
78 const bool was_enabled
= READ_ONCE(kcsan_enabled
);
81 /* We may have been called from an atomic region; reset context. */
82 memset(¤t
->kcsan_ctx
, 0, sizeof(current
->kcsan_ctx
));
84 * Disable to benchmark fast-path for all accesses, and (expected
85 * negligible) call into slow-path, but never set up watchpoints.
87 WRITE_ONCE(kcsan_enabled
, false);
89 pr_info("KCSAN: %s begin | iters: %lu\n", __func__
, iters
);
91 cycles
= get_cycles();
93 unsigned long addr
= iters
& ((PAGE_SIZE
<< 8) - 1);
94 int type
= !(iters
& 0x7f) ? KCSAN_ACCESS_ATOMIC
:
95 (!(iters
& 0xf) ? KCSAN_ACCESS_WRITE
: 0);
96 __kcsan_check_access((void *)addr
, sizeof(long), type
);
98 cycles
= get_cycles() - cycles
;
100 pr_info("KCSAN: %s end | cycles: %llu\n", __func__
, cycles
);
102 WRITE_ONCE(kcsan_enabled
, was_enabled
);
103 /* restore context */
104 current
->kcsan_ctx
= ctx_save
;
108 * Simple test to create conflicting accesses. Write 'test=<iters>' to KCSAN's
109 * debugfs file from multiple tasks to generate real conflicts and show reports.
111 static long test_dummy
;
112 static long test_flags
;
113 static long test_scoped
;
114 static noinline
void test_thread(unsigned long iters
)
116 const long CHANGE_BITS
= 0xff00ff00ff00ff00L
;
117 const struct kcsan_ctx ctx_save
= current
->kcsan_ctx
;
120 /* We may have been called from an atomic region; reset context. */
121 memset(¤t
->kcsan_ctx
, 0, sizeof(current
->kcsan_ctx
));
123 pr_info("KCSAN: %s begin | iters: %lu\n", __func__
, iters
);
124 pr_info("test_dummy@%px, test_flags@%px, test_scoped@%px,\n",
125 &test_dummy
, &test_flags
, &test_scoped
);
127 cycles
= get_cycles();
129 /* These all should generate reports. */
130 __kcsan_check_read(&test_dummy
, sizeof(test_dummy
));
131 ASSERT_EXCLUSIVE_WRITER(test_dummy
);
132 ASSERT_EXCLUSIVE_ACCESS(test_dummy
);
134 ASSERT_EXCLUSIVE_BITS(test_flags
, ~CHANGE_BITS
); /* no report */
135 __kcsan_check_read(&test_flags
, sizeof(test_flags
)); /* no report */
137 ASSERT_EXCLUSIVE_BITS(test_flags
, CHANGE_BITS
); /* report */
138 __kcsan_check_read(&test_flags
, sizeof(test_flags
)); /* no report */
140 /* not actually instrumented */
141 WRITE_ONCE(test_dummy
, iters
); /* to observe value-change */
142 __kcsan_check_write(&test_dummy
, sizeof(test_dummy
));
144 test_flags
^= CHANGE_BITS
; /* generate value-change */
145 __kcsan_check_write(&test_flags
, sizeof(test_flags
));
147 BUG_ON(current
->kcsan_ctx
.scoped_accesses
.prev
);
149 /* Should generate reports anywhere in this block. */
150 ASSERT_EXCLUSIVE_WRITER_SCOPED(test_scoped
);
151 ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_scoped
);
152 BUG_ON(!current
->kcsan_ctx
.scoped_accesses
.prev
);
153 /* Unrelated accesses. */
154 __kcsan_check_access(&cycles
, sizeof(cycles
), 0);
155 __kcsan_check_access(&cycles
, sizeof(cycles
), KCSAN_ACCESS_ATOMIC
);
157 BUG_ON(current
->kcsan_ctx
.scoped_accesses
.prev
);
159 cycles
= get_cycles() - cycles
;
161 pr_info("KCSAN: %s end | cycles: %llu\n", __func__
, cycles
);
163 /* restore context */
164 current
->kcsan_ctx
= ctx_save
;
167 static int cmp_filterlist_addrs(const void *rhs
, const void *lhs
)
169 const unsigned long a
= *(const unsigned long *)rhs
;
170 const unsigned long b
= *(const unsigned long *)lhs
;
172 return a
< b
? -1 : a
== b
? 0 : 1;
175 bool kcsan_skip_report_debugfs(unsigned long func_addr
)
177 unsigned long symbolsize
, offset
;
181 if (!kallsyms_lookup_size_offset(func_addr
, &symbolsize
, &offset
))
183 func_addr
-= offset
; /* Get function start */
185 spin_lock_irqsave(&report_filterlist_lock
, flags
);
186 if (report_filterlist
.used
== 0)
189 /* Sort array if it is unsorted, and then do a binary search. */
190 if (!report_filterlist
.sorted
) {
191 sort(report_filterlist
.addrs
, report_filterlist
.used
,
192 sizeof(unsigned long), cmp_filterlist_addrs
, NULL
);
193 report_filterlist
.sorted
= true;
195 ret
= !!bsearch(&func_addr
, report_filterlist
.addrs
,
196 report_filterlist
.used
, sizeof(unsigned long),
197 cmp_filterlist_addrs
);
198 if (report_filterlist
.whitelist
)
202 spin_unlock_irqrestore(&report_filterlist_lock
, flags
);
206 static void set_report_filterlist_whitelist(bool whitelist
)
210 spin_lock_irqsave(&report_filterlist_lock
, flags
);
211 report_filterlist
.whitelist
= whitelist
;
212 spin_unlock_irqrestore(&report_filterlist_lock
, flags
);
215 /* Returns 0 on success, error-code otherwise. */
216 static ssize_t
insert_report_filterlist(const char *func
)
219 unsigned long addr
= kallsyms_lookup_name(func
);
223 pr_err("KCSAN: could not find function: '%s'\n", func
);
227 spin_lock_irqsave(&report_filterlist_lock
, flags
);
229 if (report_filterlist
.addrs
== NULL
) {
230 /* initial allocation */
231 report_filterlist
.addrs
=
232 kmalloc_array(report_filterlist
.size
,
233 sizeof(unsigned long), GFP_ATOMIC
);
234 if (report_filterlist
.addrs
== NULL
) {
238 } else if (report_filterlist
.used
== report_filterlist
.size
) {
239 /* resize filterlist */
240 size_t new_size
= report_filterlist
.size
* 2;
241 unsigned long *new_addrs
=
242 krealloc(report_filterlist
.addrs
,
243 new_size
* sizeof(unsigned long), GFP_ATOMIC
);
245 if (new_addrs
== NULL
) {
246 /* leave filterlist itself untouched */
251 report_filterlist
.size
= new_size
;
252 report_filterlist
.addrs
= new_addrs
;
255 /* Note: deduplicating should be done in userspace. */
256 report_filterlist
.addrs
[report_filterlist
.used
++] =
257 kallsyms_lookup_name(func
);
258 report_filterlist
.sorted
= false;
261 spin_unlock_irqrestore(&report_filterlist_lock
, flags
);
266 static int show_info(struct seq_file
*file
, void *v
)
272 seq_printf(file
, "enabled: %i\n", READ_ONCE(kcsan_enabled
));
273 for (i
= 0; i
< KCSAN_COUNTER_COUNT
; ++i
)
274 seq_printf(file
, "%s: %ld\n", counter_to_name(i
),
275 atomic_long_read(&counters
[i
]));
277 /* show filter functions, and filter type */
278 spin_lock_irqsave(&report_filterlist_lock
, flags
);
279 seq_printf(file
, "\n%s functions: %s\n",
280 report_filterlist
.whitelist
? "whitelisted" : "blacklisted",
281 report_filterlist
.used
== 0 ? "none" : "");
282 for (i
= 0; i
< report_filterlist
.used
; ++i
)
283 seq_printf(file
, " %ps\n", (void *)report_filterlist
.addrs
[i
]);
284 spin_unlock_irqrestore(&report_filterlist_lock
, flags
);
289 static int debugfs_open(struct inode
*inode
, struct file
*file
)
291 return single_open(file
, show_info
, NULL
);
295 debugfs_write(struct file
*file
, const char __user
*buf
, size_t count
, loff_t
*off
)
297 char kbuf
[KSYM_NAME_LEN
];
299 int read_len
= count
< (sizeof(kbuf
) - 1) ? count
: (sizeof(kbuf
) - 1);
301 if (copy_from_user(kbuf
, buf
, read_len
))
303 kbuf
[read_len
] = '\0';
304 arg
= strstrip(kbuf
);
306 if (!strcmp(arg
, "on")) {
307 WRITE_ONCE(kcsan_enabled
, true);
308 } else if (!strcmp(arg
, "off")) {
309 WRITE_ONCE(kcsan_enabled
, false);
310 } else if (!strncmp(arg
, "microbench=", sizeof("microbench=") - 1)) {
313 if (kstrtoul(&arg
[sizeof("microbench=") - 1], 0, &iters
))
315 microbenchmark(iters
);
316 } else if (!strncmp(arg
, "test=", sizeof("test=") - 1)) {
319 if (kstrtoul(&arg
[sizeof("test=") - 1], 0, &iters
))
322 } else if (!strcmp(arg
, "whitelist")) {
323 set_report_filterlist_whitelist(true);
324 } else if (!strcmp(arg
, "blacklist")) {
325 set_report_filterlist_whitelist(false);
326 } else if (arg
[0] == '!') {
327 ssize_t ret
= insert_report_filterlist(&arg
[1]);
338 static const struct file_operations debugfs_ops
=
341 .open
= debugfs_open
,
342 .write
= debugfs_write
,
343 .release
= single_release
346 void __init
kcsan_debugfs_init(void)
348 debugfs_create_file("kcsan", 0644, NULL
, NULL
, &debugfs_ops
);