]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5c9a8750 DV |
2 | #define pr_fmt(fmt) "kcov: " fmt |
3 | ||
36f05ae8 | 4 | #define DISABLE_BRANCH_PROFILING |
db862358 | 5 | #include <linux/atomic.h> |
5c9a8750 | 6 | #include <linux/compiler.h> |
db862358 KW |
7 | #include <linux/errno.h> |
8 | #include <linux/export.h> | |
5c9a8750 DV |
9 | #include <linux/types.h> |
10 | #include <linux/file.h> | |
11 | #include <linux/fs.h> | |
db862358 | 12 | #include <linux/init.h> |
5c9a8750 | 13 | #include <linux/mm.h> |
db862358 | 14 | #include <linux/preempt.h> |
5c9a8750 | 15 | #include <linux/printk.h> |
166ad0e1 | 16 | #include <linux/sched.h> |
5c9a8750 DV |
17 | #include <linux/slab.h> |
18 | #include <linux/spinlock.h> | |
19 | #include <linux/vmalloc.h> | |
20 | #include <linux/debugfs.h> | |
21 | #include <linux/uaccess.h> | |
22 | #include <linux/kcov.h> | |
39e07cb6 | 23 | #include <linux/refcount.h> |
4983f0ab | 24 | #include <asm/setup.h> |
5c9a8750 | 25 | |
ded97d2c VC |
26 | /* Number of 64-bit words written per one comparison: */ |
27 | #define KCOV_WORDS_PER_CMP 4 | |
28 | ||
5c9a8750 DV |
29 | /* |
30 | * kcov descriptor (one per opened debugfs file). | |
31 | * State transitions of the descriptor: | |
32 | * - initial state after open() | |
33 | * - then there must be a single ioctl(KCOV_INIT_TRACE) call | |
34 | * - then, mmap() call (several calls are allowed but not useful) | |
ded97d2c VC |
35 | * - then, ioctl(KCOV_ENABLE, arg), where arg is |
36 | * KCOV_TRACE_PC - to trace only the PCs | |
37 | * or | |
38 | * KCOV_TRACE_CMP - to trace only the comparison operands | |
39 | * - then, ioctl(KCOV_DISABLE) to disable the task. | |
40 | * Enabling/disabling ioctls can be repeated (only one task a time allowed). | |
5c9a8750 DV |
41 | */ |
42 | struct kcov { | |
43 | /* | |
44 | * Reference counter. We keep one for: | |
45 | * - opened file descriptor | |
46 | * - task with enabled coverage (we can't unwire it from another task) | |
47 | */ | |
39e07cb6 | 48 | refcount_t refcount; |
5c9a8750 DV |
49 | /* The lock protects mode, size, area and t. */ |
50 | spinlock_t lock; | |
51 | enum kcov_mode mode; | |
52 | /* Size of arena (in long's for KCOV_MODE_TRACE). */ | |
53 | unsigned size; | |
54 | /* Coverage buffer shared with user space. */ | |
55 | void *area; | |
56 | /* Task for which we collect coverage, or NULL. */ | |
57 | struct task_struct *t; | |
58 | }; | |
59 | ||
903e8ff8 | 60 | static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) |
5c9a8750 | 61 | { |
0ed557aa | 62 | unsigned int mode; |
5c9a8750 | 63 | |
5c9a8750 DV |
64 | /* |
65 | * We are interested in code coverage as a function of a syscall inputs, | |
66 | * so we ignore code executed in interrupts. | |
67 | */ | |
fcf4edac | 68 | if (!in_task()) |
ded97d2c | 69 | return false; |
5c9a8750 | 70 | mode = READ_ONCE(t->kcov_mode); |
ded97d2c VC |
71 | /* |
72 | * There is some code that runs in interrupts but for which | |
73 | * in_interrupt() returns false (e.g. preempt_schedule_irq()). | |
74 | * READ_ONCE()/barrier() effectively provides load-acquire wrt | |
75 | * interrupts, there are paired barrier()/WRITE_ONCE() in | |
76 | * kcov_ioctl_locked(). | |
77 | */ | |
78 | barrier(); | |
79 | return mode == needed_mode; | |
80 | } | |
4983f0ab | 81 | |
903e8ff8 | 82 | static notrace unsigned long canonicalize_ip(unsigned long ip) |
ded97d2c | 83 | { |
4983f0ab | 84 | #ifdef CONFIG_RANDOMIZE_BASE |
ded97d2c | 85 | ip -= kaslr_offset(); |
4983f0ab | 86 | #endif |
ded97d2c VC |
87 | return ip; |
88 | } | |
5c9a8750 | 89 | |
ded97d2c VC |
90 | /* |
91 | * Entry point from instrumented code. | |
92 | * This is called once per basic-block/edge. | |
93 | */ | |
94 | void notrace __sanitizer_cov_trace_pc(void) | |
95 | { | |
96 | struct task_struct *t; | |
97 | unsigned long *area; | |
98 | unsigned long ip = canonicalize_ip(_RET_IP_); | |
99 | unsigned long pos; | |
100 | ||
101 | t = current; | |
102 | if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t)) | |
103 | return; | |
104 | ||
105 | area = t->kcov_area; | |
106 | /* The first 64-bit word is the number of subsequent PCs. */ | |
107 | pos = READ_ONCE(area[0]) + 1; | |
108 | if (likely(pos < t->kcov_size)) { | |
109 | area[pos] = ip; | |
110 | WRITE_ONCE(area[0], pos); | |
5c9a8750 DV |
111 | } |
112 | } | |
113 | EXPORT_SYMBOL(__sanitizer_cov_trace_pc); | |
114 | ||
ded97d2c | 115 | #ifdef CONFIG_KCOV_ENABLE_COMPARISONS |
63472443 | 116 | static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip) |
ded97d2c VC |
117 | { |
118 | struct task_struct *t; | |
119 | u64 *area; | |
120 | u64 count, start_index, end_pos, max_pos; | |
121 | ||
122 | t = current; | |
123 | if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t)) | |
124 | return; | |
125 | ||
126 | ip = canonicalize_ip(ip); | |
127 | ||
128 | /* | |
129 | * We write all comparison arguments and types as u64. | |
130 | * The buffer was allocated for t->kcov_size unsigned longs. | |
131 | */ | |
132 | area = (u64 *)t->kcov_area; | |
133 | max_pos = t->kcov_size * sizeof(unsigned long); | |
134 | ||
135 | count = READ_ONCE(area[0]); | |
136 | ||
137 | /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */ | |
138 | start_index = 1 + count * KCOV_WORDS_PER_CMP; | |
139 | end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64); | |
140 | if (likely(end_pos <= max_pos)) { | |
141 | area[start_index] = type; | |
142 | area[start_index + 1] = arg1; | |
143 | area[start_index + 2] = arg2; | |
144 | area[start_index + 3] = ip; | |
145 | WRITE_ONCE(area[0], count + 1); | |
146 | } | |
147 | } | |
148 | ||
149 | void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2) | |
150 | { | |
151 | write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_); | |
152 | } | |
153 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1); | |
154 | ||
155 | void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2) | |
156 | { | |
157 | write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_); | |
158 | } | |
159 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2); | |
160 | ||
689d77f0 | 161 | void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2) |
ded97d2c VC |
162 | { |
163 | write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_); | |
164 | } | |
165 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4); | |
166 | ||
167 | void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2) | |
168 | { | |
169 | write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_); | |
170 | } | |
171 | EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8); | |
172 | ||
173 | void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2) | |
174 | { | |
175 | write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, | |
176 | _RET_IP_); | |
177 | } | |
178 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1); | |
179 | ||
180 | void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2) | |
181 | { | |
182 | write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, | |
183 | _RET_IP_); | |
184 | } | |
185 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2); | |
186 | ||
689d77f0 | 187 | void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2) |
ded97d2c VC |
188 | { |
189 | write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, | |
190 | _RET_IP_); | |
191 | } | |
192 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4); | |
193 | ||
194 | void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2) | |
195 | { | |
196 | write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, | |
197 | _RET_IP_); | |
198 | } | |
199 | EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8); | |
200 | ||
201 | void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases) | |
202 | { | |
203 | u64 i; | |
204 | u64 count = cases[0]; | |
205 | u64 size = cases[1]; | |
206 | u64 type = KCOV_CMP_CONST; | |
207 | ||
208 | switch (size) { | |
209 | case 8: | |
210 | type |= KCOV_CMP_SIZE(0); | |
211 | break; | |
212 | case 16: | |
213 | type |= KCOV_CMP_SIZE(1); | |
214 | break; | |
215 | case 32: | |
216 | type |= KCOV_CMP_SIZE(2); | |
217 | break; | |
218 | case 64: | |
219 | type |= KCOV_CMP_SIZE(3); | |
220 | break; | |
221 | default: | |
222 | return; | |
223 | } | |
224 | for (i = 0; i < count; i++) | |
225 | write_comp_data(type, cases[i + 2], val, _RET_IP_); | |
226 | } | |
227 | EXPORT_SYMBOL(__sanitizer_cov_trace_switch); | |
228 | #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */ | |
229 | ||
5c9a8750 DV |
230 | static void kcov_get(struct kcov *kcov) |
231 | { | |
39e07cb6 | 232 | refcount_inc(&kcov->refcount); |
5c9a8750 DV |
233 | } |
234 | ||
235 | static void kcov_put(struct kcov *kcov) | |
236 | { | |
39e07cb6 | 237 | if (refcount_dec_and_test(&kcov->refcount)) { |
5c9a8750 DV |
238 | vfree(kcov->area); |
239 | kfree(kcov); | |
240 | } | |
241 | } | |
242 | ||
243 | void kcov_task_init(struct task_struct *t) | |
244 | { | |
c9484b98 MR |
245 | WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); |
246 | barrier(); | |
5c9a8750 DV |
247 | t->kcov_size = 0; |
248 | t->kcov_area = NULL; | |
249 | t->kcov = NULL; | |
250 | } | |
251 | ||
252 | void kcov_task_exit(struct task_struct *t) | |
253 | { | |
254 | struct kcov *kcov; | |
255 | ||
256 | kcov = t->kcov; | |
257 | if (kcov == NULL) | |
258 | return; | |
259 | spin_lock(&kcov->lock); | |
260 | if (WARN_ON(kcov->t != t)) { | |
261 | spin_unlock(&kcov->lock); | |
262 | return; | |
263 | } | |
264 | /* Just to not leave dangling references behind. */ | |
265 | kcov_task_init(t); | |
266 | kcov->t = NULL; | |
ded97d2c | 267 | kcov->mode = KCOV_MODE_INIT; |
5c9a8750 DV |
268 | spin_unlock(&kcov->lock); |
269 | kcov_put(kcov); | |
270 | } | |
271 | ||
272 | static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) | |
273 | { | |
274 | int res = 0; | |
275 | void *area; | |
276 | struct kcov *kcov = vma->vm_file->private_data; | |
277 | unsigned long size, off; | |
278 | struct page *page; | |
279 | ||
280 | area = vmalloc_user(vma->vm_end - vma->vm_start); | |
281 | if (!area) | |
282 | return -ENOMEM; | |
283 | ||
284 | spin_lock(&kcov->lock); | |
285 | size = kcov->size * sizeof(unsigned long); | |
ded97d2c | 286 | if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 || |
5c9a8750 DV |
287 | vma->vm_end - vma->vm_start != size) { |
288 | res = -EINVAL; | |
289 | goto exit; | |
290 | } | |
291 | if (!kcov->area) { | |
292 | kcov->area = area; | |
293 | vma->vm_flags |= VM_DONTEXPAND; | |
294 | spin_unlock(&kcov->lock); | |
295 | for (off = 0; off < size; off += PAGE_SIZE) { | |
296 | page = vmalloc_to_page(kcov->area + off); | |
297 | if (vm_insert_page(vma, vma->vm_start + off, page)) | |
298 | WARN_ONCE(1, "vm_insert_page() failed"); | |
299 | } | |
300 | return 0; | |
301 | } | |
302 | exit: | |
303 | spin_unlock(&kcov->lock); | |
304 | vfree(area); | |
305 | return res; | |
306 | } | |
307 | ||
308 | static int kcov_open(struct inode *inode, struct file *filep) | |
309 | { | |
310 | struct kcov *kcov; | |
311 | ||
312 | kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); | |
313 | if (!kcov) | |
314 | return -ENOMEM; | |
ded97d2c | 315 | kcov->mode = KCOV_MODE_DISABLED; |
39e07cb6 | 316 | refcount_set(&kcov->refcount, 1); |
5c9a8750 DV |
317 | spin_lock_init(&kcov->lock); |
318 | filep->private_data = kcov; | |
319 | return nonseekable_open(inode, filep); | |
320 | } | |
321 | ||
322 | static int kcov_close(struct inode *inode, struct file *filep) | |
323 | { | |
324 | kcov_put(filep->private_data); | |
325 | return 0; | |
326 | } | |
327 | ||
dc55daff MR |
328 | /* |
329 | * Fault in a lazily-faulted vmalloc area before it can be used by | |
330 | * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the | |
331 | * vmalloc fault handling path is instrumented. | |
332 | */ | |
333 | static void kcov_fault_in_area(struct kcov *kcov) | |
334 | { | |
335 | unsigned long stride = PAGE_SIZE / sizeof(unsigned long); | |
336 | unsigned long *area = kcov->area; | |
337 | unsigned long offset; | |
338 | ||
339 | for (offset = 0; offset < kcov->size; offset += stride) | |
340 | READ_ONCE(area[offset]); | |
341 | } | |
342 | ||
5c9a8750 DV |
343 | static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, |
344 | unsigned long arg) | |
345 | { | |
346 | struct task_struct *t; | |
347 | unsigned long size, unused; | |
348 | ||
349 | switch (cmd) { | |
350 | case KCOV_INIT_TRACE: | |
351 | /* | |
352 | * Enable kcov in trace mode and setup buffer size. | |
353 | * Must happen before anything else. | |
354 | */ | |
355 | if (kcov->mode != KCOV_MODE_DISABLED) | |
356 | return -EBUSY; | |
357 | /* | |
358 | * Size must be at least 2 to hold current position and one PC. | |
359 | * Later we allocate size * sizeof(unsigned long) memory, | |
360 | * that must not overflow. | |
361 | */ | |
362 | size = arg; | |
363 | if (size < 2 || size > INT_MAX / sizeof(unsigned long)) | |
364 | return -EINVAL; | |
365 | kcov->size = size; | |
ded97d2c | 366 | kcov->mode = KCOV_MODE_INIT; |
5c9a8750 DV |
367 | return 0; |
368 | case KCOV_ENABLE: | |
369 | /* | |
370 | * Enable coverage for the current task. | |
371 | * At this point user must have been enabled trace mode, | |
372 | * and mmapped the file. Coverage collection is disabled only | |
373 | * at task exit or voluntary by KCOV_DISABLE. After that it can | |
374 | * be enabled for another task. | |
375 | */ | |
ded97d2c | 376 | if (kcov->mode != KCOV_MODE_INIT || !kcov->area) |
5c9a8750 | 377 | return -EINVAL; |
a77660d2 DV |
378 | t = current; |
379 | if (kcov->t != NULL || t->kcov != NULL) | |
5c9a8750 | 380 | return -EBUSY; |
ded97d2c VC |
381 | if (arg == KCOV_TRACE_PC) |
382 | kcov->mode = KCOV_MODE_TRACE_PC; | |
383 | else if (arg == KCOV_TRACE_CMP) | |
384 | #ifdef CONFIG_KCOV_ENABLE_COMPARISONS | |
385 | kcov->mode = KCOV_MODE_TRACE_CMP; | |
386 | #else | |
387 | return -ENOTSUPP; | |
388 | #endif | |
389 | else | |
390 | return -EINVAL; | |
dc55daff | 391 | kcov_fault_in_area(kcov); |
5c9a8750 DV |
392 | /* Cache in task struct for performance. */ |
393 | t->kcov_size = kcov->size; | |
394 | t->kcov_area = kcov->area; | |
ded97d2c | 395 | /* See comment in check_kcov_mode(). */ |
5c9a8750 DV |
396 | barrier(); |
397 | WRITE_ONCE(t->kcov_mode, kcov->mode); | |
398 | t->kcov = kcov; | |
399 | kcov->t = t; | |
400 | /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */ | |
401 | kcov_get(kcov); | |
402 | return 0; | |
403 | case KCOV_DISABLE: | |
404 | /* Disable coverage for the current task. */ | |
405 | unused = arg; | |
406 | if (unused != 0 || current->kcov != kcov) | |
407 | return -EINVAL; | |
408 | t = current; | |
409 | if (WARN_ON(kcov->t != t)) | |
410 | return -EINVAL; | |
411 | kcov_task_init(t); | |
412 | kcov->t = NULL; | |
ded97d2c | 413 | kcov->mode = KCOV_MODE_INIT; |
5c9a8750 DV |
414 | kcov_put(kcov); |
415 | return 0; | |
416 | default: | |
417 | return -ENOTTY; | |
418 | } | |
419 | } | |
420 | ||
421 | static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | |
422 | { | |
423 | struct kcov *kcov; | |
424 | int res; | |
425 | ||
426 | kcov = filep->private_data; | |
427 | spin_lock(&kcov->lock); | |
428 | res = kcov_ioctl_locked(kcov, cmd, arg); | |
429 | spin_unlock(&kcov->lock); | |
430 | return res; | |
431 | } | |
432 | ||
433 | static const struct file_operations kcov_fops = { | |
434 | .open = kcov_open, | |
435 | .unlocked_ioctl = kcov_ioctl, | |
7483e5d4 | 436 | .compat_ioctl = kcov_ioctl, |
5c9a8750 DV |
437 | .mmap = kcov_mmap, |
438 | .release = kcov_close, | |
439 | }; | |
440 | ||
441 | static int __init kcov_init(void) | |
442 | { | |
df4565f9 NS |
443 | /* |
444 | * The kcov debugfs file won't ever get removed and thus, | |
445 | * there is no need to protect it against removal races. The | |
446 | * use of debugfs_create_file_unsafe() is actually safe here. | |
447 | */ | |
ec9672d5 GKH |
448 | debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops); |
449 | ||
5c9a8750 DV |
450 | return 0; |
451 | } | |
452 | ||
453 | device_initcall(kcov_init); |