]>
Commit | Line | Data |
---|---|---|
d08b9f0c ST |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Shadow Call Stack support. | |
4 | * | |
5 | * Copyright (C) 2019 Google LLC | |
6 | */ | |
7 | ||
8 | #include <linux/kasan.h> | |
628d06a4 | 9 | #include <linux/mm.h> |
d08b9f0c ST |
10 | #include <linux/scs.h> |
11 | #include <linux/slab.h> | |
628d06a4 | 12 | #include <linux/vmstat.h> |
d08b9f0c ST |
13 | #include <asm/scs.h> |
14 | ||
15 | static struct kmem_cache *scs_cache; | |
16 | ||
bee348fa WD |
17 | static void __scs_account(void *s, int account) |
18 | { | |
19 | struct page *scs_page = virt_to_page(s); | |
20 | ||
21 | mod_zone_page_state(page_zone(scs_page), NR_KERNEL_SCS_KB, | |
22 | account * (SCS_SIZE / SZ_1K)); | |
23 | } | |
24 | ||
d08b9f0c ST |
25 | static void *scs_alloc(int node) |
26 | { | |
bee348fa WD |
27 | void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node); |
28 | ||
29 | if (!s) | |
30 | return NULL; | |
d08b9f0c | 31 | |
bee348fa WD |
32 | *__scs_magic(s) = SCS_END_MAGIC; |
33 | ||
34 | /* | |
35 | * Poison the allocation to catch unintentional accesses to | |
36 | * the shadow stack when KASAN is enabled. | |
37 | */ | |
38 | kasan_poison_object_data(scs_cache, s); | |
39 | __scs_account(s, 1); | |
d08b9f0c ST |
40 | return s; |
41 | } | |
42 | ||
43 | static void scs_free(void *s) | |
44 | { | |
bee348fa | 45 | __scs_account(s, -1); |
d08b9f0c ST |
46 | kasan_unpoison_object_data(scs_cache, s); |
47 | kmem_cache_free(scs_cache, s); | |
48 | } | |
49 | ||
50 | void __init scs_init(void) | |
51 | { | |
52 | scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL); | |
53 | } | |
54 | ||
55 | int scs_prepare(struct task_struct *tsk, int node) | |
56 | { | |
57 | void *s = scs_alloc(node); | |
58 | ||
59 | if (!s) | |
60 | return -ENOMEM; | |
61 | ||
51189c7a | 62 | task_scs(tsk) = task_scs_sp(tsk) = s; |
d08b9f0c ST |
63 | return 0; |
64 | } | |
65 | ||
5bbaf9d1 ST |
66 | static void scs_check_usage(struct task_struct *tsk) |
67 | { | |
68 | static unsigned long highest; | |
69 | ||
70 | unsigned long *p, prev, curr = highest, used = 0; | |
71 | ||
72 | if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) | |
73 | return; | |
74 | ||
75 | for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { | |
76 | if (!READ_ONCE_NOCHECK(*p)) | |
77 | break; | |
78 | used++; | |
79 | } | |
80 | ||
81 | while (used > curr) { | |
82 | prev = cmpxchg_relaxed(&highest, curr, used); | |
83 | ||
84 | if (prev == curr) { | |
85 | pr_info("%s (%d): highest shadow stack usage: %lu bytes\n", | |
86 | tsk->comm, task_pid_nr(tsk), used); | |
87 | break; | |
88 | } | |
89 | ||
90 | curr = prev; | |
91 | } | |
92 | } | |
93 | ||
d08b9f0c ST |
94 | void scs_release(struct task_struct *tsk) |
95 | { | |
96 | void *s = task_scs(tsk); | |
97 | ||
98 | if (!s) | |
99 | return; | |
100 | ||
101 | WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n"); | |
5bbaf9d1 | 102 | scs_check_usage(tsk); |
d08b9f0c ST |
103 | scs_free(s); |
104 | } |