]>
Commit | Line | Data |
---|---|---|
8e86e015 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9251f904 BP |
2 | /* |
3 | * Performance events callchain code, extracted from core.c: | |
4 | * | |
5 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> | |
6 | * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar | |
90eec103 | 7 | * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra |
3723c632 | 8 | * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
9251f904 BP |
9 | */ |
10 | ||
11 | #include <linux/perf_event.h> | |
12 | #include <linux/slab.h> | |
68db0cf1 IM |
13 | #include <linux/sched/task_stack.h> |
14 | ||
9251f904 BP |
15 | #include "internal.h" |
16 | ||
17 | struct callchain_cpus_entries { | |
18 | struct rcu_head rcu_head; | |
19 | struct perf_callchain_entry *cpu_entries[0]; | |
20 | }; | |
21 | ||
c5dfd78e | 22 | int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH; |
c85b0334 | 23 | int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK; |
c5dfd78e ACM |
24 | |
25 | static inline size_t perf_callchain_entry__sizeof(void) | |
26 | { | |
27 | return (sizeof(struct perf_callchain_entry) + | |
c85b0334 ACM |
28 | sizeof(__u64) * (sysctl_perf_event_max_stack + |
29 | sysctl_perf_event_max_contexts_per_stack)); | |
c5dfd78e ACM |
30 | } |
31 | ||
9251f904 BP |
32 | static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]); |
33 | static atomic_t nr_callchain_events; | |
34 | static DEFINE_MUTEX(callchain_mutex); | |
35 | static struct callchain_cpus_entries *callchain_cpus_entries; | |
36 | ||
37 | ||
cfbcf468 | 38 | __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, |
9251f904 BP |
39 | struct pt_regs *regs) |
40 | { | |
41 | } | |
42 | ||
cfbcf468 | 43 | __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry, |
9251f904 BP |
44 | struct pt_regs *regs) |
45 | { | |
46 | } | |
47 | ||
48 | static void release_callchain_buffers_rcu(struct rcu_head *head) | |
49 | { | |
50 | struct callchain_cpus_entries *entries; | |
51 | int cpu; | |
52 | ||
53 | entries = container_of(head, struct callchain_cpus_entries, rcu_head); | |
54 | ||
55 | for_each_possible_cpu(cpu) | |
56 | kfree(entries->cpu_entries[cpu]); | |
57 | ||
58 | kfree(entries); | |
59 | } | |
60 | ||
61 | static void release_callchain_buffers(void) | |
62 | { | |
63 | struct callchain_cpus_entries *entries; | |
64 | ||
65 | entries = callchain_cpus_entries; | |
e0455e19 | 66 | RCU_INIT_POINTER(callchain_cpus_entries, NULL); |
9251f904 BP |
67 | call_rcu(&entries->rcu_head, release_callchain_buffers_rcu); |
68 | } | |
69 | ||
70 | static int alloc_callchain_buffers(void) | |
71 | { | |
72 | int cpu; | |
73 | int size; | |
74 | struct callchain_cpus_entries *entries; | |
75 | ||
76 | /* | |
77 | * We can't use the percpu allocation API for data that can be | |
78 | * accessed from NMI. Use a temporary manual per cpu allocation | |
79 | * until that gets sorted out. | |
80 | */ | |
81 | size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]); | |
82 | ||
83 | entries = kzalloc(size, GFP_KERNEL); | |
84 | if (!entries) | |
85 | return -ENOMEM; | |
86 | ||
c5dfd78e | 87 | size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS; |
9251f904 BP |
88 | |
89 | for_each_possible_cpu(cpu) { | |
90 | entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL, | |
91 | cpu_to_node(cpu)); | |
92 | if (!entries->cpu_entries[cpu]) | |
93 | goto fail; | |
94 | } | |
95 | ||
96 | rcu_assign_pointer(callchain_cpus_entries, entries); | |
97 | ||
98 | return 0; | |
99 | ||
100 | fail: | |
101 | for_each_possible_cpu(cpu) | |
102 | kfree(entries->cpu_entries[cpu]); | |
103 | kfree(entries); | |
104 | ||
105 | return -ENOMEM; | |
106 | } | |
107 | ||
97c79a38 | 108 | int get_callchain_buffers(int event_max_stack) |
9251f904 BP |
109 | { |
110 | int err = 0; | |
111 | int count; | |
112 | ||
113 | mutex_lock(&callchain_mutex); | |
114 | ||
115 | count = atomic_inc_return(&nr_callchain_events); | |
116 | if (WARN_ON_ONCE(count < 1)) { | |
117 | err = -EINVAL; | |
118 | goto exit; | |
119 | } | |
120 | ||
5af44ca5 JO |
121 | /* |
122 | * If requesting per event more than the global cap, | |
123 | * return a different error to help userspace figure | |
124 | * this out. | |
125 | * | |
126 | * And also do it here so that we have &callchain_mutex held. | |
127 | */ | |
128 | if (event_max_stack > sysctl_perf_event_max_stack) { | |
129 | err = -EOVERFLOW; | |
130 | goto exit; | |
131 | } | |
132 | ||
bfb3d7b8 JO |
133 | if (count == 1) |
134 | err = alloc_callchain_buffers(); | |
9251f904 | 135 | exit: |
90983b16 FW |
136 | if (err) |
137 | atomic_dec(&nr_callchain_events); | |
9251f904 | 138 | |
fc3b86d6 FW |
139 | mutex_unlock(&callchain_mutex); |
140 | ||
9251f904 BP |
141 | return err; |
142 | } | |
143 | ||
144 | void put_callchain_buffers(void) | |
145 | { | |
146 | if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) { | |
147 | release_callchain_buffers(); | |
148 | mutex_unlock(&callchain_mutex); | |
149 | } | |
150 | } | |
151 | ||
152 | static struct perf_callchain_entry *get_callchain_entry(int *rctx) | |
153 | { | |
154 | int cpu; | |
155 | struct callchain_cpus_entries *entries; | |
156 | ||
4a32fea9 | 157 | *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion)); |
9251f904 BP |
158 | if (*rctx == -1) |
159 | return NULL; | |
160 | ||
161 | entries = rcu_dereference(callchain_cpus_entries); | |
162 | if (!entries) | |
163 | return NULL; | |
164 | ||
165 | cpu = smp_processor_id(); | |
166 | ||
c5dfd78e ACM |
167 | return (((void *)entries->cpu_entries[cpu]) + |
168 | (*rctx * perf_callchain_entry__sizeof())); | |
9251f904 BP |
169 | } |
170 | ||
171 | static void | |
172 | put_callchain_entry(int rctx) | |
173 | { | |
4a32fea9 | 174 | put_recursion_context(this_cpu_ptr(callchain_recursion), rctx); |
9251f904 BP |
175 | } |
176 | ||
568b329a AS |
177 | struct perf_callchain_entry * |
178 | get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, | |
cfbcf468 | 179 | u32 max_stack, bool crosstask, bool add_mark) |
568b329a AS |
180 | { |
181 | struct perf_callchain_entry *entry; | |
cfbcf468 | 182 | struct perf_callchain_entry_ctx ctx; |
568b329a AS |
183 | int rctx; |
184 | ||
9251f904 BP |
185 | entry = get_callchain_entry(&rctx); |
186 | if (rctx == -1) | |
187 | return NULL; | |
188 | ||
189 | if (!entry) | |
190 | goto exit_put; | |
191 | ||
cfbcf468 ACM |
192 | ctx.entry = entry; |
193 | ctx.max_stack = max_stack; | |
3b1fff08 | 194 | ctx.nr = entry->nr = init_nr; |
c85b0334 ACM |
195 | ctx.contexts = 0; |
196 | ctx.contexts_maxed = false; | |
9251f904 | 197 | |
d0775264 | 198 | if (kernel && !user_mode(regs)) { |
568b329a | 199 | if (add_mark) |
3e4de4ec | 200 | perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL); |
cfbcf468 | 201 | perf_callchain_kernel(&ctx, regs); |
9251f904 BP |
202 | } |
203 | ||
d0775264 FW |
204 | if (user) { |
205 | if (!user_mode(regs)) { | |
206 | if (current->mm) | |
207 | regs = task_pt_regs(current); | |
208 | else | |
209 | regs = NULL; | |
210 | } | |
211 | ||
212 | if (regs) { | |
88b0193d WD |
213 | mm_segment_t fs; |
214 | ||
568b329a | 215 | if (crosstask) |
d0775264 FW |
216 | goto exit_put; |
217 | ||
568b329a | 218 | if (add_mark) |
3e4de4ec | 219 | perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); |
88b0193d WD |
220 | |
221 | fs = get_fs(); | |
222 | set_fs(USER_DS); | |
cfbcf468 | 223 | perf_callchain_user(&ctx, regs); |
88b0193d | 224 | set_fs(fs); |
d0775264 | 225 | } |
9251f904 BP |
226 | } |
227 | ||
228 | exit_put: | |
229 | put_callchain_entry(rctx); | |
230 | ||
231 | return entry; | |
232 | } | |
c5dfd78e | 233 | |
c85b0334 ACM |
234 | /* |
235 | * Used for sysctl_perf_event_max_stack and | |
236 | * sysctl_perf_event_max_contexts_per_stack. | |
237 | */ | |
c5dfd78e ACM |
238 | int perf_event_max_stack_handler(struct ctl_table *table, int write, |
239 | void __user *buffer, size_t *lenp, loff_t *ppos) | |
240 | { | |
a831100a ACM |
241 | int *value = table->data; |
242 | int new_value = *value, ret; | |
c5dfd78e ACM |
243 | struct ctl_table new_table = *table; |
244 | ||
245 | new_table.data = &new_value; | |
246 | ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos); | |
247 | if (ret || !write) | |
248 | return ret; | |
249 | ||
250 | mutex_lock(&callchain_mutex); | |
251 | if (atomic_read(&nr_callchain_events)) | |
252 | ret = -EBUSY; | |
253 | else | |
a831100a | 254 | *value = new_value; |
c5dfd78e ACM |
255 | |
256 | mutex_unlock(&callchain_mutex); | |
257 | ||
258 | return ret; | |
259 | } |