2 * Performance events callchain code, extracted from core.c:
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
12 #include <linux/perf_event.h>
13 #include <linux/slab.h>
16 struct callchain_cpus_entries
{
17 struct rcu_head rcu_head
;
18 struct perf_callchain_entry
*cpu_entries
[0];
21 int sysctl_perf_event_max_stack __read_mostly
= PERF_MAX_STACK_DEPTH
;
23 static inline size_t perf_callchain_entry__sizeof(void)
25 return (sizeof(struct perf_callchain_entry
) +
26 sizeof(__u64
) * sysctl_perf_event_max_stack
);
29 static DEFINE_PER_CPU(int, callchain_recursion
[PERF_NR_CONTEXTS
]);
30 static atomic_t nr_callchain_events
;
31 static DEFINE_MUTEX(callchain_mutex
);
32 static struct callchain_cpus_entries
*callchain_cpus_entries
;
35 __weak
void perf_callchain_kernel(struct perf_callchain_entry
*entry
,
40 __weak
void perf_callchain_user(struct perf_callchain_entry
*entry
,
45 static void release_callchain_buffers_rcu(struct rcu_head
*head
)
47 struct callchain_cpus_entries
*entries
;
50 entries
= container_of(head
, struct callchain_cpus_entries
, rcu_head
);
52 for_each_possible_cpu(cpu
)
53 kfree(entries
->cpu_entries
[cpu
]);
58 static void release_callchain_buffers(void)
60 struct callchain_cpus_entries
*entries
;
62 entries
= callchain_cpus_entries
;
63 RCU_INIT_POINTER(callchain_cpus_entries
, NULL
);
64 call_rcu(&entries
->rcu_head
, release_callchain_buffers_rcu
);
67 static int alloc_callchain_buffers(void)
71 struct callchain_cpus_entries
*entries
;
74 * We can't use the percpu allocation API for data that can be
75 * accessed from NMI. Use a temporary manual per cpu allocation
76 * until that gets sorted out.
78 size
= offsetof(struct callchain_cpus_entries
, cpu_entries
[nr_cpu_ids
]);
80 entries
= kzalloc(size
, GFP_KERNEL
);
84 size
= perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS
;
86 for_each_possible_cpu(cpu
) {
87 entries
->cpu_entries
[cpu
] = kmalloc_node(size
, GFP_KERNEL
,
89 if (!entries
->cpu_entries
[cpu
])
93 rcu_assign_pointer(callchain_cpus_entries
, entries
);
98 for_each_possible_cpu(cpu
)
99 kfree(entries
->cpu_entries
[cpu
]);
105 int get_callchain_buffers(void)
110 mutex_lock(&callchain_mutex
);
112 count
= atomic_inc_return(&nr_callchain_events
);
113 if (WARN_ON_ONCE(count
< 1)) {
119 /* If the allocation failed, give up */
120 if (!callchain_cpus_entries
)
125 err
= alloc_callchain_buffers();
128 atomic_dec(&nr_callchain_events
);
130 mutex_unlock(&callchain_mutex
);
135 void put_callchain_buffers(void)
137 if (atomic_dec_and_mutex_lock(&nr_callchain_events
, &callchain_mutex
)) {
138 release_callchain_buffers();
139 mutex_unlock(&callchain_mutex
);
143 static struct perf_callchain_entry
*get_callchain_entry(int *rctx
)
146 struct callchain_cpus_entries
*entries
;
148 *rctx
= get_recursion_context(this_cpu_ptr(callchain_recursion
));
152 entries
= rcu_dereference(callchain_cpus_entries
);
156 cpu
= smp_processor_id();
158 return (((void *)entries
->cpu_entries
[cpu
]) +
159 (*rctx
* perf_callchain_entry__sizeof()));
163 put_callchain_entry(int rctx
)
165 put_recursion_context(this_cpu_ptr(callchain_recursion
), rctx
);
168 struct perf_callchain_entry
*
169 perf_callchain(struct perf_event
*event
, struct pt_regs
*regs
)
171 bool kernel
= !event
->attr
.exclude_callchain_kernel
;
172 bool user
= !event
->attr
.exclude_callchain_user
;
173 /* Disallow cross-task user callchains. */
174 bool crosstask
= event
->ctx
->task
&& event
->ctx
->task
!= current
;
176 if (!kernel
&& !user
)
179 return get_perf_callchain(regs
, 0, kernel
, user
, crosstask
, true);
182 struct perf_callchain_entry
*
183 get_perf_callchain(struct pt_regs
*regs
, u32 init_nr
, bool kernel
, bool user
,
184 bool crosstask
, bool add_mark
)
186 struct perf_callchain_entry
*entry
;
189 entry
= get_callchain_entry(&rctx
);
198 if (kernel
&& !user_mode(regs
)) {
200 perf_callchain_store(entry
, PERF_CONTEXT_KERNEL
);
201 perf_callchain_kernel(entry
, regs
);
205 if (!user_mode(regs
)) {
207 regs
= task_pt_regs(current
);
217 perf_callchain_store(entry
, PERF_CONTEXT_USER
);
218 perf_callchain_user(entry
, regs
);
223 put_callchain_entry(rctx
);
228 int perf_event_max_stack_handler(struct ctl_table
*table
, int write
,
229 void __user
*buffer
, size_t *lenp
, loff_t
*ppos
)
231 int new_value
= sysctl_perf_event_max_stack
, ret
;
232 struct ctl_table new_table
= *table
;
234 new_table
.data
= &new_value
;
235 ret
= proc_dointvec_minmax(&new_table
, write
, buffer
, lenp
, ppos
);
239 mutex_lock(&callchain_mutex
);
240 if (atomic_read(&nr_callchain_events
))
243 sysctl_perf_event_max_stack
= new_value
;
245 mutex_unlock(&callchain_mutex
);