]>
Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7520fa99 SP |
2 | /* |
3 | * ARM DynamIQ Shared Unit (DSU) PMU driver | |
4 | * | |
5 | * Copyright (C) ARM Limited, 2017. | |
6 | * | |
7 | * Based on ARM CCI-PMU, ARMv8 PMU-v3 drivers. | |
7520fa99 SP |
8 | */ |
9 | ||
10 | #define PMUNAME "arm_dsu" | |
11 | #define DRVNAME PMUNAME "_pmu" | |
12 | #define pr_fmt(fmt) DRVNAME ": " fmt | |
13 | ||
2b694fc9 | 14 | #include <linux/acpi.h> |
7520fa99 SP |
15 | #include <linux/bitmap.h> |
16 | #include <linux/bitops.h> | |
17 | #include <linux/bug.h> | |
18 | #include <linux/cpumask.h> | |
19 | #include <linux/device.h> | |
20 | #include <linux/interrupt.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/module.h> | |
23 | #include <linux/of_device.h> | |
24 | #include <linux/perf_event.h> | |
25 | #include <linux/platform_device.h> | |
26 | #include <linux/spinlock.h> | |
27 | #include <linux/smp.h> | |
28 | #include <linux/sysfs.h> | |
29 | #include <linux/types.h> | |
30 | ||
31 | #include <asm/arm_dsu_pmu.h> | |
32 | #include <asm/local64.h> | |
33 | ||
34 | /* PMU event codes */ | |
35 | #define DSU_PMU_EVT_CYCLES 0x11 | |
36 | #define DSU_PMU_EVT_CHAIN 0x1e | |
37 | ||
38 | #define DSU_PMU_MAX_COMMON_EVENTS 0x40 | |
39 | ||
40 | #define DSU_PMU_MAX_HW_CNTRS 32 | |
41 | #define DSU_PMU_HW_COUNTER_MASK (DSU_PMU_MAX_HW_CNTRS - 1) | |
42 | ||
43 | #define CLUSTERPMCR_E BIT(0) | |
44 | #define CLUSTERPMCR_P BIT(1) | |
45 | #define CLUSTERPMCR_C BIT(2) | |
46 | #define CLUSTERPMCR_N_SHIFT 11 | |
47 | #define CLUSTERPMCR_N_MASK 0x1f | |
48 | #define CLUSTERPMCR_IDCODE_SHIFT 16 | |
49 | #define CLUSTERPMCR_IDCODE_MASK 0xff | |
50 | #define CLUSTERPMCR_IMP_SHIFT 24 | |
51 | #define CLUSTERPMCR_IMP_MASK 0xff | |
52 | #define CLUSTERPMCR_RES_MASK 0x7e8 | |
53 | #define CLUSTERPMCR_RES_VAL 0x40 | |
54 | ||
55 | #define DSU_ACTIVE_CPU_MASK 0x0 | |
56 | #define DSU_ASSOCIATED_CPU_MASK 0x1 | |
57 | ||
58 | /* | |
59 | * We use the index of the counters as they appear in the counter | |
60 | * bit maps in the PMU registers (e.g CLUSTERPMSELR). | |
61 | * i.e, | |
62 | * counter 0 - Bit 0 | |
63 | * counter 1 - Bit 1 | |
64 | * ... | |
65 | * Cycle counter - Bit 31 | |
66 | */ | |
67 | #define DSU_PMU_IDX_CYCLE_COUNTER 31 | |
68 | ||
69 | /* All event counters are 32bit, with a 64bit Cycle counter */ | |
70 | #define DSU_PMU_COUNTER_WIDTH(idx) \ | |
71 | (((idx) == DSU_PMU_IDX_CYCLE_COUNTER) ? 64 : 32) | |
72 | ||
73 | #define DSU_PMU_COUNTER_MASK(idx) \ | |
74 | GENMASK_ULL((DSU_PMU_COUNTER_WIDTH((idx)) - 1), 0) | |
75 | ||
76 | #define DSU_EXT_ATTR(_name, _func, _config) \ | |
77 | (&((struct dev_ext_attribute[]) { \ | |
78 | { \ | |
79 | .attr = __ATTR(_name, 0444, _func, NULL), \ | |
80 | .var = (void *)_config \ | |
81 | } \ | |
82 | })[0].attr.attr) | |
83 | ||
84 | #define DSU_EVENT_ATTR(_name, _config) \ | |
85 | DSU_EXT_ATTR(_name, dsu_pmu_sysfs_event_show, (unsigned long)_config) | |
86 | ||
87 | #define DSU_FORMAT_ATTR(_name, _config) \ | |
88 | DSU_EXT_ATTR(_name, dsu_pmu_sysfs_format_show, (char *)_config) | |
89 | ||
90 | #define DSU_CPUMASK_ATTR(_name, _config) \ | |
91 | DSU_EXT_ATTR(_name, dsu_pmu_cpumask_show, (unsigned long)_config) | |
92 | ||
93 | struct dsu_hw_events { | |
94 | DECLARE_BITMAP(used_mask, DSU_PMU_MAX_HW_CNTRS); | |
95 | struct perf_event *events[DSU_PMU_MAX_HW_CNTRS]; | |
96 | }; | |
97 | ||
98 | /* | |
99 | * struct dsu_pmu - DSU PMU descriptor | |
100 | * | |
101 | * @pmu_lock : Protects accesses to DSU PMU register from normal vs | |
102 | * interrupt handler contexts. | |
103 | * @hw_events : Holds the event counter state. | |
104 | * @associated_cpus : CPUs attached to the DSU. | |
105 | * @active_cpu : CPU to which the PMU is bound for accesses. | |
106 | * @cpuhp_node : Node for CPU hotplug notifier link. | |
107 | * @num_counters : Number of event counters implemented by the PMU, | |
108 | * excluding the cycle counter. | |
109 | * @irq : Interrupt line for counter overflow. | |
110 | * @cpmceid_bitmap : Bitmap for the availability of architected common | |
111 | * events (event_code < 0x40). | |
112 | */ | |
113 | struct dsu_pmu { | |
114 | struct pmu pmu; | |
115 | struct device *dev; | |
116 | raw_spinlock_t pmu_lock; | |
117 | struct dsu_hw_events hw_events; | |
118 | cpumask_t associated_cpus; | |
119 | cpumask_t active_cpu; | |
120 | struct hlist_node cpuhp_node; | |
a22fde8e | 121 | s8 num_counters; |
7520fa99 SP |
122 | int irq; |
123 | DECLARE_BITMAP(cpmceid_bitmap, DSU_PMU_MAX_COMMON_EVENTS); | |
124 | }; | |
125 | ||
126 | static unsigned long dsu_pmu_cpuhp_state; | |
127 | ||
128 | static inline struct dsu_pmu *to_dsu_pmu(struct pmu *pmu) | |
129 | { | |
130 | return container_of(pmu, struct dsu_pmu, pmu); | |
131 | } | |
132 | ||
133 | static ssize_t dsu_pmu_sysfs_event_show(struct device *dev, | |
134 | struct device_attribute *attr, | |
135 | char *buf) | |
136 | { | |
137 | struct dev_ext_attribute *eattr = container_of(attr, | |
138 | struct dev_ext_attribute, attr); | |
139 | return snprintf(buf, PAGE_SIZE, "event=0x%lx\n", | |
140 | (unsigned long)eattr->var); | |
141 | } | |
142 | ||
143 | static ssize_t dsu_pmu_sysfs_format_show(struct device *dev, | |
144 | struct device_attribute *attr, | |
145 | char *buf) | |
146 | { | |
147 | struct dev_ext_attribute *eattr = container_of(attr, | |
148 | struct dev_ext_attribute, attr); | |
149 | return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var); | |
150 | } | |
151 | ||
152 | static ssize_t dsu_pmu_cpumask_show(struct device *dev, | |
153 | struct device_attribute *attr, | |
154 | char *buf) | |
155 | { | |
156 | struct pmu *pmu = dev_get_drvdata(dev); | |
157 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); | |
158 | struct dev_ext_attribute *eattr = container_of(attr, | |
159 | struct dev_ext_attribute, attr); | |
160 | unsigned long mask_id = (unsigned long)eattr->var; | |
161 | const cpumask_t *cpumask; | |
162 | ||
163 | switch (mask_id) { | |
164 | case DSU_ACTIVE_CPU_MASK: | |
165 | cpumask = &dsu_pmu->active_cpu; | |
166 | break; | |
167 | case DSU_ASSOCIATED_CPU_MASK: | |
168 | cpumask = &dsu_pmu->associated_cpus; | |
169 | break; | |
170 | default: | |
171 | return 0; | |
172 | } | |
173 | return cpumap_print_to_pagebuf(true, buf, cpumask); | |
174 | } | |
175 | ||
176 | static struct attribute *dsu_pmu_format_attrs[] = { | |
177 | DSU_FORMAT_ATTR(event, "config:0-31"), | |
178 | NULL, | |
179 | }; | |
180 | ||
181 | static const struct attribute_group dsu_pmu_format_attr_group = { | |
182 | .name = "format", | |
183 | .attrs = dsu_pmu_format_attrs, | |
184 | }; | |
185 | ||
186 | static struct attribute *dsu_pmu_event_attrs[] = { | |
187 | DSU_EVENT_ATTR(cycles, 0x11), | |
188 | DSU_EVENT_ATTR(bus_access, 0x19), | |
189 | DSU_EVENT_ATTR(memory_error, 0x1a), | |
190 | DSU_EVENT_ATTR(bus_cycles, 0x1d), | |
191 | DSU_EVENT_ATTR(l3d_cache_allocate, 0x29), | |
192 | DSU_EVENT_ATTR(l3d_cache_refill, 0x2a), | |
193 | DSU_EVENT_ATTR(l3d_cache, 0x2b), | |
194 | DSU_EVENT_ATTR(l3d_cache_wb, 0x2c), | |
195 | NULL, | |
196 | }; | |
197 | ||
198 | static umode_t | |
199 | dsu_pmu_event_attr_is_visible(struct kobject *kobj, struct attribute *attr, | |
200 | int unused) | |
201 | { | |
202 | struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj)); | |
203 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); | |
204 | struct dev_ext_attribute *eattr = container_of(attr, | |
205 | struct dev_ext_attribute, attr.attr); | |
206 | unsigned long evt = (unsigned long)eattr->var; | |
207 | ||
208 | return test_bit(evt, dsu_pmu->cpmceid_bitmap) ? attr->mode : 0; | |
209 | } | |
210 | ||
211 | static const struct attribute_group dsu_pmu_events_attr_group = { | |
212 | .name = "events", | |
213 | .attrs = dsu_pmu_event_attrs, | |
214 | .is_visible = dsu_pmu_event_attr_is_visible, | |
215 | }; | |
216 | ||
217 | static struct attribute *dsu_pmu_cpumask_attrs[] = { | |
218 | DSU_CPUMASK_ATTR(cpumask, DSU_ACTIVE_CPU_MASK), | |
219 | DSU_CPUMASK_ATTR(associated_cpus, DSU_ASSOCIATED_CPU_MASK), | |
220 | NULL, | |
221 | }; | |
222 | ||
223 | static const struct attribute_group dsu_pmu_cpumask_attr_group = { | |
224 | .attrs = dsu_pmu_cpumask_attrs, | |
225 | }; | |
226 | ||
227 | static const struct attribute_group *dsu_pmu_attr_groups[] = { | |
228 | &dsu_pmu_cpumask_attr_group, | |
229 | &dsu_pmu_events_attr_group, | |
230 | &dsu_pmu_format_attr_group, | |
231 | NULL, | |
232 | }; | |
233 | ||
234 | static int dsu_pmu_get_online_cpu_any_but(struct dsu_pmu *dsu_pmu, int cpu) | |
235 | { | |
236 | struct cpumask online_supported; | |
237 | ||
238 | cpumask_and(&online_supported, | |
239 | &dsu_pmu->associated_cpus, cpu_online_mask); | |
240 | return cpumask_any_but(&online_supported, cpu); | |
241 | } | |
242 | ||
243 | static inline bool dsu_pmu_counter_valid(struct dsu_pmu *dsu_pmu, u32 idx) | |
244 | { | |
245 | return (idx < dsu_pmu->num_counters) || | |
246 | (idx == DSU_PMU_IDX_CYCLE_COUNTER); | |
247 | } | |
248 | ||
249 | static inline u64 dsu_pmu_read_counter(struct perf_event *event) | |
250 | { | |
251 | u64 val; | |
252 | unsigned long flags; | |
253 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
254 | int idx = event->hw.idx; | |
255 | ||
256 | if (WARN_ON(!cpumask_test_cpu(smp_processor_id(), | |
257 | &dsu_pmu->associated_cpus))) | |
258 | return 0; | |
259 | ||
260 | if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { | |
261 | dev_err(event->pmu->dev, | |
262 | "Trying reading invalid counter %d\n", idx); | |
263 | return 0; | |
264 | } | |
265 | ||
266 | raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); | |
267 | if (idx == DSU_PMU_IDX_CYCLE_COUNTER) | |
268 | val = __dsu_pmu_read_pmccntr(); | |
269 | else | |
270 | val = __dsu_pmu_read_counter(idx); | |
271 | raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); | |
272 | ||
273 | return val; | |
274 | } | |
275 | ||
276 | static void dsu_pmu_write_counter(struct perf_event *event, u64 val) | |
277 | { | |
278 | unsigned long flags; | |
279 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
280 | int idx = event->hw.idx; | |
281 | ||
282 | if (WARN_ON(!cpumask_test_cpu(smp_processor_id(), | |
283 | &dsu_pmu->associated_cpus))) | |
284 | return; | |
285 | ||
286 | if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { | |
287 | dev_err(event->pmu->dev, | |
288 | "writing to invalid counter %d\n", idx); | |
289 | return; | |
290 | } | |
291 | ||
292 | raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); | |
293 | if (idx == DSU_PMU_IDX_CYCLE_COUNTER) | |
294 | __dsu_pmu_write_pmccntr(val); | |
295 | else | |
296 | __dsu_pmu_write_counter(idx, val); | |
297 | raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); | |
298 | } | |
299 | ||
300 | static int dsu_pmu_get_event_idx(struct dsu_hw_events *hw_events, | |
301 | struct perf_event *event) | |
302 | { | |
303 | int idx; | |
304 | unsigned long evtype = event->attr.config; | |
305 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
306 | unsigned long *used_mask = hw_events->used_mask; | |
307 | ||
308 | if (evtype == DSU_PMU_EVT_CYCLES) { | |
309 | if (test_and_set_bit(DSU_PMU_IDX_CYCLE_COUNTER, used_mask)) | |
310 | return -EAGAIN; | |
311 | return DSU_PMU_IDX_CYCLE_COUNTER; | |
312 | } | |
313 | ||
314 | idx = find_first_zero_bit(used_mask, dsu_pmu->num_counters); | |
315 | if (idx >= dsu_pmu->num_counters) | |
316 | return -EAGAIN; | |
317 | set_bit(idx, hw_events->used_mask); | |
318 | return idx; | |
319 | } | |
320 | ||
321 | static void dsu_pmu_enable_counter(struct dsu_pmu *dsu_pmu, int idx) | |
322 | { | |
323 | __dsu_pmu_counter_interrupt_enable(idx); | |
324 | __dsu_pmu_enable_counter(idx); | |
325 | } | |
326 | ||
327 | static void dsu_pmu_disable_counter(struct dsu_pmu *dsu_pmu, int idx) | |
328 | { | |
329 | __dsu_pmu_disable_counter(idx); | |
330 | __dsu_pmu_counter_interrupt_disable(idx); | |
331 | } | |
332 | ||
333 | static inline void dsu_pmu_set_event(struct dsu_pmu *dsu_pmu, | |
334 | struct perf_event *event) | |
335 | { | |
336 | int idx = event->hw.idx; | |
337 | unsigned long flags; | |
338 | ||
339 | if (!dsu_pmu_counter_valid(dsu_pmu, idx)) { | |
340 | dev_err(event->pmu->dev, | |
341 | "Trying to set invalid counter %d\n", idx); | |
342 | return; | |
343 | } | |
344 | ||
345 | raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); | |
346 | __dsu_pmu_set_event(idx, event->hw.config_base); | |
347 | raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); | |
348 | } | |
349 | ||
350 | static void dsu_pmu_event_update(struct perf_event *event) | |
351 | { | |
352 | struct hw_perf_event *hwc = &event->hw; | |
353 | u64 delta, prev_count, new_count; | |
354 | ||
355 | do { | |
356 | /* We may also be called from the irq handler */ | |
357 | prev_count = local64_read(&hwc->prev_count); | |
358 | new_count = dsu_pmu_read_counter(event); | |
359 | } while (local64_cmpxchg(&hwc->prev_count, prev_count, new_count) != | |
360 | prev_count); | |
361 | delta = (new_count - prev_count) & DSU_PMU_COUNTER_MASK(hwc->idx); | |
362 | local64_add(delta, &event->count); | |
363 | } | |
364 | ||
365 | static void dsu_pmu_read(struct perf_event *event) | |
366 | { | |
367 | dsu_pmu_event_update(event); | |
368 | } | |
369 | ||
370 | static inline u32 dsu_pmu_get_reset_overflow(void) | |
371 | { | |
372 | return __dsu_pmu_get_reset_overflow(); | |
373 | } | |
374 | ||
375 | /** | |
376 | * dsu_pmu_set_event_period: Set the period for the counter. | |
377 | * | |
378 | * All DSU PMU event counters, except the cycle counter are 32bit | |
379 | * counters. To handle cases of extreme interrupt latency, we program | |
380 | * the counter with half of the max count for the counters. | |
381 | */ | |
382 | static void dsu_pmu_set_event_period(struct perf_event *event) | |
383 | { | |
384 | int idx = event->hw.idx; | |
385 | u64 val = DSU_PMU_COUNTER_MASK(idx) >> 1; | |
386 | ||
387 | local64_set(&event->hw.prev_count, val); | |
388 | dsu_pmu_write_counter(event, val); | |
389 | } | |
390 | ||
391 | static irqreturn_t dsu_pmu_handle_irq(int irq_num, void *dev) | |
392 | { | |
393 | int i; | |
394 | bool handled = false; | |
395 | struct dsu_pmu *dsu_pmu = dev; | |
396 | struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; | |
397 | unsigned long overflow; | |
398 | ||
399 | overflow = dsu_pmu_get_reset_overflow(); | |
400 | if (!overflow) | |
401 | return IRQ_NONE; | |
402 | ||
403 | for_each_set_bit(i, &overflow, DSU_PMU_MAX_HW_CNTRS) { | |
404 | struct perf_event *event = hw_events->events[i]; | |
405 | ||
406 | if (!event) | |
407 | continue; | |
408 | dsu_pmu_event_update(event); | |
409 | dsu_pmu_set_event_period(event); | |
410 | handled = true; | |
411 | } | |
412 | ||
413 | return IRQ_RETVAL(handled); | |
414 | } | |
415 | ||
416 | static void dsu_pmu_start(struct perf_event *event, int pmu_flags) | |
417 | { | |
418 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
419 | ||
420 | /* We always reprogram the counter */ | |
421 | if (pmu_flags & PERF_EF_RELOAD) | |
422 | WARN_ON(!(event->hw.state & PERF_HES_UPTODATE)); | |
423 | dsu_pmu_set_event_period(event); | |
424 | if (event->hw.idx != DSU_PMU_IDX_CYCLE_COUNTER) | |
425 | dsu_pmu_set_event(dsu_pmu, event); | |
426 | event->hw.state = 0; | |
427 | dsu_pmu_enable_counter(dsu_pmu, event->hw.idx); | |
428 | } | |
429 | ||
430 | static void dsu_pmu_stop(struct perf_event *event, int pmu_flags) | |
431 | { | |
432 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
433 | ||
434 | if (event->hw.state & PERF_HES_STOPPED) | |
435 | return; | |
436 | dsu_pmu_disable_counter(dsu_pmu, event->hw.idx); | |
437 | dsu_pmu_event_update(event); | |
438 | event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
439 | } | |
440 | ||
441 | static int dsu_pmu_add(struct perf_event *event, int flags) | |
442 | { | |
443 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
444 | struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; | |
445 | struct hw_perf_event *hwc = &event->hw; | |
446 | int idx; | |
447 | ||
448 | if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), | |
449 | &dsu_pmu->associated_cpus))) | |
450 | return -ENOENT; | |
451 | ||
452 | idx = dsu_pmu_get_event_idx(hw_events, event); | |
453 | if (idx < 0) | |
454 | return idx; | |
455 | ||
456 | hwc->idx = idx; | |
457 | hw_events->events[idx] = event; | |
458 | hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
459 | ||
460 | if (flags & PERF_EF_START) | |
461 | dsu_pmu_start(event, PERF_EF_RELOAD); | |
462 | ||
463 | perf_event_update_userpage(event); | |
464 | return 0; | |
465 | } | |
466 | ||
467 | static void dsu_pmu_del(struct perf_event *event, int flags) | |
468 | { | |
469 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
470 | struct dsu_hw_events *hw_events = &dsu_pmu->hw_events; | |
471 | struct hw_perf_event *hwc = &event->hw; | |
472 | int idx = hwc->idx; | |
473 | ||
474 | dsu_pmu_stop(event, PERF_EF_UPDATE); | |
475 | hw_events->events[idx] = NULL; | |
476 | clear_bit(idx, hw_events->used_mask); | |
477 | perf_event_update_userpage(event); | |
478 | } | |
479 | ||
480 | static void dsu_pmu_enable(struct pmu *pmu) | |
481 | { | |
482 | u32 pmcr; | |
483 | unsigned long flags; | |
484 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); | |
485 | ||
486 | /* If no counters are added, skip enabling the PMU */ | |
487 | if (bitmap_empty(dsu_pmu->hw_events.used_mask, DSU_PMU_MAX_HW_CNTRS)) | |
488 | return; | |
489 | ||
490 | raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); | |
491 | pmcr = __dsu_pmu_read_pmcr(); | |
492 | pmcr |= CLUSTERPMCR_E; | |
493 | __dsu_pmu_write_pmcr(pmcr); | |
494 | raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); | |
495 | } | |
496 | ||
497 | static void dsu_pmu_disable(struct pmu *pmu) | |
498 | { | |
499 | u32 pmcr; | |
500 | unsigned long flags; | |
501 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(pmu); | |
502 | ||
503 | raw_spin_lock_irqsave(&dsu_pmu->pmu_lock, flags); | |
504 | pmcr = __dsu_pmu_read_pmcr(); | |
505 | pmcr &= ~CLUSTERPMCR_E; | |
506 | __dsu_pmu_write_pmcr(pmcr); | |
507 | raw_spin_unlock_irqrestore(&dsu_pmu->pmu_lock, flags); | |
508 | } | |
509 | ||
510 | static bool dsu_pmu_validate_event(struct pmu *pmu, | |
511 | struct dsu_hw_events *hw_events, | |
512 | struct perf_event *event) | |
513 | { | |
514 | if (is_software_event(event)) | |
515 | return true; | |
516 | /* Reject groups spanning multiple HW PMUs. */ | |
517 | if (event->pmu != pmu) | |
518 | return false; | |
519 | return dsu_pmu_get_event_idx(hw_events, event) >= 0; | |
520 | } | |
521 | ||
522 | /* | |
523 | * Make sure the group of events can be scheduled at once | |
524 | * on the PMU. | |
525 | */ | |
526 | static bool dsu_pmu_validate_group(struct perf_event *event) | |
527 | { | |
528 | struct perf_event *sibling, *leader = event->group_leader; | |
529 | struct dsu_hw_events fake_hw; | |
530 | ||
531 | if (event->group_leader == event) | |
532 | return true; | |
533 | ||
534 | memset(fake_hw.used_mask, 0, sizeof(fake_hw.used_mask)); | |
535 | if (!dsu_pmu_validate_event(event->pmu, &fake_hw, leader)) | |
536 | return false; | |
edb39592 | 537 | for_each_sibling_event(sibling, leader) { |
7520fa99 SP |
538 | if (!dsu_pmu_validate_event(event->pmu, &fake_hw, sibling)) |
539 | return false; | |
540 | } | |
541 | return dsu_pmu_validate_event(event->pmu, &fake_hw, event); | |
542 | } | |
543 | ||
544 | static int dsu_pmu_event_init(struct perf_event *event) | |
545 | { | |
546 | struct dsu_pmu *dsu_pmu = to_dsu_pmu(event->pmu); | |
547 | ||
548 | if (event->attr.type != event->pmu->type) | |
549 | return -ENOENT; | |
550 | ||
551 | /* We don't support sampling */ | |
552 | if (is_sampling_event(event)) { | |
553 | dev_dbg(dsu_pmu->pmu.dev, "Can't support sampling events\n"); | |
554 | return -EOPNOTSUPP; | |
555 | } | |
556 | ||
557 | /* We cannot support task bound events */ | |
558 | if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) { | |
559 | dev_dbg(dsu_pmu->pmu.dev, "Can't support per-task counters\n"); | |
560 | return -EINVAL; | |
561 | } | |
562 | ||
30656398 | 563 | if (has_branch_stack(event)) { |
7520fa99 SP |
564 | dev_dbg(dsu_pmu->pmu.dev, "Can't support filtering\n"); |
565 | return -EINVAL; | |
566 | } | |
567 | ||
568 | if (!cpumask_test_cpu(event->cpu, &dsu_pmu->associated_cpus)) { | |
569 | dev_dbg(dsu_pmu->pmu.dev, | |
570 | "Requested cpu is not associated with the DSU\n"); | |
571 | return -EINVAL; | |
572 | } | |
573 | /* | |
574 | * Choose the current active CPU to read the events. We don't want | |
575 | * to migrate the event contexts, irq handling etc to the requested | |
576 | * CPU. As long as the requested CPU is within the same DSU, we | |
577 | * are fine. | |
578 | */ | |
579 | event->cpu = cpumask_first(&dsu_pmu->active_cpu); | |
580 | if (event->cpu >= nr_cpu_ids) | |
581 | return -EINVAL; | |
582 | if (!dsu_pmu_validate_group(event)) | |
583 | return -EINVAL; | |
584 | ||
585 | event->hw.config_base = event->attr.config; | |
586 | return 0; | |
587 | } | |
588 | ||
589 | static struct dsu_pmu *dsu_pmu_alloc(struct platform_device *pdev) | |
590 | { | |
591 | struct dsu_pmu *dsu_pmu; | |
592 | ||
593 | dsu_pmu = devm_kzalloc(&pdev->dev, sizeof(*dsu_pmu), GFP_KERNEL); | |
594 | if (!dsu_pmu) | |
595 | return ERR_PTR(-ENOMEM); | |
596 | ||
597 | raw_spin_lock_init(&dsu_pmu->pmu_lock); | |
598 | /* | |
599 | * Initialise the number of counters to -1, until we probe | |
600 | * the real number on a connected CPU. | |
601 | */ | |
602 | dsu_pmu->num_counters = -1; | |
603 | return dsu_pmu; | |
604 | } | |
605 | ||
606 | /** | |
2b694fc9 TP |
607 | * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster |
608 | * from device tree. | |
7520fa99 | 609 | */ |
2b694fc9 | 610 | static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask) |
7520fa99 SP |
611 | { |
612 | int i = 0, n, cpu; | |
613 | struct device_node *cpu_node; | |
614 | ||
2b694fc9 | 615 | n = of_count_phandle_with_args(dev->of_node, "cpus", NULL); |
7520fa99 SP |
616 | if (n <= 0) |
617 | return -ENODEV; | |
618 | for (; i < n; i++) { | |
2b694fc9 | 619 | cpu_node = of_parse_phandle(dev->of_node, "cpus", i); |
7520fa99 SP |
620 | if (!cpu_node) |
621 | break; | |
622 | cpu = of_cpu_node_to_id(cpu_node); | |
623 | of_node_put(cpu_node); | |
624 | /* | |
625 | * We have to ignore the failures here and continue scanning | |
626 | * the list to handle cases where the nr_cpus could be capped | |
627 | * in the running kernel. | |
628 | */ | |
629 | if (cpu < 0) | |
630 | continue; | |
631 | cpumask_set_cpu(cpu, mask); | |
632 | } | |
633 | return 0; | |
634 | } | |
635 | ||
2b694fc9 TP |
636 | /** |
637 | * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster | |
638 | * from ACPI. | |
639 | */ | |
640 | static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask) | |
641 | { | |
642 | #ifdef CONFIG_ACPI | |
643 | int cpu; | |
644 | ||
645 | /* | |
646 | * A dsu pmu node is inside a cluster parent node along with cpu nodes. | |
647 | * We need to find out all cpus that have the same parent with this pmu. | |
648 | */ | |
649 | for_each_possible_cpu(cpu) { | |
650 | struct acpi_device *acpi_dev; | |
651 | struct device *cpu_dev = get_cpu_device(cpu); | |
652 | ||
653 | if (!cpu_dev) | |
654 | continue; | |
655 | ||
656 | acpi_dev = ACPI_COMPANION(cpu_dev); | |
657 | if (acpi_dev && | |
658 | acpi_dev->parent == ACPI_COMPANION(dev)->parent) | |
659 | cpumask_set_cpu(cpu, mask); | |
660 | } | |
661 | #endif | |
662 | ||
663 | return 0; | |
664 | } | |
665 | ||
7520fa99 SP |
666 | /* |
667 | * dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster. | |
668 | */ | |
669 | static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu) | |
670 | { | |
671 | u64 num_counters; | |
672 | u32 cpmceid[2]; | |
673 | ||
674 | num_counters = (__dsu_pmu_read_pmcr() >> CLUSTERPMCR_N_SHIFT) & | |
675 | CLUSTERPMCR_N_MASK; | |
676 | /* We can only support up to 31 independent counters */ | |
677 | if (WARN_ON(num_counters > 31)) | |
678 | num_counters = 31; | |
679 | dsu_pmu->num_counters = num_counters; | |
680 | if (!dsu_pmu->num_counters) | |
681 | return; | |
682 | cpmceid[0] = __dsu_pmu_read_pmceid(0); | |
683 | cpmceid[1] = __dsu_pmu_read_pmceid(1); | |
3aa56885 YN |
684 | bitmap_from_arr32(dsu_pmu->cpmceid_bitmap, cpmceid, |
685 | DSU_PMU_MAX_COMMON_EVENTS); | |
7520fa99 SP |
686 | } |
687 | ||
688 | static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu) | |
689 | { | |
690 | cpumask_set_cpu(cpu, &dsu_pmu->active_cpu); | |
691 | if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu)) | |
692 | pr_warn("Failed to set irq affinity to %d\n", cpu); | |
693 | } | |
694 | ||
695 | /* | |
696 | * dsu_pmu_init_pmu: Initialise the DSU PMU configurations if | |
697 | * we haven't done it already. | |
698 | */ | |
699 | static void dsu_pmu_init_pmu(struct dsu_pmu *dsu_pmu) | |
700 | { | |
701 | if (dsu_pmu->num_counters == -1) | |
702 | dsu_pmu_probe_pmu(dsu_pmu); | |
703 | /* Reset the interrupt overflow mask */ | |
704 | dsu_pmu_get_reset_overflow(); | |
705 | } | |
706 | ||
707 | static int dsu_pmu_device_probe(struct platform_device *pdev) | |
708 | { | |
709 | int irq, rc; | |
710 | struct dsu_pmu *dsu_pmu; | |
2b694fc9 | 711 | struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev); |
7520fa99 SP |
712 | char *name; |
713 | static atomic_t pmu_idx = ATOMIC_INIT(-1); | |
714 | ||
715 | dsu_pmu = dsu_pmu_alloc(pdev); | |
716 | if (IS_ERR(dsu_pmu)) | |
717 | return PTR_ERR(dsu_pmu); | |
718 | ||
2b694fc9 TP |
719 | if (is_of_node(fwnode)) |
720 | rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus); | |
721 | else if (is_acpi_device_node(fwnode)) | |
722 | rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus); | |
723 | else | |
724 | return -ENOENT; | |
725 | ||
7520fa99 SP |
726 | if (rc) { |
727 | dev_warn(&pdev->dev, "Failed to parse the CPUs\n"); | |
728 | return rc; | |
729 | } | |
730 | ||
731 | irq = platform_get_irq(pdev, 0); | |
5810f00a | 732 | if (irq < 0) |
7520fa99 | 733 | return -EINVAL; |
7520fa99 SP |
734 | |
735 | name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s_%d", | |
736 | PMUNAME, atomic_inc_return(&pmu_idx)); | |
737 | if (!name) | |
738 | return -ENOMEM; | |
739 | rc = devm_request_irq(&pdev->dev, irq, dsu_pmu_handle_irq, | |
740 | IRQF_NOBALANCING, name, dsu_pmu); | |
741 | if (rc) { | |
742 | dev_warn(&pdev->dev, "Failed to request IRQ %d\n", irq); | |
743 | return rc; | |
744 | } | |
745 | ||
746 | dsu_pmu->irq = irq; | |
747 | platform_set_drvdata(pdev, dsu_pmu); | |
748 | rc = cpuhp_state_add_instance(dsu_pmu_cpuhp_state, | |
749 | &dsu_pmu->cpuhp_node); | |
750 | if (rc) | |
751 | return rc; | |
752 | ||
753 | dsu_pmu->pmu = (struct pmu) { | |
754 | .task_ctx_nr = perf_invalid_context, | |
755 | .module = THIS_MODULE, | |
756 | .pmu_enable = dsu_pmu_enable, | |
757 | .pmu_disable = dsu_pmu_disable, | |
758 | .event_init = dsu_pmu_event_init, | |
759 | .add = dsu_pmu_add, | |
760 | .del = dsu_pmu_del, | |
761 | .start = dsu_pmu_start, | |
762 | .stop = dsu_pmu_stop, | |
763 | .read = dsu_pmu_read, | |
764 | ||
765 | .attr_groups = dsu_pmu_attr_groups, | |
30656398 | 766 | .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
7520fa99 SP |
767 | }; |
768 | ||
769 | rc = perf_pmu_register(&dsu_pmu->pmu, name, -1); | |
770 | if (rc) { | |
771 | cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, | |
772 | &dsu_pmu->cpuhp_node); | |
773 | irq_set_affinity_hint(dsu_pmu->irq, NULL); | |
774 | } | |
775 | ||
776 | return rc; | |
777 | } | |
778 | ||
779 | static int dsu_pmu_device_remove(struct platform_device *pdev) | |
780 | { | |
781 | struct dsu_pmu *dsu_pmu = platform_get_drvdata(pdev); | |
782 | ||
783 | perf_pmu_unregister(&dsu_pmu->pmu); | |
784 | cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); | |
785 | irq_set_affinity_hint(dsu_pmu->irq, NULL); | |
786 | ||
787 | return 0; | |
788 | } | |
789 | ||
790 | static const struct of_device_id dsu_pmu_of_match[] = { | |
791 | { .compatible = "arm,dsu-pmu", }, | |
792 | {}, | |
793 | }; | |
2b694fc9 TP |
794 | MODULE_DEVICE_TABLE(of, dsu_pmu_of_match); |
795 | ||
796 | #ifdef CONFIG_ACPI | |
797 | static const struct acpi_device_id dsu_pmu_acpi_match[] = { | |
798 | { "ARMHD500", 0}, | |
799 | {}, | |
800 | }; | |
801 | MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match); | |
802 | #endif | |
7520fa99 SP |
803 | |
804 | static struct platform_driver dsu_pmu_driver = { | |
805 | .driver = { | |
806 | .name = DRVNAME, | |
807 | .of_match_table = of_match_ptr(dsu_pmu_of_match), | |
2b694fc9 | 808 | .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match), |
f32ed8eb | 809 | .suppress_bind_attrs = true, |
7520fa99 SP |
810 | }, |
811 | .probe = dsu_pmu_device_probe, | |
812 | .remove = dsu_pmu_device_remove, | |
813 | }; | |
814 | ||
815 | static int dsu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) | |
816 | { | |
817 | struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, | |
818 | cpuhp_node); | |
819 | ||
820 | if (!cpumask_test_cpu(cpu, &dsu_pmu->associated_cpus)) | |
821 | return 0; | |
822 | ||
823 | /* If the PMU is already managed, there is nothing to do */ | |
824 | if (!cpumask_empty(&dsu_pmu->active_cpu)) | |
825 | return 0; | |
826 | ||
827 | dsu_pmu_init_pmu(dsu_pmu); | |
828 | dsu_pmu_set_active_cpu(cpu, dsu_pmu); | |
829 | ||
830 | return 0; | |
831 | } | |
832 | ||
833 | static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) | |
834 | { | |
835 | int dst; | |
836 | struct dsu_pmu *dsu_pmu = hlist_entry_safe(node, struct dsu_pmu, | |
837 | cpuhp_node); | |
838 | ||
839 | if (!cpumask_test_and_clear_cpu(cpu, &dsu_pmu->active_cpu)) | |
840 | return 0; | |
841 | ||
842 | dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu); | |
843 | /* If there are no active CPUs in the DSU, leave IRQ disabled */ | |
844 | if (dst >= nr_cpu_ids) { | |
845 | irq_set_affinity_hint(dsu_pmu->irq, NULL); | |
846 | return 0; | |
847 | } | |
848 | ||
849 | perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst); | |
850 | dsu_pmu_set_active_cpu(dst, dsu_pmu); | |
851 | ||
852 | return 0; | |
853 | } | |
854 | ||
855 | static int __init dsu_pmu_init(void) | |
856 | { | |
857 | int ret; | |
858 | ||
859 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, | |
860 | DRVNAME, | |
861 | dsu_pmu_cpu_online, | |
862 | dsu_pmu_cpu_teardown); | |
863 | if (ret < 0) | |
864 | return ret; | |
865 | dsu_pmu_cpuhp_state = ret; | |
866 | return platform_driver_register(&dsu_pmu_driver); | |
867 | } | |
868 | ||
869 | static void __exit dsu_pmu_exit(void) | |
870 | { | |
871 | platform_driver_unregister(&dsu_pmu_driver); | |
872 | cpuhp_remove_multi_state(dsu_pmu_cpuhp_state); | |
873 | } | |
874 | ||
875 | module_init(dsu_pmu_init); | |
876 | module_exit(dsu_pmu_exit); | |
877 | ||
7520fa99 SP |
878 | MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit"); |
879 | MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>"); | |
880 | MODULE_LICENSE("GPL v2"); |