1 // SPDX-License-Identifier: GPL-2.0-only
3 * ARM DMC-620 memory controller PMU driver
5 * Copyright (C) 2020 Ampere Computing LLC.
8 #define DMC620_PMUNAME "arm_dmc620"
9 #define DMC620_DRVNAME DMC620_PMUNAME "_pmu"
10 #define pr_fmt(fmt) DMC620_DRVNAME ": " fmt
12 #include <linux/acpi.h>
13 #include <linux/bitfield.h>
14 #include <linux/bitops.h>
15 #include <linux/cpuhotplug.h>
16 #include <linux/cpumask.h>
17 #include <linux/device.h>
18 #include <linux/errno.h>
19 #include <linux/interrupt.h>
20 #include <linux/irq.h>
21 #include <linux/kernel.h>
22 #include <linux/list.h>
23 #include <linux/module.h>
24 #include <linux/mutex.h>
25 #include <linux/perf_event.h>
26 #include <linux/platform_device.h>
27 #include <linux/printk.h>
28 #include <linux/rculist.h>
29 #include <linux/refcount.h>
31 #define DMC620_PA_SHIFT 12
32 #define DMC620_CNT_INIT 0x80000000
33 #define DMC620_CNT_MAX_PERIOD 0xffffffff
34 #define DMC620_PMU_CLKDIV2_MAX_COUNTERS 8
35 #define DMC620_PMU_CLK_MAX_COUNTERS 2
36 #define DMC620_PMU_MAX_COUNTERS \
37 (DMC620_PMU_CLKDIV2_MAX_COUNTERS + DMC620_PMU_CLK_MAX_COUNTERS)
40 * The PMU registers start at 0xA00 in the DMC-620 memory map, and these
41 * offsets are relative to that base.
43 * Each counter has a group of control/value registers, and the
44 * DMC620_PMU_COUNTERn offsets are within a counter group.
46 * The counter registers groups start at 0xA10.
48 #define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2 0x8
49 #define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK \
50 (DMC620_PMU_CLKDIV2_MAX_COUNTERS - 1)
51 #define DMC620_PMU_OVERFLOW_STATUS_CLK 0xC
52 #define DMC620_PMU_OVERFLOW_STATUS_CLK_MASK \
53 (DMC620_PMU_CLK_MAX_COUNTERS - 1)
54 #define DMC620_PMU_COUNTERS_BASE 0x10
55 #define DMC620_PMU_COUNTERn_MASK_31_00 0x0
56 #define DMC620_PMU_COUNTERn_MASK_63_32 0x4
57 #define DMC620_PMU_COUNTERn_MATCH_31_00 0x8
58 #define DMC620_PMU_COUNTERn_MATCH_63_32 0xC
59 #define DMC620_PMU_COUNTERn_CONTROL 0x10
60 #define DMC620_PMU_COUNTERn_CONTROL_ENABLE BIT(0)
61 #define DMC620_PMU_COUNTERn_CONTROL_INVERT BIT(1)
62 #define DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX GENMASK(6, 2)
63 #define DMC620_PMU_COUNTERn_CONTROL_INCR_MUX GENMASK(8, 7)
64 #define DMC620_PMU_COUNTERn_VALUE 0x20
65 /* Offset of the registers for a given counter, relative to 0xA00 */
66 #define DMC620_PMU_COUNTERn_OFFSET(n) \
67 (DMC620_PMU_COUNTERS_BASE + 0x28 * (n))
69 static LIST_HEAD(dmc620_pmu_irqs
);
70 static DEFINE_MUTEX(dmc620_pmu_irqs_lock
);
72 struct dmc620_pmu_irq
{
73 struct hlist_node node
;
74 struct list_head pmus_node
;
75 struct list_head irqs_node
;
85 struct dmc620_pmu_irq
*irq
;
86 struct list_head pmus_node
;
89 * We put all clkdiv2 and clk counters to a same array.
90 * The first DMC620_PMU_CLKDIV2_MAX_COUNTERS bits belong to
91 * clkdiv2 counters, the last DMC620_PMU_CLK_MAX_COUNTERS
92 * belong to clk counters.
94 DECLARE_BITMAP(used_mask
, DMC620_PMU_MAX_COUNTERS
);
95 struct perf_event
*events
[DMC620_PMU_MAX_COUNTERS
];
98 #define to_dmc620_pmu(p) (container_of(p, struct dmc620_pmu, pmu))
100 static int cpuhp_state_num
;
102 struct dmc620_pmu_event_attr
{
103 struct device_attribute attr
;
109 dmc620_pmu_event_show(struct device
*dev
,
110 struct device_attribute
*attr
, char *page
)
112 struct dmc620_pmu_event_attr
*eattr
;
114 eattr
= container_of(attr
, typeof(*eattr
), attr
);
116 return sprintf(page
, "event=0x%x,clkdiv2=0x%x\n", eattr
->eventid
, eattr
->clkdiv2
);
119 #define DMC620_PMU_EVENT_ATTR(_name, _eventid, _clkdiv2) \
120 (&((struct dmc620_pmu_event_attr[]) {{ \
121 .attr = __ATTR(_name, 0444, dmc620_pmu_event_show, NULL), \
122 .clkdiv2 = _clkdiv2, \
123 .eventid = _eventid, \
126 static struct attribute
*dmc620_pmu_events_attrs
[] = {
127 /* clkdiv2 events list */
128 DMC620_PMU_EVENT_ATTR(clkdiv2_cycle_count
, 0x0, 1),
129 DMC620_PMU_EVENT_ATTR(clkdiv2_allocate
, 0x1, 1),
130 DMC620_PMU_EVENT_ATTR(clkdiv2_queue_depth
, 0x2, 1),
131 DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_wr_data
, 0x3, 1),
132 DMC620_PMU_EVENT_ATTR(clkdiv2_read_backlog
, 0x4, 1),
133 DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_mi
, 0x5, 1),
134 DMC620_PMU_EVENT_ATTR(clkdiv2_hazard_resolution
, 0x6, 1),
135 DMC620_PMU_EVENT_ATTR(clkdiv2_enqueue
, 0x7, 1),
136 DMC620_PMU_EVENT_ATTR(clkdiv2_arbitrate
, 0x8, 1),
137 DMC620_PMU_EVENT_ATTR(clkdiv2_lrank_turnaround_activate
, 0x9, 1),
138 DMC620_PMU_EVENT_ATTR(clkdiv2_prank_turnaround_activate
, 0xa, 1),
139 DMC620_PMU_EVENT_ATTR(clkdiv2_read_depth
, 0xb, 1),
140 DMC620_PMU_EVENT_ATTR(clkdiv2_write_depth
, 0xc, 1),
141 DMC620_PMU_EVENT_ATTR(clkdiv2_highigh_qos_depth
, 0xd, 1),
142 DMC620_PMU_EVENT_ATTR(clkdiv2_high_qos_depth
, 0xe, 1),
143 DMC620_PMU_EVENT_ATTR(clkdiv2_medium_qos_depth
, 0xf, 1),
144 DMC620_PMU_EVENT_ATTR(clkdiv2_low_qos_depth
, 0x10, 1),
145 DMC620_PMU_EVENT_ATTR(clkdiv2_activate
, 0x11, 1),
146 DMC620_PMU_EVENT_ATTR(clkdiv2_rdwr
, 0x12, 1),
147 DMC620_PMU_EVENT_ATTR(clkdiv2_refresh
, 0x13, 1),
148 DMC620_PMU_EVENT_ATTR(clkdiv2_training_request
, 0x14, 1),
149 DMC620_PMU_EVENT_ATTR(clkdiv2_t_mac_tracker
, 0x15, 1),
150 DMC620_PMU_EVENT_ATTR(clkdiv2_bk_fsm_tracker
, 0x16, 1),
151 DMC620_PMU_EVENT_ATTR(clkdiv2_bk_open_tracker
, 0x17, 1),
152 DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_pwr_down
, 0x18, 1),
153 DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_sref
, 0x19, 1),
155 /* clk events list */
156 DMC620_PMU_EVENT_ATTR(clk_cycle_count
, 0x0, 0),
157 DMC620_PMU_EVENT_ATTR(clk_request
, 0x1, 0),
158 DMC620_PMU_EVENT_ATTR(clk_upload_stall
, 0x2, 0),
162 static const struct attribute_group dmc620_pmu_events_attr_group
= {
164 .attrs
= dmc620_pmu_events_attrs
,
168 #define ATTR_CFG_FLD_mask_CFG config
169 #define ATTR_CFG_FLD_mask_LO 0
170 #define ATTR_CFG_FLD_mask_HI 44
171 #define ATTR_CFG_FLD_match_CFG config1
172 #define ATTR_CFG_FLD_match_LO 0
173 #define ATTR_CFG_FLD_match_HI 44
174 #define ATTR_CFG_FLD_invert_CFG config2
175 #define ATTR_CFG_FLD_invert_LO 0
176 #define ATTR_CFG_FLD_invert_HI 0
177 #define ATTR_CFG_FLD_incr_CFG config2
178 #define ATTR_CFG_FLD_incr_LO 1
179 #define ATTR_CFG_FLD_incr_HI 2
180 #define ATTR_CFG_FLD_event_CFG config2
181 #define ATTR_CFG_FLD_event_LO 3
182 #define ATTR_CFG_FLD_event_HI 8
183 #define ATTR_CFG_FLD_clkdiv2_CFG config2
184 #define ATTR_CFG_FLD_clkdiv2_LO 9
185 #define ATTR_CFG_FLD_clkdiv2_HI 9
187 #define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
188 (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
190 #define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
191 __GEN_PMU_FORMAT_ATTR(cfg, lo, hi)
193 #define GEN_PMU_FORMAT_ATTR(name) \
194 PMU_FORMAT_ATTR(name, \
195 _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \
196 ATTR_CFG_FLD_##name##_LO, \
197 ATTR_CFG_FLD_##name##_HI))
199 #define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \
200 ((((attr)->cfg) >> lo) & GENMASK_ULL(hi - lo, 0))
202 #define ATTR_CFG_GET_FLD(attr, name) \
203 _ATTR_CFG_GET_FLD(attr, \
204 ATTR_CFG_FLD_##name##_CFG, \
205 ATTR_CFG_FLD_##name##_LO, \
206 ATTR_CFG_FLD_##name##_HI)
208 GEN_PMU_FORMAT_ATTR(mask
);
209 GEN_PMU_FORMAT_ATTR(match
);
210 GEN_PMU_FORMAT_ATTR(invert
);
211 GEN_PMU_FORMAT_ATTR(incr
);
212 GEN_PMU_FORMAT_ATTR(event
);
213 GEN_PMU_FORMAT_ATTR(clkdiv2
);
215 static struct attribute
*dmc620_pmu_formats_attrs
[] = {
216 &format_attr_mask
.attr
,
217 &format_attr_match
.attr
,
218 &format_attr_invert
.attr
,
219 &format_attr_incr
.attr
,
220 &format_attr_event
.attr
,
221 &format_attr_clkdiv2
.attr
,
225 static const struct attribute_group dmc620_pmu_format_attr_group
= {
227 .attrs
= dmc620_pmu_formats_attrs
,
230 static const struct attribute_group
*dmc620_pmu_attr_groups
[] = {
231 &dmc620_pmu_events_attr_group
,
232 &dmc620_pmu_format_attr_group
,
237 u32
dmc620_pmu_creg_read(struct dmc620_pmu
*dmc620_pmu
,
238 unsigned int idx
, unsigned int reg
)
240 return readl(dmc620_pmu
->base
+ DMC620_PMU_COUNTERn_OFFSET(idx
) + reg
);
244 void dmc620_pmu_creg_write(struct dmc620_pmu
*dmc620_pmu
,
245 unsigned int idx
, unsigned int reg
, u32 val
)
247 writel(val
, dmc620_pmu
->base
+ DMC620_PMU_COUNTERn_OFFSET(idx
) + reg
);
251 unsigned int dmc620_event_to_counter_control(struct perf_event
*event
)
253 struct perf_event_attr
*attr
= &event
->attr
;
254 unsigned int reg
= 0;
256 reg
|= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INVERT
,
257 ATTR_CFG_GET_FLD(attr
, invert
));
258 reg
|= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX
,
259 ATTR_CFG_GET_FLD(attr
, event
));
260 reg
|= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INCR_MUX
,
261 ATTR_CFG_GET_FLD(attr
, incr
));
266 static int dmc620_get_event_idx(struct perf_event
*event
)
268 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
269 int idx
, start_idx
, end_idx
;
271 if (ATTR_CFG_GET_FLD(&event
->attr
, clkdiv2
)) {
273 end_idx
= DMC620_PMU_CLKDIV2_MAX_COUNTERS
;
275 start_idx
= DMC620_PMU_CLKDIV2_MAX_COUNTERS
;
276 end_idx
= DMC620_PMU_MAX_COUNTERS
;
279 for (idx
= start_idx
; idx
< end_idx
; ++idx
) {
280 if (!test_and_set_bit(idx
, dmc620_pmu
->used_mask
))
284 /* The counters are all in use. */
289 u64
dmc620_pmu_read_counter(struct perf_event
*event
)
291 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
293 return dmc620_pmu_creg_read(dmc620_pmu
,
294 event
->hw
.idx
, DMC620_PMU_COUNTERn_VALUE
);
297 static void dmc620_pmu_event_update(struct perf_event
*event
)
299 struct hw_perf_event
*hwc
= &event
->hw
;
300 u64 delta
, prev_count
, new_count
;
303 /* We may also be called from the irq handler */
304 prev_count
= local64_read(&hwc
->prev_count
);
305 new_count
= dmc620_pmu_read_counter(event
);
306 } while (local64_cmpxchg(&hwc
->prev_count
,
307 prev_count
, new_count
) != prev_count
);
308 delta
= (new_count
- prev_count
) & DMC620_CNT_MAX_PERIOD
;
309 local64_add(delta
, &event
->count
);
312 static void dmc620_pmu_event_set_period(struct perf_event
*event
)
314 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
316 local64_set(&event
->hw
.prev_count
, DMC620_CNT_INIT
);
317 dmc620_pmu_creg_write(dmc620_pmu
,
318 event
->hw
.idx
, DMC620_PMU_COUNTERn_VALUE
, DMC620_CNT_INIT
);
321 static void dmc620_pmu_enable_counter(struct perf_event
*event
)
323 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
326 reg
= dmc620_event_to_counter_control(event
) | DMC620_PMU_COUNTERn_CONTROL_ENABLE
;
327 dmc620_pmu_creg_write(dmc620_pmu
,
328 event
->hw
.idx
, DMC620_PMU_COUNTERn_CONTROL
, reg
);
331 static void dmc620_pmu_disable_counter(struct perf_event
*event
)
333 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
335 dmc620_pmu_creg_write(dmc620_pmu
,
336 event
->hw
.idx
, DMC620_PMU_COUNTERn_CONTROL
, 0);
339 static irqreturn_t
dmc620_pmu_handle_irq(int irq_num
, void *data
)
341 struct dmc620_pmu_irq
*irq
= data
;
342 struct dmc620_pmu
*dmc620_pmu
;
343 irqreturn_t ret
= IRQ_NONE
;
346 list_for_each_entry_rcu(dmc620_pmu
, &irq
->pmus_node
, pmus_node
) {
347 unsigned long status
;
348 struct perf_event
*event
;
352 * HW doesn't provide a control to atomically disable all counters.
353 * To prevent race condition (overflow happens while clearing status register),
354 * disable all events before continuing
356 for (idx
= 0; idx
< DMC620_PMU_MAX_COUNTERS
; idx
++) {
357 event
= dmc620_pmu
->events
[idx
];
360 dmc620_pmu_disable_counter(event
);
363 status
= readl(dmc620_pmu
->base
+ DMC620_PMU_OVERFLOW_STATUS_CLKDIV2
);
364 status
|= (readl(dmc620_pmu
->base
+ DMC620_PMU_OVERFLOW_STATUS_CLK
) <<
365 DMC620_PMU_CLKDIV2_MAX_COUNTERS
);
367 for_each_set_bit(idx
, &status
,
368 DMC620_PMU_MAX_COUNTERS
) {
369 event
= dmc620_pmu
->events
[idx
];
370 if (WARN_ON_ONCE(!event
))
372 dmc620_pmu_event_update(event
);
373 dmc620_pmu_event_set_period(event
);
376 if (status
& DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK
)
377 writel(0, dmc620_pmu
->base
+ DMC620_PMU_OVERFLOW_STATUS_CLKDIV2
);
379 if ((status
>> DMC620_PMU_CLKDIV2_MAX_COUNTERS
) &
380 DMC620_PMU_OVERFLOW_STATUS_CLK_MASK
)
381 writel(0, dmc620_pmu
->base
+ DMC620_PMU_OVERFLOW_STATUS_CLK
);
384 for (idx
= 0; idx
< DMC620_PMU_MAX_COUNTERS
; idx
++) {
385 event
= dmc620_pmu
->events
[idx
];
388 if (!(event
->hw
.state
& PERF_HES_STOPPED
))
389 dmc620_pmu_enable_counter(event
);
399 static struct dmc620_pmu_irq
*__dmc620_pmu_get_irq(int irq_num
)
401 struct dmc620_pmu_irq
*irq
;
404 list_for_each_entry(irq
, &dmc620_pmu_irqs
, irqs_node
)
405 if (irq
->irq_num
== irq_num
&& refcount_inc_not_zero(&irq
->refcount
))
408 irq
= kzalloc(sizeof(*irq
), GFP_KERNEL
);
410 return ERR_PTR(-ENOMEM
);
412 INIT_LIST_HEAD(&irq
->pmus_node
);
414 /* Pick one CPU to be the preferred one to use */
415 irq
->cpu
= raw_smp_processor_id();
416 refcount_set(&irq
->refcount
, 1);
418 ret
= request_irq(irq_num
, dmc620_pmu_handle_irq
,
419 IRQF_NOBALANCING
| IRQF_NO_THREAD
,
424 ret
= irq_set_affinity_hint(irq_num
, cpumask_of(irq
->cpu
));
428 ret
= cpuhp_state_add_instance_nocalls(cpuhp_state_num
, &irq
->node
);
432 irq
->irq_num
= irq_num
;
433 list_add(&irq
->irqs_node
, &dmc620_pmu_irqs
);
438 free_irq(irq_num
, irq
);
444 static int dmc620_pmu_get_irq(struct dmc620_pmu
*dmc620_pmu
, int irq_num
)
446 struct dmc620_pmu_irq
*irq
;
448 mutex_lock(&dmc620_pmu_irqs_lock
);
449 irq
= __dmc620_pmu_get_irq(irq_num
);
450 mutex_unlock(&dmc620_pmu_irqs_lock
);
455 dmc620_pmu
->irq
= irq
;
456 mutex_lock(&dmc620_pmu_irqs_lock
);
457 list_add_rcu(&dmc620_pmu
->pmus_node
, &irq
->pmus_node
);
458 mutex_unlock(&dmc620_pmu_irqs_lock
);
463 static void dmc620_pmu_put_irq(struct dmc620_pmu
*dmc620_pmu
)
465 struct dmc620_pmu_irq
*irq
= dmc620_pmu
->irq
;
467 mutex_lock(&dmc620_pmu_irqs_lock
);
468 list_del_rcu(&dmc620_pmu
->pmus_node
);
470 if (!refcount_dec_and_test(&irq
->refcount
)) {
471 mutex_unlock(&dmc620_pmu_irqs_lock
);
475 list_del(&irq
->irqs_node
);
476 mutex_unlock(&dmc620_pmu_irqs_lock
);
478 WARN_ON(irq_set_affinity_hint(irq
->irq_num
, NULL
));
479 free_irq(irq
->irq_num
, irq
);
480 cpuhp_state_remove_instance_nocalls(cpuhp_state_num
, &irq
->node
);
484 static int dmc620_pmu_event_init(struct perf_event
*event
)
486 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
487 struct hw_perf_event
*hwc
= &event
->hw
;
488 struct perf_event
*sibling
;
490 if (event
->attr
.type
!= event
->pmu
->type
)
494 * DMC 620 PMUs are shared across all cpus and cannot
495 * support task bound and sampling events.
497 if (is_sampling_event(event
) ||
498 event
->attach_state
& PERF_ATTACH_TASK
) {
499 dev_dbg(dmc620_pmu
->pmu
.dev
,
500 "Can't support per-task counters\n");
505 * Many perf core operations (eg. events rotation) operate on a
506 * single CPU context. This is obvious for CPU PMUs, where one
507 * expects the same sets of events being observed on all CPUs,
508 * but can lead to issues for off-core PMUs, where each
509 * event could be theoretically assigned to a different CPU. To
510 * mitigate this, we enforce CPU assignment to one, selected
513 event
->cpu
= dmc620_pmu
->irq
->cpu
;
518 * We can't atomically disable all HW counters so only one event allowed,
519 * although software events are acceptable.
521 if (event
->group_leader
!= event
&&
522 !is_software_event(event
->group_leader
))
525 for_each_sibling_event(sibling
, event
->group_leader
) {
526 if (sibling
!= event
&&
527 !is_software_event(sibling
))
535 static void dmc620_pmu_read(struct perf_event
*event
)
537 dmc620_pmu_event_update(event
);
540 static void dmc620_pmu_start(struct perf_event
*event
, int flags
)
543 dmc620_pmu_event_set_period(event
);
544 dmc620_pmu_enable_counter(event
);
547 static void dmc620_pmu_stop(struct perf_event
*event
, int flags
)
549 if (event
->hw
.state
& PERF_HES_STOPPED
)
552 dmc620_pmu_disable_counter(event
);
553 dmc620_pmu_event_update(event
);
554 event
->hw
.state
|= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
557 static int dmc620_pmu_add(struct perf_event
*event
, int flags
)
559 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
560 struct perf_event_attr
*attr
= &event
->attr
;
561 struct hw_perf_event
*hwc
= &event
->hw
;
565 idx
= dmc620_get_event_idx(event
);
570 dmc620_pmu
->events
[idx
] = event
;
571 hwc
->state
= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
573 reg
= ATTR_CFG_GET_FLD(attr
, mask
);
574 dmc620_pmu_creg_write(dmc620_pmu
,
575 idx
, DMC620_PMU_COUNTERn_MASK_31_00
, lower_32_bits(reg
));
576 dmc620_pmu_creg_write(dmc620_pmu
,
577 idx
, DMC620_PMU_COUNTERn_MASK_63_32
, upper_32_bits(reg
));
579 reg
= ATTR_CFG_GET_FLD(attr
, match
);
580 dmc620_pmu_creg_write(dmc620_pmu
,
581 idx
, DMC620_PMU_COUNTERn_MATCH_31_00
, lower_32_bits(reg
));
582 dmc620_pmu_creg_write(dmc620_pmu
,
583 idx
, DMC620_PMU_COUNTERn_MATCH_63_32
, upper_32_bits(reg
));
585 if (flags
& PERF_EF_START
)
586 dmc620_pmu_start(event
, PERF_EF_RELOAD
);
588 perf_event_update_userpage(event
);
592 static void dmc620_pmu_del(struct perf_event
*event
, int flags
)
594 struct dmc620_pmu
*dmc620_pmu
= to_dmc620_pmu(event
->pmu
);
595 struct hw_perf_event
*hwc
= &event
->hw
;
598 dmc620_pmu_stop(event
, PERF_EF_UPDATE
);
599 dmc620_pmu
->events
[idx
] = NULL
;
600 clear_bit(idx
, dmc620_pmu
->used_mask
);
601 perf_event_update_userpage(event
);
604 static int dmc620_pmu_cpu_teardown(unsigned int cpu
,
605 struct hlist_node
*node
)
607 struct dmc620_pmu_irq
*irq
;
608 struct dmc620_pmu
*dmc620_pmu
;
611 irq
= hlist_entry_safe(node
, struct dmc620_pmu_irq
, node
);
615 target
= cpumask_any_but(cpu_online_mask
, cpu
);
616 if (target
>= nr_cpu_ids
)
619 /* We're only reading, but this isn't the place to be involving RCU */
620 mutex_lock(&dmc620_pmu_irqs_lock
);
621 list_for_each_entry(dmc620_pmu
, &irq
->pmus_node
, pmus_node
)
622 perf_pmu_migrate_context(&dmc620_pmu
->pmu
, irq
->cpu
, target
);
623 mutex_unlock(&dmc620_pmu_irqs_lock
);
625 WARN_ON(irq_set_affinity_hint(irq
->irq_num
, cpumask_of(target
)));
631 static int dmc620_pmu_device_probe(struct platform_device
*pdev
)
633 struct dmc620_pmu
*dmc620_pmu
;
634 struct resource
*res
;
639 dmc620_pmu
= devm_kzalloc(&pdev
->dev
,
640 sizeof(struct dmc620_pmu
), GFP_KERNEL
);
644 platform_set_drvdata(pdev
, dmc620_pmu
);
646 dmc620_pmu
->pmu
= (struct pmu
) {
647 .module
= THIS_MODULE
,
648 .capabilities
= PERF_PMU_CAP_NO_EXCLUDE
,
649 .task_ctx_nr
= perf_invalid_context
,
650 .event_init
= dmc620_pmu_event_init
,
651 .add
= dmc620_pmu_add
,
652 .del
= dmc620_pmu_del
,
653 .start
= dmc620_pmu_start
,
654 .stop
= dmc620_pmu_stop
,
655 .read
= dmc620_pmu_read
,
656 .attr_groups
= dmc620_pmu_attr_groups
,
659 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
660 dmc620_pmu
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
661 if (IS_ERR(dmc620_pmu
->base
))
662 return PTR_ERR(dmc620_pmu
->base
);
664 /* Make sure device is reset before enabling interrupt */
665 for (i
= 0; i
< DMC620_PMU_MAX_COUNTERS
; i
++)
666 dmc620_pmu_creg_write(dmc620_pmu
, i
, DMC620_PMU_COUNTERn_CONTROL
, 0);
667 writel(0, dmc620_pmu
->base
+ DMC620_PMU_OVERFLOW_STATUS_CLKDIV2
);
668 writel(0, dmc620_pmu
->base
+ DMC620_PMU_OVERFLOW_STATUS_CLK
);
670 irq_num
= platform_get_irq(pdev
, 0);
674 ret
= dmc620_pmu_get_irq(dmc620_pmu
, irq_num
);
678 name
= devm_kasprintf(&pdev
->dev
, GFP_KERNEL
,
679 "%s_%llx", DMC620_PMUNAME
,
680 (u64
)(res
->start
>> DMC620_PA_SHIFT
));
683 "Create name failed, PMU @%pa\n", &res
->start
);
685 goto out_teardown_dev
;
688 ret
= perf_pmu_register(&dmc620_pmu
->pmu
, name
, -1);
690 goto out_teardown_dev
;
695 dmc620_pmu_put_irq(dmc620_pmu
);
700 static int dmc620_pmu_device_remove(struct platform_device
*pdev
)
702 struct dmc620_pmu
*dmc620_pmu
= platform_get_drvdata(pdev
);
704 dmc620_pmu_put_irq(dmc620_pmu
);
706 /* perf will synchronise RCU before devres can free dmc620_pmu */
707 perf_pmu_unregister(&dmc620_pmu
->pmu
);
712 static const struct acpi_device_id dmc620_acpi_match
[] = {
716 MODULE_DEVICE_TABLE(acpi
, dmc620_acpi_match
);
717 static struct platform_driver dmc620_pmu_driver
= {
719 .name
= DMC620_DRVNAME
,
720 .acpi_match_table
= dmc620_acpi_match
,
721 .suppress_bind_attrs
= true,
723 .probe
= dmc620_pmu_device_probe
,
724 .remove
= dmc620_pmu_device_remove
,
727 static int __init
dmc620_pmu_init(void)
729 cpuhp_state_num
= cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN
,
732 dmc620_pmu_cpu_teardown
);
733 if (cpuhp_state_num
< 0)
734 return cpuhp_state_num
;
736 return platform_driver_register(&dmc620_pmu_driver
);
739 static void __exit
dmc620_pmu_exit(void)
741 platform_driver_unregister(&dmc620_pmu_driver
);
742 cpuhp_remove_multi_state(cpuhp_state_num
);
745 module_init(dmc620_pmu_init
);
746 module_exit(dmc620_pmu_exit
);
748 MODULE_DESCRIPTION("Perf driver for the ARM DMC-620 memory controller");
749 MODULE_AUTHOR("Tuan Phan <tuanphan@os.amperecomputing.com");
750 MODULE_LICENSE("GPL v2");