2 * Performance event support - Freescale Embedded Performance Monitor
4 * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
5 * Copyright 2010 Freescale Semiconductor, Inc.
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
12 #include <linux/kernel.h>
13 #include <linux/sched.h>
14 #include <linux/perf_event.h>
15 #include <linux/percpu.h>
16 #include <linux/hardirq.h>
17 #include <asm/reg_fsl_emb.h>
19 #include <asm/machdep.h>
20 #include <asm/firmware.h>
21 #include <asm/ptrace.h>
23 struct cpu_hw_events
{
27 struct perf_event
*event
[MAX_HWEVENTS
];
29 static DEFINE_PER_CPU(struct cpu_hw_events
, cpu_hw_events
);
31 static struct fsl_emb_pmu
*ppmu
;
33 /* Number of perf_events counting hardware events */
34 static atomic_t num_events
;
35 /* Used to avoid races in calling reserve/release_pmc_hardware */
36 static DEFINE_MUTEX(pmc_reserve_mutex
);
39 * If interrupts were soft-disabled when a PMU interrupt occurs, treat
42 static inline int perf_intr_is_nmi(struct pt_regs
*regs
)
51 static void perf_event_interrupt(struct pt_regs
*regs
);
54 * Read one performance monitor counter (PMC).
56 static unsigned long read_pmc(int idx
)
62 val
= mfpmr(PMRN_PMC0
);
65 val
= mfpmr(PMRN_PMC1
);
68 val
= mfpmr(PMRN_PMC2
);
71 val
= mfpmr(PMRN_PMC3
);
74 printk(KERN_ERR
"oops trying to read PMC%d\n", idx
);
83 static void write_pmc(int idx
, unsigned long val
)
87 mtpmr(PMRN_PMC0
, val
);
90 mtpmr(PMRN_PMC1
, val
);
93 mtpmr(PMRN_PMC2
, val
);
96 mtpmr(PMRN_PMC3
, val
);
99 printk(KERN_ERR
"oops trying to write PMC%d\n", idx
);
106 * Write one local control A register
108 static void write_pmlca(int idx
, unsigned long val
)
112 mtpmr(PMRN_PMLCA0
, val
);
115 mtpmr(PMRN_PMLCA1
, val
);
118 mtpmr(PMRN_PMLCA2
, val
);
121 mtpmr(PMRN_PMLCA3
, val
);
124 printk(KERN_ERR
"oops trying to write PMLCA%d\n", idx
);
131 * Write one local control B register
133 static void write_pmlcb(int idx
, unsigned long val
)
137 mtpmr(PMRN_PMLCB0
, val
);
140 mtpmr(PMRN_PMLCB1
, val
);
143 mtpmr(PMRN_PMLCB2
, val
);
146 mtpmr(PMRN_PMLCB3
, val
);
149 printk(KERN_ERR
"oops trying to write PMLCB%d\n", idx
);
155 static void fsl_emb_pmu_read(struct perf_event
*event
)
157 s64 val
, delta
, prev
;
159 if (event
->hw
.state
& PERF_HES_STOPPED
)
163 * Performance monitor interrupts come even when interrupts
164 * are soft-disabled, as long as interrupts are hard-enabled.
165 * Therefore we treat them like NMIs.
168 prev
= local64_read(&event
->hw
.prev_count
);
170 val
= read_pmc(event
->hw
.idx
);
171 } while (local64_cmpxchg(&event
->hw
.prev_count
, prev
, val
) != prev
);
173 /* The counters are only 32 bits wide */
174 delta
= (val
- prev
) & 0xfffffffful
;
175 local64_add(delta
, &event
->count
);
176 local64_sub(delta
, &event
->hw
.period_left
);
180 * Disable all events to prevent PMU interrupts and to allow
181 * events to be added or removed.
183 static void fsl_emb_pmu_disable(struct pmu
*pmu
)
185 struct cpu_hw_events
*cpuhw
;
188 local_irq_save(flags
);
189 cpuhw
= &__get_cpu_var(cpu_hw_events
);
191 if (!cpuhw
->disabled
) {
195 * Check if we ever enabled the PMU on this cpu.
197 if (!cpuhw
->pmcs_enabled
) {
199 cpuhw
->pmcs_enabled
= 1;
202 if (atomic_read(&num_events
)) {
204 * Set the 'freeze all counters' bit, and disable
205 * interrupts. The barrier is to make sure the
206 * mtpmr has been executed and the PMU has frozen
207 * the events before we return.
210 mtpmr(PMRN_PMGC0
, PMGC0_FAC
);
214 local_irq_restore(flags
);
218 * Re-enable all events if disable == 0.
219 * If we were previously disabled and events were added, then
220 * put the new config on the PMU.
222 static void fsl_emb_pmu_enable(struct pmu
*pmu
)
224 struct cpu_hw_events
*cpuhw
;
227 local_irq_save(flags
);
228 cpuhw
= &__get_cpu_var(cpu_hw_events
);
229 if (!cpuhw
->disabled
)
233 ppc_set_pmu_inuse(cpuhw
->n_events
!= 0);
235 if (cpuhw
->n_events
> 0) {
236 mtpmr(PMRN_PMGC0
, PMGC0_PMIE
| PMGC0_FCECE
);
241 local_irq_restore(flags
);
244 static int collect_events(struct perf_event
*group
, int max_count
,
245 struct perf_event
*ctrs
[])
248 struct perf_event
*event
;
250 if (!is_software_event(group
)) {
256 list_for_each_entry(event
, &group
->sibling_list
, group_entry
) {
257 if (!is_software_event(event
) &&
258 event
->state
!= PERF_EVENT_STATE_OFF
) {
268 /* context locked on entry */
269 static int fsl_emb_pmu_add(struct perf_event
*event
, int flags
)
271 struct cpu_hw_events
*cpuhw
;
273 int num_counters
= ppmu
->n_counter
;
277 perf_pmu_disable(event
->pmu
);
278 cpuhw
= &get_cpu_var(cpu_hw_events
);
280 if (event
->hw
.config
& FSL_EMB_EVENT_RESTRICTED
)
281 num_counters
= ppmu
->n_restricted
;
284 * Allocate counters from top-down, so that restricted-capable
285 * counters are kept free as long as possible.
287 for (i
= num_counters
- 1; i
>= 0; i
--) {
298 cpuhw
->event
[i
] = event
;
302 if (event
->hw
.sample_period
) {
303 s64 left
= local64_read(&event
->hw
.period_left
);
304 if (left
< 0x80000000L
)
305 val
= 0x80000000L
- left
;
307 local64_set(&event
->hw
.prev_count
, val
);
309 if (!(flags
& PERF_EF_START
)) {
310 event
->hw
.state
= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
315 perf_event_update_userpage(event
);
317 write_pmlcb(i
, event
->hw
.config
>> 32);
318 write_pmlca(i
, event
->hw
.config_base
);
322 put_cpu_var(cpu_hw_events
);
323 perf_pmu_enable(event
->pmu
);
327 /* context locked on entry */
328 static void fsl_emb_pmu_del(struct perf_event
*event
, int flags
)
330 struct cpu_hw_events
*cpuhw
;
331 int i
= event
->hw
.idx
;
333 perf_pmu_disable(event
->pmu
);
337 fsl_emb_pmu_read(event
);
339 cpuhw
= &get_cpu_var(cpu_hw_events
);
341 WARN_ON(event
!= cpuhw
->event
[event
->hw
.idx
]);
347 cpuhw
->event
[i
] = NULL
;
351 * TODO: if at least one restricted event exists, and we
352 * just freed up a non-restricted-capable counter, and
353 * there is a restricted-capable counter occupied by
354 * a non-restricted event, migrate that event to the
361 perf_pmu_enable(event
->pmu
);
362 put_cpu_var(cpu_hw_events
);
365 static void fsl_emb_pmu_start(struct perf_event
*event
, int ef_flags
)
370 if (event
->hw
.idx
< 0 || !event
->hw
.sample_period
)
373 if (!(event
->hw
.state
& PERF_HES_STOPPED
))
376 if (ef_flags
& PERF_EF_RELOAD
)
377 WARN_ON_ONCE(!(event
->hw
.state
& PERF_HES_UPTODATE
));
379 local_irq_save(flags
);
380 perf_pmu_disable(event
->pmu
);
383 left
= local64_read(&event
->hw
.period_left
);
384 write_pmc(event
->hw
.idx
, left
);
386 perf_event_update_userpage(event
);
387 perf_pmu_enable(event
->pmu
);
388 local_irq_restore(flags
);
391 static void fsl_emb_pmu_stop(struct perf_event
*event
, int ef_flags
)
395 if (event
->hw
.idx
< 0 || !event
->hw
.sample_period
)
398 if (event
->hw
.state
& PERF_HES_STOPPED
)
401 local_irq_save(flags
);
402 perf_pmu_disable(event
->pmu
);
404 fsl_emb_pmu_read(event
);
405 event
->hw
.state
|= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
406 write_pmc(event
->hw
.idx
, 0);
408 perf_event_update_userpage(event
);
409 perf_pmu_enable(event
->pmu
);
410 local_irq_restore(flags
);
414 * Release the PMU if this is the last perf_event.
416 static void hw_perf_event_destroy(struct perf_event
*event
)
418 if (!atomic_add_unless(&num_events
, -1, 1)) {
419 mutex_lock(&pmc_reserve_mutex
);
420 if (atomic_dec_return(&num_events
) == 0)
421 release_pmc_hardware();
422 mutex_unlock(&pmc_reserve_mutex
);
427 * Translate a generic cache event_id config to a raw event_id code.
429 static int hw_perf_cache_event(u64 config
, u64
*eventp
)
431 unsigned long type
, op
, result
;
434 if (!ppmu
->cache_events
)
438 type
= config
& 0xff;
439 op
= (config
>> 8) & 0xff;
440 result
= (config
>> 16) & 0xff;
442 if (type
>= PERF_COUNT_HW_CACHE_MAX
||
443 op
>= PERF_COUNT_HW_CACHE_OP_MAX
||
444 result
>= PERF_COUNT_HW_CACHE_RESULT_MAX
)
447 ev
= (*ppmu
->cache_events
)[type
][op
][result
];
456 static int fsl_emb_pmu_event_init(struct perf_event
*event
)
459 struct perf_event
*events
[MAX_HWEVENTS
];
465 switch (event
->attr
.type
) {
466 case PERF_TYPE_HARDWARE
:
467 ev
= event
->attr
.config
;
468 if (ev
>= ppmu
->n_generic
|| ppmu
->generic_events
[ev
] == 0)
470 ev
= ppmu
->generic_events
[ev
];
473 case PERF_TYPE_HW_CACHE
:
474 err
= hw_perf_cache_event(event
->attr
.config
, &ev
);
480 ev
= event
->attr
.config
;
487 event
->hw
.config
= ppmu
->xlate_event(ev
);
488 if (!(event
->hw
.config
& FSL_EMB_EVENT_VALID
))
492 * If this is in a group, check if it can go on with all the
493 * other hardware events in the group. We assume the event
494 * hasn't been linked into its leader's sibling list at this point.
497 if (event
->group_leader
!= event
) {
498 n
= collect_events(event
->group_leader
,
499 ppmu
->n_counter
- 1, events
);
504 if (event
->hw
.config
& FSL_EMB_EVENT_RESTRICTED
) {
506 for (i
= 0; i
< n
; i
++) {
507 if (events
[i
]->hw
.config
& FSL_EMB_EVENT_RESTRICTED
)
511 if (num_restricted
>= ppmu
->n_restricted
)
517 event
->hw
.config_base
= PMLCA_CE
| PMLCA_FCM1
|
518 (u32
)((ev
<< 16) & PMLCA_EVENT_MASK
);
520 if (event
->attr
.exclude_user
)
521 event
->hw
.config_base
|= PMLCA_FCU
;
522 if (event
->attr
.exclude_kernel
)
523 event
->hw
.config_base
|= PMLCA_FCS
;
524 if (event
->attr
.exclude_idle
)
527 event
->hw
.last_period
= event
->hw
.sample_period
;
528 local64_set(&event
->hw
.period_left
, event
->hw
.last_period
);
531 * See if we need to reserve the PMU.
532 * If no events are currently in use, then we have to take a
533 * mutex to ensure that we don't race with another task doing
534 * reserve_pmc_hardware or release_pmc_hardware.
537 if (!atomic_inc_not_zero(&num_events
)) {
538 mutex_lock(&pmc_reserve_mutex
);
539 if (atomic_read(&num_events
) == 0 &&
540 reserve_pmc_hardware(perf_event_interrupt
))
543 atomic_inc(&num_events
);
544 mutex_unlock(&pmc_reserve_mutex
);
546 mtpmr(PMRN_PMGC0
, PMGC0_FAC
);
549 event
->destroy
= hw_perf_event_destroy
;
554 static struct pmu fsl_emb_pmu
= {
555 .pmu_enable
= fsl_emb_pmu_enable
,
556 .pmu_disable
= fsl_emb_pmu_disable
,
557 .event_init
= fsl_emb_pmu_event_init
,
558 .add
= fsl_emb_pmu_add
,
559 .del
= fsl_emb_pmu_del
,
560 .start
= fsl_emb_pmu_start
,
561 .stop
= fsl_emb_pmu_stop
,
562 .read
= fsl_emb_pmu_read
,
566 * A counter has overflowed; update its count and record
567 * things if requested. Note that interrupts are hard-disabled
568 * here so there is no possibility of being interrupted.
570 static void record_and_restart(struct perf_event
*event
, unsigned long val
,
571 struct pt_regs
*regs
, int nmi
)
573 u64 period
= event
->hw
.sample_period
;
574 s64 prev
, delta
, left
;
577 if (event
->hw
.state
& PERF_HES_STOPPED
) {
578 write_pmc(event
->hw
.idx
, 0);
582 /* we don't have to worry about interrupts here */
583 prev
= local64_read(&event
->hw
.prev_count
);
584 delta
= (val
- prev
) & 0xfffffffful
;
585 local64_add(delta
, &event
->count
);
588 * See if the total period for this event has expired,
589 * and update for the next period.
592 left
= local64_read(&event
->hw
.period_left
) - delta
;
600 if (left
< 0x80000000LL
)
601 val
= 0x80000000LL
- left
;
604 write_pmc(event
->hw
.idx
, val
);
605 local64_set(&event
->hw
.prev_count
, val
);
606 local64_set(&event
->hw
.period_left
, left
);
607 perf_event_update_userpage(event
);
610 * Finally record data if requested.
613 struct perf_sample_data data
;
615 perf_sample_data_init(&data
, 0);
616 data
.period
= event
->hw
.last_period
;
618 if (perf_event_overflow(event
, nmi
, &data
, regs
))
619 fsl_emb_pmu_stop(event
, 0);
623 static void perf_event_interrupt(struct pt_regs
*regs
)
626 struct cpu_hw_events
*cpuhw
= &__get_cpu_var(cpu_hw_events
);
627 struct perf_event
*event
;
632 nmi
= perf_intr_is_nmi(regs
);
638 for (i
= 0; i
< ppmu
->n_counter
; ++i
) {
639 event
= cpuhw
->event
[i
];
644 /* event has overflowed */
646 record_and_restart(event
, val
, regs
, nmi
);
649 * Disabled counter is negative,
650 * reset it just in case.
657 /* PMM will keep counters frozen until we return from the interrupt. */
658 mtmsr(mfmsr() | MSR_PMM
);
659 mtpmr(PMRN_PMGC0
, PMGC0_PMIE
| PMGC0_FCECE
);
668 void hw_perf_event_setup(int cpu
)
670 struct cpu_hw_events
*cpuhw
= &per_cpu(cpu_hw_events
, cpu
);
672 memset(cpuhw
, 0, sizeof(*cpuhw
));
675 int register_fsl_emb_pmu(struct fsl_emb_pmu
*pmu
)
678 return -EBUSY
; /* something's already registered */
681 pr_info("%s performance monitor hardware support registered\n",
684 perf_pmu_register(&fsl_emb_pmu
, "cpu", PERF_TYPE_RAW
);