2 * arch/s390/oprofile/hwsampler.c
4 * Copyright IBM Corp. 2010
5 * Author: Heinz Graalfs <graalfs@de.ibm.com>
8 #include <linux/kernel_stat.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/smp.h>
12 #include <linux/errno.h>
13 #include <linux/workqueue.h>
14 #include <linux/interrupt.h>
15 #include <linux/notifier.h>
16 #include <linux/cpu.h>
17 #include <linux/semaphore.h>
18 #include <linux/oom.h>
19 #include <linux/oprofile.h>
21 #include <asm/lowcore.h>
24 #include "hwsampler.h"
26 #define MAX_NUM_SDB 511
29 #define ALERT_REQ_MASK 0x4000000000000000ul
30 #define BUFFER_FULL_MASK 0x8000000000000000ul
32 #define EI_IEA (1 << 31) /* invalid entry address */
33 #define EI_ISE (1 << 30) /* incorrect SDBT entry */
34 #define EI_PRA (1 << 29) /* program request alert */
35 #define EI_SACA (1 << 23) /* sampler authorization change alert */
36 #define EI_LSDA (1 << 22) /* loss of sample data alert */
38 DECLARE_PER_CPU(struct hws_cpu_buffer
, sampler_cpu_buffer
);
40 struct hws_execute_parms
{
45 DEFINE_PER_CPU(struct hws_cpu_buffer
, sampler_cpu_buffer
);
46 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer
);
48 static DEFINE_MUTEX(hws_sem
);
49 static DEFINE_MUTEX(hws_sem_oom
);
51 static unsigned char hws_flush_all
;
52 static unsigned int hws_oom
;
53 static struct workqueue_struct
*hws_wq
;
55 static unsigned int hws_state
;
63 /* set to 1 if called by kernel during memory allocation */
64 static unsigned char oom_killer_was_active
;
65 /* size of SDBT and SDB as of allocate API */
66 static unsigned long num_sdbt
= 100;
67 static unsigned long num_sdb
= 511;
68 /* sampling interval (machine cycles) */
69 static unsigned long interval
;
71 static unsigned long min_sampler_rate
;
72 static unsigned long max_sampler_rate
;
74 static int ssctl(void *buffer
)
78 /* set in order to detect a program check */
82 "0: .insn s,0xB2870000,0(%1)\n"
86 EX_TABLE(0b
, 2b
) EX_TABLE(1b
, 2b
)
87 : "+d" (cc
), "+a" (buffer
)
88 : "m" (*((struct hws_ssctl_request_block
*)buffer
))
91 return cc
? -EINVAL
: 0 ;
94 static int qsi(void *buffer
)
100 "0: .insn s,0xB2860000,0(%1)\n"
103 EX_TABLE(0b
, 2b
) EX_TABLE(1b
, 2b
)
104 : "=d" (cc
), "+a" (buffer
)
105 : "m" (*((struct hws_qsi_info_block
*)buffer
))
108 return cc
? -EINVAL
: 0;
111 static void execute_qsi(void *parms
)
113 struct hws_execute_parms
*ep
= parms
;
115 ep
->rc
= qsi(ep
->buffer
);
118 static void execute_ssctl(void *parms
)
120 struct hws_execute_parms
*ep
= parms
;
122 ep
->rc
= ssctl(ep
->buffer
);
125 static int smp_ctl_ssctl_stop(int cpu
)
128 struct hws_execute_parms ep
;
129 struct hws_cpu_buffer
*cb
;
131 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
136 ep
.buffer
= &cb
->ssctl
;
137 smp_call_function_single(cpu
, execute_ssctl
, &ep
, 1);
140 printk(KERN_ERR
"hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu
);
144 ep
.buffer
= &cb
->qsi
;
145 smp_call_function_single(cpu
, execute_qsi
, &ep
, 1);
147 if (cb
->qsi
.es
|| cb
->qsi
.cs
) {
148 printk(KERN_EMERG
"CPUMF sampling did not stop properly.\n");
155 static int smp_ctl_ssctl_deactivate(int cpu
)
158 struct hws_execute_parms ep
;
159 struct hws_cpu_buffer
*cb
;
161 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
166 ep
.buffer
= &cb
->ssctl
;
167 smp_call_function_single(cpu
, execute_ssctl
, &ep
, 1);
170 printk(KERN_ERR
"hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu
);
172 ep
.buffer
= &cb
->qsi
;
173 smp_call_function_single(cpu
, execute_qsi
, &ep
, 1);
176 printk(KERN_EMERG
"CPUMF sampling was not set inactive.\n");
181 static int smp_ctl_ssctl_enable_activate(int cpu
, unsigned long interval
)
184 struct hws_execute_parms ep
;
185 struct hws_cpu_buffer
*cb
;
187 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
190 cb
->ssctl
.tear
= cb
->first_sdbt
;
191 cb
->ssctl
.dear
= *(unsigned long *) cb
->first_sdbt
;
192 cb
->ssctl
.interval
= interval
;
196 ep
.buffer
= &cb
->ssctl
;
197 smp_call_function_single(cpu
, execute_ssctl
, &ep
, 1);
200 printk(KERN_ERR
"hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu
);
202 ep
.buffer
= &cb
->qsi
;
203 smp_call_function_single(cpu
, execute_qsi
, &ep
, 1);
205 printk(KERN_ERR
"hwsampler: CPU %d CPUMF QSI failed.\n", cpu
);
210 static int smp_ctl_qsi(int cpu
)
212 struct hws_execute_parms ep
;
213 struct hws_cpu_buffer
*cb
;
215 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
217 ep
.buffer
= &cb
->qsi
;
218 smp_call_function_single(cpu
, execute_qsi
, &ep
, 1);
223 static inline unsigned long *trailer_entry_ptr(unsigned long v
)
229 ret
-= sizeof(struct hws_trailer_entry
);
231 return (unsigned long *) ret
;
234 /* prototypes for external interrupt handler and worker */
235 static void hws_ext_handler(unsigned int ext_int_code
,
236 unsigned int param32
, unsigned long param64
);
238 static void worker(struct work_struct
*work
);
240 static void add_samples_to_oprofile(unsigned cpu
, unsigned long *,
241 unsigned long *dear
);
243 static void init_all_cpu_buffers(void)
246 struct hws_cpu_buffer
*cb
;
248 for_each_online_cpu(cpu
) {
249 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
250 memset(cb
, 0, sizeof(struct hws_cpu_buffer
));
254 static int is_link_entry(unsigned long *s
)
256 return *s
& 0x1ul
? 1 : 0;
259 static unsigned long *get_next_sdbt(unsigned long *s
)
261 return (unsigned long *) (*s
& ~0x1ul
);
264 static int prepare_cpu_buffers(void)
268 struct hws_cpu_buffer
*cb
;
271 for_each_online_cpu(cpu
) {
272 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
273 atomic_set(&cb
->ext_params
, 0);
274 cb
->worker_entry
= 0;
275 cb
->sample_overflow
= 0;
277 cb
->incorrect_sdbt_entry
= 0;
278 cb
->invalid_entry_address
= 0;
279 cb
->loss_of_sample_data
= 0;
280 cb
->sample_auth_change_alert
= 0;
290 * allocate_sdbt() - allocate sampler memory
291 * @cpu: the cpu for which sampler memory is allocated
293 * A 4K page is allocated for each requested SDBT.
294 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
295 * Set ALERT_REQ mask in each SDBs trailer.
296 * Returns zero if successful, <0 otherwise.
298 static int allocate_sdbt(int cpu
)
304 unsigned long *trailer
;
305 struct hws_cpu_buffer
*cb
;
307 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
315 for (j
= 0; j
< num_sdbt
; j
++) {
316 sdbt
= (unsigned long *)get_zeroed_page(GFP_KERNEL
);
318 mutex_lock(&hws_sem_oom
);
319 /* OOM killer might have been activated */
321 if (oom_killer_was_active
|| !sdbt
) {
323 free_page((unsigned long)sdbt
);
325 goto allocate_sdbt_error
;
327 if (cb
->first_sdbt
== 0)
328 cb
->first_sdbt
= (unsigned long)sdbt
;
330 /* link current page to tail of chain */
332 *tail
= (unsigned long)(void *)sdbt
+ 1;
334 mutex_unlock(&hws_sem_oom
);
336 for (k
= 0; k
< num_sdb
; k
++) {
337 /* get and set SDB page */
338 sdb
= get_zeroed_page(GFP_KERNEL
);
340 mutex_lock(&hws_sem_oom
);
341 /* OOM killer might have been activated */
343 if (oom_killer_was_active
|| !sdb
) {
347 goto allocate_sdbt_error
;
350 trailer
= trailer_entry_ptr(*sdbt
);
351 *trailer
= ALERT_REQ_MASK
;
353 mutex_unlock(&hws_sem_oom
);
357 mutex_lock(&hws_sem_oom
);
358 if (oom_killer_was_active
)
359 goto allocate_sdbt_error
;
363 *tail
= (unsigned long)
364 ((void *)cb
->first_sdbt
) + 1;
367 mutex_unlock(&hws_sem_oom
);
372 goto allocate_sdbt_exit
;
376 * deallocate_sdbt() - deallocate all sampler memory
378 * For each online CPU all SDBT trees are deallocated.
379 * Returns the number of freed pages.
381 static int deallocate_sdbt(void)
388 for_each_online_cpu(cpu
) {
392 struct hws_cpu_buffer
*cb
;
394 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
399 sdbt
= cb
->first_sdbt
;
400 curr
= (unsigned long *) sdbt
;
403 /* we'll free the SDBT after all SDBs are processed... */
408 /* watch for link entry reset if found */
409 if (is_link_entry(curr
)) {
410 curr
= get_next_sdbt(curr
);
414 /* we are done if we reach the start */
415 if ((unsigned long) curr
== start
)
418 sdbt
= (unsigned long) curr
;
420 /* process SDB pointer */
433 static int start_sampling(int cpu
)
436 struct hws_cpu_buffer
*cb
;
438 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
439 rc
= smp_ctl_ssctl_enable_activate(cpu
, interval
);
441 printk(KERN_INFO
"hwsampler: CPU %d ssctl failed.\n", cpu
);
447 printk(KERN_INFO
"hwsampler: CPU %d ssctl not enabled.\n", cpu
);
452 printk(KERN_INFO
"hwsampler: CPU %d ssctl not active.\n", cpu
);
457 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
466 static int stop_sampling(int cpu
)
470 struct hws_cpu_buffer
*cb
;
472 rc
= smp_ctl_qsi(cpu
);
475 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
476 if (!rc
&& !cb
->qsi
.es
)
477 printk(KERN_INFO
"hwsampler: CPU %d, already stopped.\n", cpu
);
479 rc
= smp_ctl_ssctl_stop(cpu
);
481 printk(KERN_INFO
"hwsampler: CPU %d, ssctl stop error %d.\n",
486 printk(KERN_INFO
"hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu
);
491 printk(KERN_ERR
"hwsampler: CPU %d CPUMF Request alert,"
492 " count=%lu.\n", cpu
, v
);
494 v
= cb
->loss_of_sample_data
;
496 printk(KERN_ERR
"hwsampler: CPU %d CPUMF Loss of sample data,"
497 " count=%lu.\n", cpu
, v
);
499 v
= cb
->invalid_entry_address
;
501 printk(KERN_ERR
"hwsampler: CPU %d CPUMF Invalid entry address,"
502 " count=%lu.\n", cpu
, v
);
504 v
= cb
->incorrect_sdbt_entry
;
507 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
508 " count=%lu.\n", cpu
, v
);
510 v
= cb
->sample_auth_change_alert
;
513 "hwsampler: CPU %d CPUMF Sample authorization change,"
514 " count=%lu.\n", cpu
, v
);
519 static int check_hardware_prerequisites(void)
521 if (!test_facility(68))
526 * hws_oom_callback() - the OOM callback function
528 * In case the callback is invoked during memory allocation for the
529 * hw sampler, all obtained memory is deallocated and a flag is set
530 * so main sampler memory allocation can exit with a failure code.
531 * In case the callback is invoked during sampling the hw sampler
532 * is deactivated for all CPUs.
534 static int hws_oom_callback(struct notifier_block
*nfb
,
535 unsigned long dummy
, void *parm
)
537 unsigned long *freed
;
539 struct hws_cpu_buffer
*cb
;
543 mutex_lock(&hws_sem_oom
);
545 if (hws_state
== HWS_DEALLOCATED
) {
546 /* during memory allocation */
547 if (oom_killer_was_active
== 0) {
548 oom_killer_was_active
= 1;
549 *freed
+= deallocate_sdbt();
554 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
557 for_each_online_cpu(i
) {
558 smp_ctl_ssctl_deactivate(i
);
564 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
569 mutex_unlock(&hws_sem_oom
);
574 static struct notifier_block hws_oom_notifier
= {
575 .notifier_call
= hws_oom_callback
578 static int hws_cpu_callback(struct notifier_block
*nfb
,
579 unsigned long action
, void *hcpu
)
581 /* We do not have sampler space available for all possible CPUs.
582 All CPUs should be online when hw sampling is activated. */
586 static struct notifier_block hws_cpu_notifier
= {
587 .notifier_call
= hws_cpu_callback
591 * hwsampler_deactivate() - set hardware sampling temporarily inactive
592 * @cpu: specifies the CPU to be set inactive.
594 * Returns 0 on success, !0 on failure.
596 int hwsampler_deactivate(unsigned int cpu
)
599 * Deactivate hw sampling temporarily and flush the buffer
600 * by pushing all the pending samples to oprofile buffer.
602 * This function can be called under one of the following conditions:
603 * Memory unmap, task is exiting.
606 struct hws_cpu_buffer
*cb
;
609 mutex_lock(&hws_sem
);
611 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
612 if (hws_state
== HWS_STARTED
) {
613 rc
= smp_ctl_qsi(cpu
);
616 rc
= smp_ctl_ssctl_deactivate(cpu
);
619 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu
);
621 hws_state
= HWS_STOPPING
;
624 /* Add work to queue to read pending samples.*/
625 queue_work_on(cpu
, hws_wq
, &cb
->worker
);
629 mutex_unlock(&hws_sem
);
632 flush_workqueue(hws_wq
);
638 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
639 * @cpu: specifies the CPU to be set active.
641 * Returns 0 on success, !0 on failure.
643 int hwsampler_activate(unsigned int cpu
)
646 * Re-activate hw sampling. This should be called in pair with
647 * hwsampler_deactivate().
650 struct hws_cpu_buffer
*cb
;
653 mutex_lock(&hws_sem
);
655 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
656 if (hws_state
== HWS_STARTED
) {
657 rc
= smp_ctl_qsi(cpu
);
661 rc
= smp_ctl_ssctl_enable_activate(cpu
, interval
);
664 "CPU %d, CPUMF activate sampling failed.\n",
670 mutex_unlock(&hws_sem
);
675 static void hws_ext_handler(unsigned int ext_int_code
,
676 unsigned int param32
, unsigned long param64
)
678 struct hws_cpu_buffer
*cb
;
680 kstat_cpu(smp_processor_id()).irqs
[EXTINT_CPM
]++;
681 cb
= &__get_cpu_var(sampler_cpu_buffer
);
682 atomic_xchg(&cb
->ext_params
, atomic_read(&cb
->ext_params
) | param32
);
684 queue_work(hws_wq
, &cb
->worker
);
687 static int check_qsi_on_setup(void)
691 struct hws_cpu_buffer
*cb
;
693 for_each_online_cpu(cpu
) {
694 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
695 rc
= smp_ctl_qsi(cpu
);
701 printk(KERN_INFO
"hwsampler: CPUMF sampling is not authorized.\n");
706 printk(KERN_WARNING
"hwsampler: CPUMF is still enabled.\n");
707 rc
= smp_ctl_ssctl_stop(cpu
);
712 "CPU %d, CPUMF Sampling stopped now.\n", cpu
);
718 static int check_qsi_on_start(void)
722 struct hws_cpu_buffer
*cb
;
724 for_each_online_cpu(cpu
) {
725 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
726 rc
= smp_ctl_qsi(cpu
);
741 static void worker_on_start(unsigned int cpu
)
743 struct hws_cpu_buffer
*cb
;
745 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
746 cb
->worker_entry
= cb
->first_sdbt
;
749 static int worker_check_error(unsigned int cpu
, int ext_params
)
753 struct hws_cpu_buffer
*cb
;
756 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
757 sdbt
= (unsigned long *) cb
->worker_entry
;
762 if (ext_params
& EI_PRA
)
765 if (ext_params
& EI_LSDA
)
766 cb
->loss_of_sample_data
++;
768 if (ext_params
& EI_IEA
) {
769 cb
->invalid_entry_address
++;
773 if (ext_params
& EI_ISE
) {
774 cb
->incorrect_sdbt_entry
++;
778 if (ext_params
& EI_SACA
) {
779 cb
->sample_auth_change_alert
++;
786 static void worker_on_finish(unsigned int cpu
)
789 struct hws_cpu_buffer
*cb
;
791 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
794 rc
= smp_ctl_qsi(cpu
);
798 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
800 rc
= smp_ctl_ssctl_stop(cpu
);
803 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
806 for_each_online_cpu(i
) {
811 queue_work_on(i
, hws_wq
,
819 static void worker_on_interrupt(unsigned int cpu
)
823 struct hws_cpu_buffer
*cb
;
825 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
827 sdbt
= (unsigned long *) cb
->worker_entry
;
830 /* do not proceed if stop was entered,
831 * forget the buffers not yet processed */
832 while (!done
&& !cb
->stop_mode
) {
833 unsigned long *trailer
;
834 struct hws_trailer_entry
*te
;
835 unsigned long *dear
= 0;
837 trailer
= trailer_entry_ptr(*sdbt
);
838 /* leave loop if no more work to do */
839 if (!(*trailer
& BUFFER_FULL_MASK
)) {
845 te
= (struct hws_trailer_entry
*)trailer
;
846 cb
->sample_overflow
+= te
->overflow
;
848 add_samples_to_oprofile(cpu
, sdbt
, dear
);
851 xchg((unsigned char *) te
, 0x40);
853 /* advance to next sdb slot in current sdbt */
855 /* in case link bit is set use address w/o link bit */
856 if (is_link_entry(sdbt
))
857 sdbt
= get_next_sdbt(sdbt
);
859 cb
->worker_entry
= (unsigned long)sdbt
;
863 static void add_samples_to_oprofile(unsigned int cpu
, unsigned long *sdbt
,
866 struct hws_data_entry
*sample_data_ptr
;
867 unsigned long *trailer
;
869 trailer
= trailer_entry_ptr(*sdbt
);
876 sample_data_ptr
= (struct hws_data_entry
*)(*sdbt
);
878 while ((unsigned long *)sample_data_ptr
< trailer
) {
879 struct pt_regs
*regs
= NULL
;
880 struct task_struct
*tsk
= NULL
;
883 * Check sampling mode, 1 indicates basic (=customer) sampling
886 if (sample_data_ptr
->def
!= 1) {
887 /* sample slot is not yet written */
890 /* make sure we don't use it twice,
891 * the next time the sampler will set it again */
892 sample_data_ptr
->def
= 0;
896 if (sample_data_ptr
->P
== 1) {
897 /* userspace sample */
898 unsigned int pid
= sample_data_ptr
->prim_asn
;
900 tsk
= pid_task(find_vpid(pid
), PIDTYPE_PID
);
902 regs
= task_pt_regs(tsk
);
905 /* kernelspace sample */
906 regs
= task_pt_regs(current
);
909 mutex_lock(&hws_sem
);
910 oprofile_add_ext_hw_sample(sample_data_ptr
->ia
, regs
, 0,
911 !sample_data_ptr
->P
, tsk
);
912 mutex_unlock(&hws_sem
);
918 static void worker(struct work_struct
*work
)
922 struct hws_cpu_buffer
*cb
;
924 cb
= container_of(work
, struct hws_cpu_buffer
, worker
);
925 cpu
= smp_processor_id();
926 ext_params
= atomic_xchg(&cb
->ext_params
, 0);
928 if (!cb
->worker_entry
)
929 worker_on_start(cpu
);
931 if (worker_check_error(cpu
, ext_params
))
935 worker_on_interrupt(cpu
);
938 worker_on_finish(cpu
);
942 * hwsampler_allocate() - allocate memory for the hardware sampler
943 * @sdbt: number of SDBTs per online CPU (must be > 0)
944 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
946 * Returns 0 on success, !0 on failure.
948 int hwsampler_allocate(unsigned long sdbt
, unsigned long sdb
)
951 mutex_lock(&hws_sem
);
954 if (hws_state
!= HWS_DEALLOCATED
)
960 if (sdb
> MAX_NUM_SDB
|| sdb
< MIN_NUM_SDB
)
966 oom_killer_was_active
= 0;
967 register_oom_notifier(&hws_oom_notifier
);
969 for_each_online_cpu(cpu
) {
970 if (allocate_sdbt(cpu
)) {
971 unregister_oom_notifier(&hws_oom_notifier
);
975 unregister_oom_notifier(&hws_oom_notifier
);
976 if (oom_killer_was_active
)
979 hws_state
= HWS_STOPPED
;
983 mutex_unlock(&hws_sem
);
988 printk(KERN_ERR
"hwsampler: CPUMF Memory allocation failed.\n");
993 * hwsampler_deallocate() - deallocate hardware sampler memory
995 * Returns 0 on success, !0 on failure.
997 int hwsampler_deallocate()
1001 mutex_lock(&hws_sem
);
1004 if (hws_state
!= HWS_STOPPED
)
1005 goto deallocate_exit
;
1007 ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1010 hws_state
= HWS_DEALLOCATED
;
1014 mutex_unlock(&hws_sem
);
1019 unsigned long hwsampler_query_min_interval(void)
1021 return min_sampler_rate
;
1024 unsigned long hwsampler_query_max_interval(void)
1026 return max_sampler_rate
;
1029 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu
)
1031 struct hws_cpu_buffer
*cb
;
1033 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
1035 return cb
->sample_overflow
;
1038 int hwsampler_setup()
1042 struct hws_cpu_buffer
*cb
;
1044 mutex_lock(&hws_sem
);
1050 hws_state
= HWS_INIT
;
1052 init_all_cpu_buffers();
1054 rc
= check_hardware_prerequisites();
1058 rc
= check_qsi_on_setup();
1063 hws_wq
= create_workqueue("hwsampler");
1067 register_cpu_notifier(&hws_cpu_notifier
);
1069 for_each_online_cpu(cpu
) {
1070 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
1071 INIT_WORK(&cb
->worker
, worker
);
1072 rc
= smp_ctl_qsi(cpu
);
1074 if (min_sampler_rate
!= cb
->qsi
.min_sampl_rate
) {
1075 if (min_sampler_rate
) {
1077 "hwsampler: different min sampler rate values.\n");
1078 if (min_sampler_rate
< cb
->qsi
.min_sampl_rate
)
1080 cb
->qsi
.min_sampl_rate
;
1082 min_sampler_rate
= cb
->qsi
.min_sampl_rate
;
1084 if (max_sampler_rate
!= cb
->qsi
.max_sampl_rate
) {
1085 if (max_sampler_rate
) {
1087 "hwsampler: different max sampler rate values.\n");
1088 if (max_sampler_rate
> cb
->qsi
.max_sampl_rate
)
1090 cb
->qsi
.max_sampl_rate
;
1092 max_sampler_rate
= cb
->qsi
.max_sampl_rate
;
1095 register_external_interrupt(0x1407, hws_ext_handler
);
1097 hws_state
= HWS_DEALLOCATED
;
1101 mutex_unlock(&hws_sem
);
1105 int hwsampler_shutdown()
1109 mutex_lock(&hws_sem
);
1112 if (hws_state
== HWS_DEALLOCATED
|| hws_state
== HWS_STOPPED
) {
1113 mutex_unlock(&hws_sem
);
1116 flush_workqueue(hws_wq
);
1118 mutex_lock(&hws_sem
);
1120 if (hws_state
== HWS_STOPPED
) {
1121 ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1125 destroy_workqueue(hws_wq
);
1129 unregister_external_interrupt(0x1407, hws_ext_handler
);
1130 hws_state
= HWS_INIT
;
1133 mutex_unlock(&hws_sem
);
1135 unregister_cpu_notifier(&hws_cpu_notifier
);
1141 * hwsampler_start_all() - start hardware sampling on all online CPUs
1142 * @rate: specifies the used interval when samples are taken
1144 * Returns 0 on success, !0 on failure.
1146 int hwsampler_start_all(unsigned long rate
)
1150 mutex_lock(&hws_sem
);
1155 if (hws_state
!= HWS_STOPPED
)
1156 goto start_all_exit
;
1160 /* fail if rate is not valid */
1161 if (interval
< min_sampler_rate
|| interval
> max_sampler_rate
)
1162 goto start_all_exit
;
1164 rc
= check_qsi_on_start();
1166 goto start_all_exit
;
1168 rc
= prepare_cpu_buffers();
1170 goto start_all_exit
;
1172 for_each_online_cpu(cpu
) {
1173 rc
= start_sampling(cpu
);
1178 for_each_online_cpu(cpu
) {
1181 goto start_all_exit
;
1183 hws_state
= HWS_STARTED
;
1187 mutex_unlock(&hws_sem
);
1192 register_oom_notifier(&hws_oom_notifier
);
1195 /* now let them in, 1407 CPUMF external interrupts */
1196 ctl_set_bit(0, 5); /* set CR0 bit 58 */
1202 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1204 * Returns 0 on success, !0 on failure.
1206 int hwsampler_stop_all()
1208 int tmp_rc
, rc
, cpu
;
1209 struct hws_cpu_buffer
*cb
;
1211 mutex_lock(&hws_sem
);
1214 if (hws_state
== HWS_INIT
) {
1215 mutex_unlock(&hws_sem
);
1218 hws_state
= HWS_STOPPING
;
1219 mutex_unlock(&hws_sem
);
1221 for_each_online_cpu(cpu
) {
1222 cb
= &per_cpu(sampler_cpu_buffer
, cpu
);
1224 tmp_rc
= stop_sampling(cpu
);
1230 flush_workqueue(hws_wq
);
1232 mutex_lock(&hws_sem
);
1234 unregister_oom_notifier(&hws_oom_notifier
);
1237 hws_state
= HWS_STOPPED
;
1238 mutex_unlock(&hws_sem
);