2 * Fast batching percpu counters.
5 #include <linux/percpu_counter.h>
6 #include <linux/notifier.h>
7 #include <linux/mutex.h>
8 #include <linux/init.h>
10 #include <linux/module.h>
12 #ifdef CONFIG_HOTPLUG_CPU
13 static LIST_HEAD(percpu_counters
);
14 static DEFINE_MUTEX(percpu_counters_lock
);
17 void __percpu_counter_add(struct percpu_counter
*fbc
, s32 amount
, s32 batch
)
23 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
24 count
= *pcount
+ amount
;
25 if (count
>= batch
|| count
<= -batch
) {
26 spin_lock(&fbc
->lock
);
29 spin_unlock(&fbc
->lock
);
35 EXPORT_SYMBOL(__percpu_counter_add
);
38 * Add up all the per-cpu counts, return the result. This is a more accurate
39 * but much slower version of percpu_counter_read_positive()
41 s64
percpu_counter_sum(struct percpu_counter
*fbc
)
46 spin_lock(&fbc
->lock
);
48 for_each_online_cpu(cpu
) {
49 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
52 spin_unlock(&fbc
->lock
);
53 return ret
< 0 ? 0 : ret
;
55 EXPORT_SYMBOL(percpu_counter_sum
);
57 void percpu_counter_init(struct percpu_counter
*fbc
, s64 amount
)
59 spin_lock_init(&fbc
->lock
);
61 fbc
->counters
= alloc_percpu(s32
);
62 #ifdef CONFIG_HOTPLUG_CPU
63 mutex_lock(&percpu_counters_lock
);
64 list_add(&fbc
->list
, &percpu_counters
);
65 mutex_unlock(&percpu_counters_lock
);
68 EXPORT_SYMBOL(percpu_counter_init
);
70 void percpu_counter_destroy(struct percpu_counter
*fbc
)
72 free_percpu(fbc
->counters
);
73 #ifdef CONFIG_HOTPLUG_CPU
74 mutex_lock(&percpu_counters_lock
);
76 mutex_unlock(&percpu_counters_lock
);
79 EXPORT_SYMBOL(percpu_counter_destroy
);
81 #ifdef CONFIG_HOTPLUG_CPU
82 static int __cpuinit
percpu_counter_hotcpu_callback(struct notifier_block
*nb
,
83 unsigned long action
, void *hcpu
)
86 struct percpu_counter
*fbc
;
88 if (action
!= CPU_DEAD
)
91 cpu
= (unsigned long)hcpu
;
92 mutex_lock(&percpu_counters_lock
);
93 list_for_each_entry(fbc
, &percpu_counters
, list
) {
96 spin_lock(&fbc
->lock
);
97 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
98 fbc
->count
+= *pcount
;
100 spin_unlock(&fbc
->lock
);
102 mutex_unlock(&percpu_counters_lock
);
106 static int __init
percpu_counter_startup(void)
108 hotcpu_notifier(percpu_counter_hotcpu_callback
, 0);
111 module_init(percpu_counter_startup
);