]>
Commit | Line | Data |
---|---|---|
d5de8841 JF |
1 | /* |
2 | * Split spinlock implementation out into its own file, so it can be | |
3 | * compiled in a FTRACE-compatible way. | |
4 | */ | |
5 | #include <linux/kernel_stat.h> | |
6 | #include <linux/spinlock.h> | |
994025ca JF |
7 | #include <linux/debugfs.h> |
8 | #include <linux/log2.h> | |
d5de8841 JF |
9 | |
10 | #include <asm/paravirt.h> | |
11 | ||
12 | #include <xen/interface/xen.h> | |
13 | #include <xen/events.h> | |
14 | ||
15 | #include "xen-ops.h" | |
994025ca JF |
16 | #include "debugfs.h" |
17 | ||
18 | #ifdef CONFIG_XEN_DEBUG_FS | |
19 | static struct xen_spinlock_stats | |
20 | { | |
21 | u64 taken; | |
22 | u32 taken_slow; | |
23 | u32 taken_slow_nested; | |
24 | u32 taken_slow_pickup; | |
25 | u32 taken_slow_spurious; | |
1e696f63 | 26 | u32 taken_slow_irqenable; |
994025ca JF |
27 | |
28 | u64 released; | |
29 | u32 released_slow; | |
30 | u32 released_slow_kicked; | |
31 | ||
f8eca41f JF |
32 | #define HISTO_BUCKETS 30 |
33 | u32 histo_spin_total[HISTO_BUCKETS+1]; | |
34 | u32 histo_spin_spinning[HISTO_BUCKETS+1]; | |
35 | u32 histo_spin_blocked[HISTO_BUCKETS+1]; | |
36 | ||
37 | u64 time_total; | |
38 | u64 time_spinning; | |
39 | u64 time_blocked; | |
994025ca JF |
40 | } spinlock_stats; |
41 | ||
42 | static u8 zero_stats; | |
43 | ||
44 | static unsigned lock_timeout = 1 << 10; | |
45 | #define TIMEOUT lock_timeout | |
46 | ||
47 | static inline void check_zero(void) | |
48 | { | |
49 | if (unlikely(zero_stats)) { | |
50 | memset(&spinlock_stats, 0, sizeof(spinlock_stats)); | |
51 | zero_stats = 0; | |
52 | } | |
53 | } | |
54 | ||
55 | #define ADD_STATS(elem, val) \ | |
56 | do { check_zero(); spinlock_stats.elem += (val); } while(0) | |
57 | ||
58 | static inline u64 spin_time_start(void) | |
59 | { | |
60 | return xen_clocksource_read(); | |
61 | } | |
62 | ||
63 | static void __spin_time_accum(u64 delta, u32 *array) | |
64 | { | |
65 | unsigned index = ilog2(delta); | |
66 | ||
67 | check_zero(); | |
68 | ||
69 | if (index < HISTO_BUCKETS) | |
70 | array[index]++; | |
71 | else | |
72 | array[HISTO_BUCKETS]++; | |
73 | } | |
74 | ||
f8eca41f JF |
75 | static inline void spin_time_accum_spinning(u64 start) |
76 | { | |
77 | u32 delta = xen_clocksource_read() - start; | |
78 | ||
79 | __spin_time_accum(delta, spinlock_stats.histo_spin_spinning); | |
80 | spinlock_stats.time_spinning += delta; | |
81 | } | |
82 | ||
83 | static inline void spin_time_accum_total(u64 start) | |
994025ca JF |
84 | { |
85 | u32 delta = xen_clocksource_read() - start; | |
86 | ||
f8eca41f JF |
87 | __spin_time_accum(delta, spinlock_stats.histo_spin_total); |
88 | spinlock_stats.time_total += delta; | |
994025ca JF |
89 | } |
90 | ||
f8eca41f | 91 | static inline void spin_time_accum_blocked(u64 start) |
994025ca JF |
92 | { |
93 | u32 delta = xen_clocksource_read() - start; | |
94 | ||
f8eca41f JF |
95 | __spin_time_accum(delta, spinlock_stats.histo_spin_blocked); |
96 | spinlock_stats.time_blocked += delta; | |
994025ca JF |
97 | } |
98 | #else /* !CONFIG_XEN_DEBUG_FS */ | |
99 | #define TIMEOUT (1 << 10) | |
100 | #define ADD_STATS(elem, val) do { (void)(val); } while(0) | |
101 | ||
102 | static inline u64 spin_time_start(void) | |
103 | { | |
104 | return 0; | |
105 | } | |
106 | ||
f8eca41f JF |
107 | static inline void spin_time_accum_total(u64 start) |
108 | { | |
109 | } | |
110 | static inline void spin_time_accum_spinning(u64 start) | |
994025ca JF |
111 | { |
112 | } | |
f8eca41f | 113 | static inline void spin_time_accum_blocked(u64 start) |
994025ca JF |
114 | { |
115 | } | |
116 | #endif /* CONFIG_XEN_DEBUG_FS */ | |
d5de8841 JF |
117 | |
118 | struct xen_spinlock { | |
119 | unsigned char lock; /* 0 -> free; 1 -> locked */ | |
120 | unsigned short spinners; /* count of waiting cpus */ | |
121 | }; | |
122 | ||
123 | static int xen_spin_is_locked(struct raw_spinlock *lock) | |
124 | { | |
125 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
126 | ||
127 | return xl->lock != 0; | |
128 | } | |
129 | ||
130 | static int xen_spin_is_contended(struct raw_spinlock *lock) | |
131 | { | |
132 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
133 | ||
134 | /* Not strictly true; this is only the count of contended | |
135 | lock-takers entering the slow path. */ | |
136 | return xl->spinners != 0; | |
137 | } | |
138 | ||
139 | static int xen_spin_trylock(struct raw_spinlock *lock) | |
140 | { | |
141 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
142 | u8 old = 1; | |
143 | ||
144 | asm("xchgb %b0,%1" | |
145 | : "+q" (old), "+m" (xl->lock) : : "memory"); | |
146 | ||
147 | return old == 0; | |
148 | } | |
149 | ||
150 | static DEFINE_PER_CPU(int, lock_kicker_irq) = -1; | |
151 | static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners); | |
152 | ||
168d2f46 JF |
153 | /* |
154 | * Mark a cpu as interested in a lock. Returns the CPU's previous | |
155 | * lock of interest, in case we got preempted by an interrupt. | |
156 | */ | |
157 | static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl) | |
d5de8841 | 158 | { |
168d2f46 JF |
159 | struct xen_spinlock *prev; |
160 | ||
161 | prev = __get_cpu_var(lock_spinners); | |
d5de8841 | 162 | __get_cpu_var(lock_spinners) = xl; |
168d2f46 | 163 | |
d5de8841 | 164 | wmb(); /* set lock of interest before count */ |
168d2f46 | 165 | |
d5de8841 JF |
166 | asm(LOCK_PREFIX " incw %0" |
167 | : "+m" (xl->spinners) : : "memory"); | |
168d2f46 JF |
168 | |
169 | return prev; | |
d5de8841 JF |
170 | } |
171 | ||
168d2f46 JF |
172 | /* |
173 | * Mark a cpu as no longer interested in a lock. Restores previous | |
174 | * lock of interest (NULL for none). | |
175 | */ | |
176 | static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev) | |
d5de8841 JF |
177 | { |
178 | asm(LOCK_PREFIX " decw %0" | |
179 | : "+m" (xl->spinners) : : "memory"); | |
168d2f46 JF |
180 | wmb(); /* decrement count before restoring lock */ |
181 | __get_cpu_var(lock_spinners) = prev; | |
d5de8841 JF |
182 | } |
183 | ||
1e696f63 | 184 | static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enable) |
d5de8841 JF |
185 | { |
186 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
168d2f46 | 187 | struct xen_spinlock *prev; |
d5de8841 JF |
188 | int irq = __get_cpu_var(lock_kicker_irq); |
189 | int ret; | |
f8eca41f | 190 | u64 start; |
d5de8841 JF |
191 | |
192 | /* If kicker interrupts not initialized yet, just spin */ | |
193 | if (irq == -1) | |
194 | return 0; | |
195 | ||
f8eca41f JF |
196 | start = spin_time_start(); |
197 | ||
d5de8841 | 198 | /* announce we're spinning */ |
168d2f46 | 199 | prev = spinning_lock(xl); |
d5de8841 | 200 | |
994025ca JF |
201 | ADD_STATS(taken_slow, 1); |
202 | ADD_STATS(taken_slow_nested, prev != NULL); | |
203 | ||
168d2f46 | 204 | do { |
4d576b57 JF |
205 | unsigned long flags; |
206 | ||
168d2f46 JF |
207 | /* clear pending */ |
208 | xen_clear_irq_pending(irq); | |
209 | ||
210 | /* check again make sure it didn't become free while | |
211 | we weren't looking */ | |
212 | ret = xen_spin_trylock(lock); | |
213 | if (ret) { | |
994025ca JF |
214 | ADD_STATS(taken_slow_pickup, 1); |
215 | ||
168d2f46 JF |
216 | /* |
217 | * If we interrupted another spinlock while it | |
218 | * was blocking, make sure it doesn't block | |
219 | * without rechecking the lock. | |
220 | */ | |
221 | if (prev != NULL) | |
222 | xen_set_irq_pending(irq); | |
223 | goto out; | |
224 | } | |
d5de8841 | 225 | |
4d576b57 JF |
226 | flags = __raw_local_save_flags(); |
227 | if (irq_enable) { | |
228 | ADD_STATS(taken_slow_irqenable, 1); | |
229 | raw_local_irq_enable(); | |
230 | } | |
231 | ||
168d2f46 JF |
232 | /* |
233 | * Block until irq becomes pending. If we're | |
234 | * interrupted at this point (after the trylock but | |
235 | * before entering the block), then the nested lock | |
236 | * handler guarantees that the irq will be left | |
237 | * pending if there's any chance the lock became free; | |
238 | * xen_poll_irq() returns immediately if the irq is | |
239 | * pending. | |
240 | */ | |
241 | xen_poll_irq(irq); | |
4d576b57 JF |
242 | |
243 | raw_local_irq_restore(flags); | |
244 | ||
994025ca | 245 | ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq)); |
168d2f46 | 246 | } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */ |
d5de8841 | 247 | |
d6c88a50 | 248 | kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); |
d5de8841 JF |
249 | |
250 | out: | |
168d2f46 | 251 | unspinning_lock(xl, prev); |
f8eca41f JF |
252 | spin_time_accum_blocked(start); |
253 | ||
d5de8841 JF |
254 | return ret; |
255 | } | |
256 | ||
1e696f63 | 257 | static inline void __xen_spin_lock(struct raw_spinlock *lock, bool irq_enable) |
d5de8841 JF |
258 | { |
259 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
994025ca | 260 | unsigned timeout; |
d5de8841 | 261 | u8 oldval; |
994025ca JF |
262 | u64 start_spin; |
263 | ||
264 | ADD_STATS(taken, 1); | |
265 | ||
266 | start_spin = spin_time_start(); | |
d5de8841 JF |
267 | |
268 | do { | |
994025ca JF |
269 | u64 start_spin_fast = spin_time_start(); |
270 | ||
271 | timeout = TIMEOUT; | |
d5de8841 JF |
272 | |
273 | asm("1: xchgb %1,%0\n" | |
274 | " testb %1,%1\n" | |
275 | " jz 3f\n" | |
276 | "2: rep;nop\n" | |
277 | " cmpb $0,%0\n" | |
278 | " je 1b\n" | |
279 | " dec %2\n" | |
280 | " jnz 2b\n" | |
281 | "3:\n" | |
282 | : "+m" (xl->lock), "=q" (oldval), "+r" (timeout) | |
283 | : "1" (1) | |
284 | : "memory"); | |
285 | ||
f8eca41f | 286 | spin_time_accum_spinning(start_spin_fast); |
1e696f63 JF |
287 | |
288 | } while (unlikely(oldval != 0 && | |
289 | (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable)))); | |
994025ca | 290 | |
f8eca41f | 291 | spin_time_accum_total(start_spin); |
d5de8841 JF |
292 | } |
293 | ||
1e696f63 JF |
294 | static void xen_spin_lock(struct raw_spinlock *lock) |
295 | { | |
296 | __xen_spin_lock(lock, false); | |
297 | } | |
298 | ||
299 | static void xen_spin_lock_flags(struct raw_spinlock *lock, unsigned long flags) | |
300 | { | |
301 | __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags)); | |
302 | } | |
303 | ||
d5de8841 JF |
304 | static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) |
305 | { | |
306 | int cpu; | |
307 | ||
994025ca JF |
308 | ADD_STATS(released_slow, 1); |
309 | ||
d5de8841 JF |
310 | for_each_online_cpu(cpu) { |
311 | /* XXX should mix up next cpu selection */ | |
312 | if (per_cpu(lock_spinners, cpu) == xl) { | |
994025ca | 313 | ADD_STATS(released_slow_kicked, 1); |
d5de8841 JF |
314 | xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR); |
315 | break; | |
316 | } | |
317 | } | |
318 | } | |
319 | ||
320 | static void xen_spin_unlock(struct raw_spinlock *lock) | |
321 | { | |
322 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | |
323 | ||
994025ca JF |
324 | ADD_STATS(released, 1); |
325 | ||
d5de8841 JF |
326 | smp_wmb(); /* make sure no writes get moved after unlock */ |
327 | xl->lock = 0; /* release lock */ | |
328 | ||
329 | /* make sure unlock happens before kick */ | |
330 | barrier(); | |
331 | ||
332 | if (unlikely(xl->spinners)) | |
333 | xen_spin_unlock_slow(xl); | |
334 | } | |
335 | ||
336 | static irqreturn_t dummy_handler(int irq, void *dev_id) | |
337 | { | |
338 | BUG(); | |
339 | return IRQ_HANDLED; | |
340 | } | |
341 | ||
342 | void __cpuinit xen_init_lock_cpu(int cpu) | |
343 | { | |
344 | int irq; | |
345 | const char *name; | |
346 | ||
347 | name = kasprintf(GFP_KERNEL, "spinlock%d", cpu); | |
348 | irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR, | |
349 | cpu, | |
350 | dummy_handler, | |
351 | IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, | |
352 | name, | |
353 | NULL); | |
354 | ||
355 | if (irq >= 0) { | |
356 | disable_irq(irq); /* make sure it's never delivered */ | |
357 | per_cpu(lock_kicker_irq, cpu) = irq; | |
358 | } | |
359 | ||
360 | printk("cpu %d spinlock event irq %d\n", cpu, irq); | |
361 | } | |
362 | ||
d68d82af AN |
363 | void xen_uninit_lock_cpu(int cpu) |
364 | { | |
365 | unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL); | |
366 | } | |
367 | ||
d5de8841 JF |
368 | void __init xen_init_spinlocks(void) |
369 | { | |
370 | pv_lock_ops.spin_is_locked = xen_spin_is_locked; | |
371 | pv_lock_ops.spin_is_contended = xen_spin_is_contended; | |
372 | pv_lock_ops.spin_lock = xen_spin_lock; | |
1e696f63 | 373 | pv_lock_ops.spin_lock_flags = xen_spin_lock_flags; |
d5de8841 JF |
374 | pv_lock_ops.spin_trylock = xen_spin_trylock; |
375 | pv_lock_ops.spin_unlock = xen_spin_unlock; | |
376 | } | |
994025ca JF |
377 | |
378 | #ifdef CONFIG_XEN_DEBUG_FS | |
379 | ||
380 | static struct dentry *d_spin_debug; | |
381 | ||
382 | static int __init xen_spinlock_debugfs(void) | |
383 | { | |
384 | struct dentry *d_xen = xen_init_debugfs(); | |
385 | ||
386 | if (d_xen == NULL) | |
387 | return -ENOMEM; | |
388 | ||
389 | d_spin_debug = debugfs_create_dir("spinlocks", d_xen); | |
390 | ||
391 | debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats); | |
392 | ||
393 | debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout); | |
394 | ||
395 | debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken); | |
396 | debugfs_create_u32("taken_slow", 0444, d_spin_debug, | |
397 | &spinlock_stats.taken_slow); | |
398 | debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug, | |
399 | &spinlock_stats.taken_slow_nested); | |
400 | debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug, | |
401 | &spinlock_stats.taken_slow_pickup); | |
402 | debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, | |
403 | &spinlock_stats.taken_slow_spurious); | |
1e696f63 JF |
404 | debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug, |
405 | &spinlock_stats.taken_slow_irqenable); | |
994025ca JF |
406 | |
407 | debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released); | |
408 | debugfs_create_u32("released_slow", 0444, d_spin_debug, | |
409 | &spinlock_stats.released_slow); | |
410 | debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug, | |
411 | &spinlock_stats.released_slow_kicked); | |
412 | ||
413 | debugfs_create_u64("time_spinning", 0444, d_spin_debug, | |
f8eca41f JF |
414 | &spinlock_stats.time_spinning); |
415 | debugfs_create_u64("time_blocked", 0444, d_spin_debug, | |
416 | &spinlock_stats.time_blocked); | |
994025ca | 417 | debugfs_create_u64("time_total", 0444, d_spin_debug, |
f8eca41f | 418 | &spinlock_stats.time_total); |
994025ca JF |
419 | |
420 | xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug, | |
f8eca41f | 421 | spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1); |
994025ca | 422 | xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug, |
f8eca41f JF |
423 | spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1); |
424 | xen_debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug, | |
425 | spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1); | |
994025ca JF |
426 | |
427 | return 0; | |
428 | } | |
429 | fs_initcall(xen_spinlock_debugfs); | |
430 | ||
431 | #endif /* CONFIG_XEN_DEBUG_FS */ |