]>
Commit | Line | Data |
---|---|---|
951f22d5 | 1 | /* |
951f22d5 MS |
2 | * Out of line spinlock code. |
3 | * | |
a53c8fab | 4 | * Copyright IBM Corp. 2004, 2006 |
951f22d5 MS |
5 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) |
6 | */ | |
7 | ||
8 | #include <linux/types.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/init.h> | |
8b646bd7 | 12 | #include <linux/smp.h> |
951f22d5 MS |
13 | #include <asm/io.h> |
14 | ||
951f22d5 MS |
15 | int spin_retry = 1000; |
16 | ||
17 | /** | |
18 | * spin_retry= parameter | |
19 | */ | |
20 | static int __init spin_retry_setup(char *str) | |
21 | { | |
22 | spin_retry = simple_strtoul(str, &str, 0); | |
23 | return 1; | |
24 | } | |
25 | __setup("spin_retry=", spin_retry_setup); | |
26 | ||
0199c4e6 | 27 | void arch_spin_lock_wait(arch_spinlock_t *lp) |
951f22d5 | 28 | { |
6c8cd5bb | 29 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 30 | unsigned int owner; |
2e4006b3 | 31 | int count; |
951f22d5 MS |
32 | |
33 | while (1) { | |
470ada6b MS |
34 | owner = ACCESS_ONCE(lp->lock); |
35 | /* Try to get the lock if it is free. */ | |
36 | if (!owner) { | |
37 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
38 | return; | |
39 | continue; | |
951f22d5 | 40 | } |
470ada6b MS |
41 | /* Check if the lock owner is running. */ |
42 | if (!smp_vcpu_scheduled(~owner)) { | |
43 | smp_yield_cpu(~owner); | |
44 | continue; | |
45 | } | |
46 | /* Loop for a while on the lock value. */ | |
47 | count = spin_retry; | |
48 | do { | |
49 | owner = ACCESS_ONCE(lp->lock); | |
50 | } while (owner && count-- > 0); | |
51 | if (!owner) | |
52 | continue; | |
53 | /* | |
54 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
55 | * yield the CPU if the lock is still unavailable. | |
56 | */ | |
57 | if (!MACHINE_IS_LPAR) | |
8b646bd7 | 58 | smp_yield_cpu(~owner); |
951f22d5 MS |
59 | } |
60 | } | |
0199c4e6 | 61 | EXPORT_SYMBOL(arch_spin_lock_wait); |
951f22d5 | 62 | |
0199c4e6 | 63 | void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags) |
894cdde2 | 64 | { |
6c8cd5bb | 65 | unsigned int cpu = SPINLOCK_LOCKVAL; |
59b69787 | 66 | unsigned int owner; |
2e4006b3 | 67 | int count; |
894cdde2 HH |
68 | |
69 | local_irq_restore(flags); | |
70 | while (1) { | |
470ada6b MS |
71 | owner = ACCESS_ONCE(lp->lock); |
72 | /* Try to get the lock if it is free. */ | |
73 | if (!owner) { | |
74 | local_irq_disable(); | |
75 | if (_raw_compare_and_swap(&lp->lock, 0, cpu)) | |
76 | return; | |
77 | local_irq_restore(flags); | |
78 | } | |
79 | /* Check if the lock owner is running. */ | |
80 | if (!smp_vcpu_scheduled(~owner)) { | |
81 | smp_yield_cpu(~owner); | |
82 | continue; | |
894cdde2 | 83 | } |
470ada6b MS |
84 | /* Loop for a while on the lock value. */ |
85 | count = spin_retry; | |
86 | do { | |
87 | owner = ACCESS_ONCE(lp->lock); | |
88 | } while (owner && count-- > 0); | |
89 | if (!owner) | |
90 | continue; | |
91 | /* | |
92 | * For multiple layers of hypervisors, e.g. z/VM + LPAR | |
93 | * yield the CPU if the lock is still unavailable. | |
94 | */ | |
95 | if (!MACHINE_IS_LPAR) | |
8b646bd7 | 96 | smp_yield_cpu(~owner); |
894cdde2 HH |
97 | } |
98 | } | |
0199c4e6 | 99 | EXPORT_SYMBOL(arch_spin_lock_wait_flags); |
894cdde2 | 100 | |
0199c4e6 | 101 | int arch_spin_trylock_retry(arch_spinlock_t *lp) |
951f22d5 | 102 | { |
3c1fcfe2 | 103 | int count; |
951f22d5 | 104 | |
bae8f567 | 105 | for (count = spin_retry; count > 0; count--) |
5b3f683e | 106 | if (arch_spin_trylock_once(lp)) |
951f22d5 | 107 | return 1; |
951f22d5 MS |
108 | return 0; |
109 | } | |
0199c4e6 | 110 | EXPORT_SYMBOL(arch_spin_trylock_retry); |
951f22d5 | 111 | |
fb3a6bbc | 112 | void _raw_read_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 113 | { |
d59b93da | 114 | unsigned int owner, old; |
951f22d5 MS |
115 | int count = spin_retry; |
116 | ||
d59b93da | 117 | owner = 0; |
951f22d5 MS |
118 | while (1) { |
119 | if (count-- <= 0) { | |
d59b93da MS |
120 | if (owner && !smp_vcpu_scheduled(~owner)) |
121 | smp_yield_cpu(~owner); | |
951f22d5 MS |
122 | count = spin_retry; |
123 | } | |
bae8f567 | 124 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 125 | owner = ACCESS_ONCE(rw->owner); |
bae8f567 | 126 | if ((int) old < 0) |
96567161 | 127 | continue; |
5b3f683e | 128 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
129 | return; |
130 | } | |
131 | } | |
132 | EXPORT_SYMBOL(_raw_read_lock_wait); | |
133 | ||
fb3a6bbc | 134 | int _raw_read_trylock_retry(arch_rwlock_t *rw) |
951f22d5 MS |
135 | { |
136 | unsigned int old; | |
137 | int count = spin_retry; | |
138 | ||
139 | while (count-- > 0) { | |
bae8f567 MS |
140 | old = ACCESS_ONCE(rw->lock); |
141 | if ((int) old < 0) | |
96567161 | 142 | continue; |
5b3f683e | 143 | if (_raw_compare_and_swap(&rw->lock, old, old + 1)) |
951f22d5 MS |
144 | return 1; |
145 | } | |
146 | return 0; | |
147 | } | |
148 | EXPORT_SYMBOL(_raw_read_trylock_retry); | |
149 | ||
fb3a6bbc | 150 | void _raw_write_lock_wait(arch_rwlock_t *rw) |
951f22d5 | 151 | { |
94232a43 | 152 | unsigned int owner, old, prev; |
951f22d5 MS |
153 | int count = spin_retry; |
154 | ||
94232a43 | 155 | prev = 0x80000000; |
d59b93da | 156 | owner = 0; |
951f22d5 MS |
157 | while (1) { |
158 | if (count-- <= 0) { | |
d59b93da MS |
159 | if (owner && !smp_vcpu_scheduled(~owner)) |
160 | smp_yield_cpu(~owner); | |
951f22d5 MS |
161 | count = spin_retry; |
162 | } | |
bae8f567 | 163 | old = ACCESS_ONCE(rw->lock); |
d59b93da | 164 | owner = ACCESS_ONCE(rw->owner); |
94232a43 MS |
165 | if ((int) old >= 0 && |
166 | _raw_compare_and_swap(&rw->lock, old, old | 0x80000000)) | |
167 | prev = old; | |
168 | else | |
169 | smp_rmb(); | |
170 | if ((old & 0x7fffffff) == 0 && (int) prev >= 0) | |
171 | break; | |
951f22d5 MS |
172 | } |
173 | } | |
174 | EXPORT_SYMBOL(_raw_write_lock_wait); | |
175 | ||
fb3a6bbc | 176 | int _raw_write_trylock_retry(arch_rwlock_t *rw) |
951f22d5 | 177 | { |
bae8f567 | 178 | unsigned int old; |
951f22d5 MS |
179 | int count = spin_retry; |
180 | ||
181 | while (count-- > 0) { | |
bae8f567 MS |
182 | old = ACCESS_ONCE(rw->lock); |
183 | if (old) | |
96567161 | 184 | continue; |
5b3f683e | 185 | if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000)) |
951f22d5 MS |
186 | return 1; |
187 | } | |
188 | return 0; | |
189 | } | |
190 | EXPORT_SYMBOL(_raw_write_trylock_retry); | |
d59b93da MS |
191 | |
192 | void arch_lock_relax(unsigned int cpu) | |
193 | { | |
194 | if (!cpu) | |
195 | return; | |
196 | if (MACHINE_IS_LPAR && smp_vcpu_scheduled(~cpu)) | |
197 | return; | |
198 | smp_yield_cpu(~cpu); | |
199 | } | |
200 | EXPORT_SYMBOL(arch_lock_relax); |