]>
Commit | Line | Data |
---|---|---|
a0e60b20 DG |
1 | /* |
2 | * PowerPC atomic bit operations. | |
3 | * | |
4 | * Merged version by David Gibson <david@gibson.dropbear.id.au>. | |
5 | * Based on ppc64 versions by: Dave Engebretsen, Todd Inglett, Don | |
6 | * Reed, Pat McCarthy, Peter Bergner, Anton Blanchard. They | |
7 | * originally took it from the ppc32 code. | |
8 | * | |
9 | * Within a word, bits are numbered LSB first. Lot's of places make | |
10 | * this assumption by directly testing bits with (val & (1<<nr)). | |
11 | * This can cause confusion for large (> 1 word) bitmaps on a | |
12 | * big-endian system because, unlike little endian, the number of each | |
13 | * bit depends on the word size. | |
14 | * | |
15 | * The bitop functions are defined to work on unsigned longs, so for a | |
16 | * ppc64 system the bits end up numbered: | |
17 | * |63..............0|127............64|191...........128|255...........196| | |
18 | * and on ppc32: | |
19 | * |31.....0|63....31|95....64|127...96|159..128|191..160|223..192|255..224| | |
20 | * | |
21 | * There are a few little-endian macros used mostly for filesystem | |
22 | * bitmaps, these work on similar bit arrays layouts, but | |
23 | * byte-oriented: | |
24 | * |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56| | |
25 | * | |
26 | * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit | |
27 | * number field needs to be reversed compared to the big-endian bit | |
28 | * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b). | |
29 | * | |
30 | * This program is free software; you can redistribute it and/or | |
31 | * modify it under the terms of the GNU General Public License | |
32 | * as published by the Free Software Foundation; either version | |
33 | * 2 of the License, or (at your option) any later version. | |
34 | */ | |
35 | ||
36 | #ifndef _ASM_POWERPC_BITOPS_H | |
37 | #define _ASM_POWERPC_BITOPS_H | |
38 | ||
39 | #ifdef __KERNEL__ | |
40 | ||
0624517d JS |
41 | #ifndef _LINUX_BITOPS_H |
42 | #error only <linux/bitops.h> can be included directly | |
43 | #endif | |
44 | ||
a0e60b20 | 45 | #include <linux/compiler.h> |
3ddfbcf1 | 46 | #include <asm/asm-compat.h> |
a0e60b20 DG |
47 | #include <asm/synch.h> |
48 | ||
49 | /* | |
50 | * clear_bit doesn't imply a memory barrier | |
51 | */ | |
52 | #define smp_mb__before_clear_bit() smp_mb() | |
53 | #define smp_mb__after_clear_bit() smp_mb() | |
54 | ||
55 | #define BITOP_MASK(nr) (1UL << ((nr) % BITS_PER_LONG)) | |
56 | #define BITOP_WORD(nr) ((nr) / BITS_PER_LONG) | |
57 | #define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) | |
58 | ||
a0e60b20 DG |
59 | static __inline__ void set_bit(int nr, volatile unsigned long *addr) |
60 | { | |
61 | unsigned long old; | |
62 | unsigned long mask = BITOP_MASK(nr); | |
63 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
64 | ||
65 | __asm__ __volatile__( | |
3ddfbcf1 | 66 | "1:" PPC_LLARX "%0,0,%3 # set_bit\n" |
a0e60b20 DG |
67 | "or %0,%0,%2\n" |
68 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 69 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 | 70 | "bne- 1b" |
e2a3d402 LT |
71 | : "=&r" (old), "+m" (*p) |
72 | : "r" (mask), "r" (p) | |
a0e60b20 DG |
73 | : "cc" ); |
74 | } | |
75 | ||
76 | static __inline__ void clear_bit(int nr, volatile unsigned long *addr) | |
77 | { | |
78 | unsigned long old; | |
79 | unsigned long mask = BITOP_MASK(nr); | |
80 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
81 | ||
82 | __asm__ __volatile__( | |
3ddfbcf1 | 83 | "1:" PPC_LLARX "%0,0,%3 # clear_bit\n" |
a0e60b20 DG |
84 | "andc %0,%0,%2\n" |
85 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 86 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 | 87 | "bne- 1b" |
e2a3d402 LT |
88 | : "=&r" (old), "+m" (*p) |
89 | : "r" (mask), "r" (p) | |
a0e60b20 DG |
90 | : "cc" ); |
91 | } | |
92 | ||
66ffb04c NP |
93 | static __inline__ void clear_bit_unlock(int nr, volatile unsigned long *addr) |
94 | { | |
95 | unsigned long old; | |
96 | unsigned long mask = BITOP_MASK(nr); | |
97 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
98 | ||
99 | __asm__ __volatile__( | |
100 | LWSYNC_ON_SMP | |
101 | "1:" PPC_LLARX "%0,0,%3 # clear_bit_unlock\n" | |
102 | "andc %0,%0,%2\n" | |
103 | PPC405_ERR77(0,%3) | |
104 | PPC_STLCX "%0,0,%3\n" | |
105 | "bne- 1b" | |
106 | : "=&r" (old), "+m" (*p) | |
107 | : "r" (mask), "r" (p) | |
108 | : "cc", "memory"); | |
109 | } | |
110 | ||
a0e60b20 DG |
111 | static __inline__ void change_bit(int nr, volatile unsigned long *addr) |
112 | { | |
113 | unsigned long old; | |
114 | unsigned long mask = BITOP_MASK(nr); | |
115 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
116 | ||
117 | __asm__ __volatile__( | |
3ddfbcf1 | 118 | "1:" PPC_LLARX "%0,0,%3 # change_bit\n" |
a0e60b20 DG |
119 | "xor %0,%0,%2\n" |
120 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 121 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 | 122 | "bne- 1b" |
e2a3d402 LT |
123 | : "=&r" (old), "+m" (*p) |
124 | : "r" (mask), "r" (p) | |
a0e60b20 DG |
125 | : "cc" ); |
126 | } | |
127 | ||
128 | static __inline__ int test_and_set_bit(unsigned long nr, | |
129 | volatile unsigned long *addr) | |
130 | { | |
131 | unsigned long old, t; | |
132 | unsigned long mask = BITOP_MASK(nr); | |
133 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
134 | ||
135 | __asm__ __volatile__( | |
144b9c13 | 136 | LWSYNC_ON_SMP |
3ddfbcf1 | 137 | "1:" PPC_LLARX "%0,0,%3 # test_and_set_bit\n" |
a0e60b20 DG |
138 | "or %1,%0,%2 \n" |
139 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 140 | PPC_STLCX "%1,0,%3 \n" |
a0e60b20 DG |
141 | "bne- 1b" |
142 | ISYNC_ON_SMP | |
143 | : "=&r" (old), "=&r" (t) | |
144 | : "r" (mask), "r" (p) | |
145 | : "cc", "memory"); | |
146 | ||
147 | return (old & mask) != 0; | |
148 | } | |
149 | ||
66ffb04c NP |
150 | static __inline__ int test_and_set_bit_lock(unsigned long nr, |
151 | volatile unsigned long *addr) | |
152 | { | |
153 | unsigned long old, t; | |
154 | unsigned long mask = BITOP_MASK(nr); | |
155 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
156 | ||
157 | __asm__ __volatile__( | |
158 | "1:" PPC_LLARX "%0,0,%3 # test_and_set_bit_lock\n" | |
159 | "or %1,%0,%2 \n" | |
160 | PPC405_ERR77(0,%3) | |
161 | PPC_STLCX "%1,0,%3 \n" | |
162 | "bne- 1b" | |
163 | ISYNC_ON_SMP | |
164 | : "=&r" (old), "=&r" (t) | |
165 | : "r" (mask), "r" (p) | |
166 | : "cc", "memory"); | |
167 | ||
168 | return (old & mask) != 0; | |
169 | } | |
170 | ||
a0e60b20 DG |
171 | static __inline__ int test_and_clear_bit(unsigned long nr, |
172 | volatile unsigned long *addr) | |
173 | { | |
174 | unsigned long old, t; | |
175 | unsigned long mask = BITOP_MASK(nr); | |
176 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
177 | ||
178 | __asm__ __volatile__( | |
144b9c13 | 179 | LWSYNC_ON_SMP |
3ddfbcf1 | 180 | "1:" PPC_LLARX "%0,0,%3 # test_and_clear_bit\n" |
a0e60b20 DG |
181 | "andc %1,%0,%2 \n" |
182 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 183 | PPC_STLCX "%1,0,%3 \n" |
a0e60b20 DG |
184 | "bne- 1b" |
185 | ISYNC_ON_SMP | |
186 | : "=&r" (old), "=&r" (t) | |
187 | : "r" (mask), "r" (p) | |
188 | : "cc", "memory"); | |
189 | ||
190 | return (old & mask) != 0; | |
191 | } | |
192 | ||
193 | static __inline__ int test_and_change_bit(unsigned long nr, | |
194 | volatile unsigned long *addr) | |
195 | { | |
196 | unsigned long old, t; | |
197 | unsigned long mask = BITOP_MASK(nr); | |
198 | unsigned long *p = ((unsigned long *)addr) + BITOP_WORD(nr); | |
199 | ||
200 | __asm__ __volatile__( | |
144b9c13 | 201 | LWSYNC_ON_SMP |
3ddfbcf1 | 202 | "1:" PPC_LLARX "%0,0,%3 # test_and_change_bit\n" |
a0e60b20 DG |
203 | "xor %1,%0,%2 \n" |
204 | PPC405_ERR77(0,%3) | |
3ddfbcf1 | 205 | PPC_STLCX "%1,0,%3 \n" |
a0e60b20 DG |
206 | "bne- 1b" |
207 | ISYNC_ON_SMP | |
208 | : "=&r" (old), "=&r" (t) | |
209 | : "r" (mask), "r" (p) | |
210 | : "cc", "memory"); | |
211 | ||
212 | return (old & mask) != 0; | |
213 | } | |
214 | ||
215 | static __inline__ void set_bits(unsigned long mask, unsigned long *addr) | |
216 | { | |
217 | unsigned long old; | |
218 | ||
219 | __asm__ __volatile__( | |
3ddfbcf1 | 220 | "1:" PPC_LLARX "%0,0,%3 # set_bits\n" |
a0e60b20 | 221 | "or %0,%0,%2\n" |
3ddfbcf1 | 222 | PPC_STLCX "%0,0,%3\n" |
a0e60b20 | 223 | "bne- 1b" |
e2a3d402 LT |
224 | : "=&r" (old), "+m" (*addr) |
225 | : "r" (mask), "r" (addr) | |
a0e60b20 DG |
226 | : "cc"); |
227 | } | |
228 | ||
e779b2f9 | 229 | #include <asm-generic/bitops/non-atomic.h> |
a0e60b20 | 230 | |
66ffb04c NP |
231 | static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr) |
232 | { | |
233 | __asm__ __volatile__(LWSYNC_ON_SMP "" ::: "memory"); | |
234 | __clear_bit(nr, addr); | |
235 | } | |
236 | ||
a0e60b20 DG |
237 | /* |
238 | * Return the zero-based bit position (LE, not IBM bit numbering) of | |
239 | * the most significant 1-bit in a double word. | |
240 | */ | |
ef55d53c DH |
241 | static __inline__ __attribute__((const)) |
242 | int __ilog2(unsigned long x) | |
a0e60b20 DG |
243 | { |
244 | int lz; | |
245 | ||
3ddfbcf1 | 246 | asm (PPC_CNTLZL "%0,%1" : "=r" (lz) : "r" (x)); |
a0e60b20 DG |
247 | return BITS_PER_LONG - 1 - lz; |
248 | } | |
249 | ||
ef55d53c DH |
250 | static inline __attribute__((const)) |
251 | int __ilog2_u32(u32 n) | |
252 | { | |
253 | int bit; | |
254 | asm ("cntlzw %0,%1" : "=r" (bit) : "r" (n)); | |
255 | return 31 - bit; | |
256 | } | |
257 | ||
258 | #ifdef __powerpc64__ | |
259 | static inline __attribute__((const)) | |
02241696 | 260 | int __ilog2_u64(u64 n) |
ef55d53c DH |
261 | { |
262 | int bit; | |
263 | asm ("cntlzd %0,%1" : "=r" (bit) : "r" (n)); | |
264 | return 63 - bit; | |
265 | } | |
266 | #endif | |
267 | ||
a0e60b20 DG |
268 | /* |
269 | * Determines the bit position of the least significant 0 bit in the | |
270 | * specified double word. The returned bit position will be | |
271 | * zero-based, starting from the right side (63/31 - 0). | |
272 | */ | |
273 | static __inline__ unsigned long ffz(unsigned long x) | |
274 | { | |
275 | /* no zero exists anywhere in the 8 byte area. */ | |
276 | if ((x = ~x) == 0) | |
277 | return BITS_PER_LONG; | |
278 | ||
279 | /* | |
280 | * Calculate the bit position of the least signficant '1' bit in x | |
281 | * (since x has been changed this will actually be the least signficant | |
282 | * '0' bit in * the original x). Note: (x & -x) gives us a mask that | |
283 | * is the least significant * (RIGHT-most) 1-bit of the value in x. | |
284 | */ | |
285 | return __ilog2(x & -x); | |
286 | } | |
287 | ||
288 | static __inline__ int __ffs(unsigned long x) | |
289 | { | |
290 | return __ilog2(x & -x); | |
291 | } | |
292 | ||
293 | /* | |
294 | * ffs: find first bit set. This is defined the same way as | |
295 | * the libc and compiler builtin ffs routines, therefore | |
296 | * differs in spirit from the above ffz (man ffs). | |
297 | */ | |
298 | static __inline__ int ffs(int x) | |
299 | { | |
300 | unsigned long i = (unsigned long)x; | |
301 | return __ilog2(i & -i) + 1; | |
302 | } | |
303 | ||
304 | /* | |
305 | * fls: find last (most-significant) bit set. | |
306 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. | |
307 | */ | |
308 | static __inline__ int fls(unsigned int x) | |
309 | { | |
310 | int lz; | |
311 | ||
312 | asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x)); | |
313 | return 32 - lz; | |
314 | } | |
9f264be6 | 315 | |
56a6b1eb AH |
316 | static __inline__ unsigned long __fls(unsigned long x) |
317 | { | |
318 | return __ilog2(x); | |
319 | } | |
320 | ||
9f264be6 PM |
321 | /* |
322 | * 64-bit can do this using one cntlzd (count leading zeroes doubleword) | |
323 | * instruction; for 32-bit we use the generic version, which does two | |
324 | * 32-bit fls calls. | |
325 | */ | |
326 | #ifdef __powerpc64__ | |
327 | static __inline__ int fls64(__u64 x) | |
328 | { | |
329 | int lz; | |
330 | ||
331 | asm ("cntlzd %0,%1" : "=r" (lz) : "r" (x)); | |
332 | return 64 - lz; | |
333 | } | |
334 | #else | |
e779b2f9 | 335 | #include <asm-generic/bitops/fls64.h> |
9f264be6 PM |
336 | #endif /* __powerpc64__ */ |
337 | ||
e779b2f9 | 338 | #include <asm-generic/bitops/hweight.h> |
47b9d9bd | 339 | #include <asm-generic/bitops/find.h> |
a0e60b20 DG |
340 | |
341 | /* Little-endian versions */ | |
342 | ||
343 | static __inline__ int test_le_bit(unsigned long nr, | |
344 | __const__ unsigned long *addr) | |
345 | { | |
346 | __const__ unsigned char *tmp = (__const__ unsigned char *) addr; | |
347 | return (tmp[nr >> 3] >> (nr & 7)) & 1; | |
348 | } | |
349 | ||
350 | #define __set_le_bit(nr, addr) \ | |
351 | __set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
352 | #define __clear_le_bit(nr, addr) \ | |
353 | __clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
354 | ||
355 | #define test_and_set_le_bit(nr, addr) \ | |
356 | test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
357 | #define test_and_clear_le_bit(nr, addr) \ | |
358 | test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
359 | ||
360 | #define __test_and_set_le_bit(nr, addr) \ | |
361 | __test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
362 | #define __test_and_clear_le_bit(nr, addr) \ | |
363 | __test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, (addr)) | |
364 | ||
0a9cb46a JM |
365 | #define find_first_zero_le_bit(addr, size) generic_find_next_zero_le_bit((addr), (size), 0) |
366 | unsigned long generic_find_next_zero_le_bit(const unsigned long *addr, | |
a0e60b20 DG |
367 | unsigned long size, unsigned long offset); |
368 | ||
aa02ad67 AK |
369 | unsigned long generic_find_next_le_bit(const unsigned long *addr, |
370 | unsigned long size, unsigned long offset); | |
a0e60b20 DG |
371 | /* Bitmap functions for the ext2 filesystem */ |
372 | ||
373 | #define ext2_set_bit(nr,addr) \ | |
374 | __test_and_set_le_bit((nr), (unsigned long*)addr) | |
375 | #define ext2_clear_bit(nr, addr) \ | |
376 | __test_and_clear_le_bit((nr), (unsigned long*)addr) | |
377 | ||
378 | #define ext2_set_bit_atomic(lock, nr, addr) \ | |
379 | test_and_set_le_bit((nr), (unsigned long*)addr) | |
380 | #define ext2_clear_bit_atomic(lock, nr, addr) \ | |
381 | test_and_clear_le_bit((nr), (unsigned long*)addr) | |
382 | ||
383 | #define ext2_test_bit(nr, addr) test_le_bit((nr),(unsigned long*)addr) | |
384 | ||
385 | #define ext2_find_first_zero_bit(addr, size) \ | |
386 | find_first_zero_le_bit((unsigned long*)addr, size) | |
387 | #define ext2_find_next_zero_bit(addr, size, off) \ | |
0a9cb46a | 388 | generic_find_next_zero_le_bit((unsigned long*)addr, size, off) |
a0e60b20 | 389 | |
aa02ad67 AK |
390 | #define ext2_find_next_bit(addr, size, off) \ |
391 | generic_find_next_le_bit((unsigned long *)addr, size, off) | |
a0e60b20 DG |
392 | /* Bitmap functions for the minix filesystem. */ |
393 | ||
394 | #define minix_test_and_set_bit(nr,addr) \ | |
395 | __test_and_set_le_bit(nr, (unsigned long *)addr) | |
396 | #define minix_set_bit(nr,addr) \ | |
397 | __set_le_bit(nr, (unsigned long *)addr) | |
398 | #define minix_test_and_clear_bit(nr,addr) \ | |
399 | __test_and_clear_le_bit(nr, (unsigned long *)addr) | |
400 | #define minix_test_bit(nr,addr) \ | |
401 | test_le_bit(nr, (unsigned long *)addr) | |
402 | ||
403 | #define minix_find_first_zero_bit(addr,size) \ | |
404 | find_first_zero_le_bit((unsigned long *)addr, size) | |
405 | ||
e779b2f9 | 406 | #include <asm-generic/bitops/sched.h> |
a0e60b20 DG |
407 | |
408 | #endif /* __KERNEL__ */ | |
409 | ||
410 | #endif /* _ASM_POWERPC_BITOPS_H */ |