]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
a0e60b20 DG |
2 | /* |
3 | * PowerPC atomic bit operations. | |
4 | * | |
5 | * Merged version by David Gibson <david@gibson.dropbear.id.au>. | |
6 | * Based on ppc64 versions by: Dave Engebretsen, Todd Inglett, Don | |
7 | * Reed, Pat McCarthy, Peter Bergner, Anton Blanchard. They | |
8 | * originally took it from the ppc32 code. | |
9 | * | |
10 | * Within a word, bits are numbered LSB first. Lot's of places make | |
11 | * this assumption by directly testing bits with (val & (1<<nr)). | |
12 | * This can cause confusion for large (> 1 word) bitmaps on a | |
13 | * big-endian system because, unlike little endian, the number of each | |
14 | * bit depends on the word size. | |
15 | * | |
16 | * The bitop functions are defined to work on unsigned longs, so for a | |
17 | * ppc64 system the bits end up numbered: | |
e7a7a65e | 18 | * |63..............0|127............64|191...........128|255...........192| |
a0e60b20 | 19 | * and on ppc32: |
e7a7a65e | 20 | * |31.....0|63....32|95....64|127...96|159..128|191..160|223..192|255..224| |
a0e60b20 DG |
21 | * |
22 | * There are a few little-endian macros used mostly for filesystem | |
23 | * bitmaps, these work on similar bit arrays layouts, but | |
24 | * byte-oriented: | |
25 | * |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56| | |
26 | * | |
27 | * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit | |
28 | * number field needs to be reversed compared to the big-endian bit | |
29 | * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b). | |
a0e60b20 DG |
30 | */ |
31 | ||
32 | #ifndef _ASM_POWERPC_BITOPS_H | |
33 | #define _ASM_POWERPC_BITOPS_H | |
34 | ||
35 | #ifdef __KERNEL__ | |
36 | ||
0624517d JS |
37 | #ifndef _LINUX_BITOPS_H |
38 | #error only <linux/bitops.h> can be included directly | |
39 | #endif | |
40 | ||
a0e60b20 | 41 | #include <linux/compiler.h> |
3ddfbcf1 | 42 | #include <asm/asm-compat.h> |
a0e60b20 DG |
43 | #include <asm/synch.h> |
44 | ||
e22a2274 MS |
45 | /* PPC bit number conversion */ |
46 | #define PPC_BITLSHIFT(be) (BITS_PER_LONG - 1 - (be)) | |
47 | #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) | |
48 | #define PPC_BITMASK(bs, be) ((PPC_BIT(bs) - PPC_BIT(be)) | PPC_BIT(bs)) | |
49 | ||
7b9f71f9 NP |
50 | /* Put a PPC bit into a "normal" bit position */ |
51 | #define PPC_BITEXTRACT(bits, ppc_bit, dst_bit) \ | |
52 | ((((bits) >> PPC_BITLSHIFT(ppc_bit)) & 1) << (dst_bit)) | |
53 | ||
22bd64a6 BH |
54 | #define PPC_BITLSHIFT32(be) (32 - 1 - (be)) |
55 | #define PPC_BIT32(bit) (1UL << PPC_BITLSHIFT32(bit)) | |
56 | #define PPC_BITMASK32(bs, be) ((PPC_BIT32(bs) - PPC_BIT32(be))|PPC_BIT32(bs)) | |
57 | ||
58 | #define PPC_BITLSHIFT8(be) (8 - 1 - (be)) | |
59 | #define PPC_BIT8(bit) (1UL << PPC_BITLSHIFT8(bit)) | |
60 | #define PPC_BITMASK8(bs, be) ((PPC_BIT8(bs) - PPC_BIT8(be))|PPC_BIT8(bs)) | |
61 | ||
c645073f | 62 | #include <asm/barrier.h> |
a0e60b20 | 63 | |
0d2d3e38 | 64 | /* Macro for generating the ***_bits() functions */ |
576be130 | 65 | #define DEFINE_BITOP(fn, op, prefix) \ |
5bece3d6 | 66 | static inline void fn(unsigned long mask, \ |
0d2d3e38 GT |
67 | volatile unsigned long *_p) \ |
68 | { \ | |
69 | unsigned long old; \ | |
70 | unsigned long *p = (unsigned long *)_p; \ | |
71 | __asm__ __volatile__ ( \ | |
72 | prefix \ | |
9401f4e4 | 73 | "1:" PPC_LLARX "%0,0,%3,0\n" \ |
0d2d3e38 | 74 | stringify_in_c(op) "%0,%0,%2\n" \ |
0d2d3e38 GT |
75 | PPC_STLCX "%0,0,%3\n" \ |
76 | "bne- 1b\n" \ | |
0d2d3e38 GT |
77 | : "=&r" (old), "+m" (*p) \ |
78 | : "r" (mask), "r" (p) \ | |
79 | : "cc", "memory"); \ | |
80 | } | |
81 | ||
576be130 ME |
82 | DEFINE_BITOP(set_bits, or, "") |
83 | DEFINE_BITOP(clear_bits, andc, "") | |
84 | DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER) | |
85 | DEFINE_BITOP(change_bits, xor, "") | |
0d2d3e38 | 86 | |
5bece3d6 | 87 | static inline void arch_set_bit(int nr, volatile unsigned long *addr) |
a0e60b20 | 88 | { |
2237f4f4 | 89 | set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); |
a0e60b20 DG |
90 | } |
91 | ||
5bece3d6 | 92 | static inline void arch_clear_bit(int nr, volatile unsigned long *addr) |
a0e60b20 | 93 | { |
2237f4f4 | 94 | clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); |
a0e60b20 DG |
95 | } |
96 | ||
5bece3d6 | 97 | static inline void arch_clear_bit_unlock(int nr, volatile unsigned long *addr) |
66ffb04c | 98 | { |
2237f4f4 | 99 | clear_bits_unlock(BIT_MASK(nr), addr + BIT_WORD(nr)); |
66ffb04c NP |
100 | } |
101 | ||
5bece3d6 | 102 | static inline void arch_change_bit(int nr, volatile unsigned long *addr) |
a0e60b20 | 103 | { |
2237f4f4 | 104 | change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); |
0d2d3e38 | 105 | } |
a0e60b20 | 106 | |
0d2d3e38 GT |
107 | /* Like DEFINE_BITOP(), with changes to the arguments to 'op' and the output |
108 | * operands. */ | |
864b9e6f | 109 | #define DEFINE_TESTOP(fn, op, prefix, postfix, eh) \ |
5bece3d6 | 110 | static inline unsigned long fn( \ |
864b9e6f AB |
111 | unsigned long mask, \ |
112 | volatile unsigned long *_p) \ | |
113 | { \ | |
114 | unsigned long old, t; \ | |
115 | unsigned long *p = (unsigned long *)_p; \ | |
116 | __asm__ __volatile__ ( \ | |
117 | prefix \ | |
9401f4e4 | 118 | "1:" PPC_LLARX "%0,0,%3,%4\n" \ |
864b9e6f | 119 | stringify_in_c(op) "%1,%0,%2\n" \ |
864b9e6f AB |
120 | PPC_STLCX "%1,0,%3\n" \ |
121 | "bne- 1b\n" \ | |
122 | postfix \ | |
123 | : "=&r" (old), "=&r" (t) \ | |
9401f4e4 | 124 | : "r" (mask), "r" (p), "i" (IS_ENABLED(CONFIG_PPC64) ? eh : 0) \ |
864b9e6f AB |
125 | : "cc", "memory"); \ |
126 | return (old & mask); \ | |
a0e60b20 DG |
127 | } |
128 | ||
b97021f8 BH |
129 | DEFINE_TESTOP(test_and_set_bits, or, PPC_ATOMIC_ENTRY_BARRIER, |
130 | PPC_ATOMIC_EXIT_BARRIER, 0) | |
f10e2e5b AB |
131 | DEFINE_TESTOP(test_and_set_bits_lock, or, "", |
132 | PPC_ACQUIRE_BARRIER, 1) | |
b97021f8 BH |
133 | DEFINE_TESTOP(test_and_clear_bits, andc, PPC_ATOMIC_ENTRY_BARRIER, |
134 | PPC_ATOMIC_EXIT_BARRIER, 0) | |
135 | DEFINE_TESTOP(test_and_change_bits, xor, PPC_ATOMIC_ENTRY_BARRIER, | |
136 | PPC_ATOMIC_EXIT_BARRIER, 0) | |
0d2d3e38 | 137 | |
5bece3d6 DA |
138 | static inline int arch_test_and_set_bit(unsigned long nr, |
139 | volatile unsigned long *addr) | |
a0e60b20 | 140 | { |
2237f4f4 | 141 | return test_and_set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; |
a0e60b20 DG |
142 | } |
143 | ||
5bece3d6 DA |
144 | static inline int arch_test_and_set_bit_lock(unsigned long nr, |
145 | volatile unsigned long *addr) | |
66ffb04c | 146 | { |
2237f4f4 AM |
147 | return test_and_set_bits_lock(BIT_MASK(nr), |
148 | addr + BIT_WORD(nr)) != 0; | |
66ffb04c NP |
149 | } |
150 | ||
5bece3d6 DA |
151 | static inline int arch_test_and_clear_bit(unsigned long nr, |
152 | volatile unsigned long *addr) | |
a0e60b20 | 153 | { |
2237f4f4 | 154 | return test_and_clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; |
a0e60b20 DG |
155 | } |
156 | ||
5bece3d6 DA |
157 | static inline int arch_test_and_change_bit(unsigned long nr, |
158 | volatile unsigned long *addr) | |
a0e60b20 | 159 | { |
2237f4f4 | 160 | return test_and_change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; |
a0e60b20 DG |
161 | } |
162 | ||
d11914b2 | 163 | #ifdef CONFIG_PPC64 |
5bece3d6 DA |
164 | static inline unsigned long |
165 | clear_bit_unlock_return_word(int nr, volatile unsigned long *addr) | |
d11914b2 NP |
166 | { |
167 | unsigned long old, t; | |
168 | unsigned long *p = (unsigned long *)addr + BIT_WORD(nr); | |
169 | unsigned long mask = BIT_MASK(nr); | |
170 | ||
171 | __asm__ __volatile__ ( | |
172 | PPC_RELEASE_BARRIER | |
9401f4e4 | 173 | "1:" PPC_LLARX "%0,0,%3,0\n" |
d11914b2 | 174 | "andc %1,%0,%2\n" |
d11914b2 NP |
175 | PPC_STLCX "%1,0,%3\n" |
176 | "bne- 1b\n" | |
177 | : "=&r" (old), "=&r" (t) | |
178 | : "r" (mask), "r" (p) | |
179 | : "cc", "memory"); | |
180 | ||
181 | return old; | |
182 | } | |
183 | ||
5bece3d6 DA |
184 | /* |
185 | * This is a special function for mm/filemap.c | |
186 | * Bit 7 corresponds to PG_waiters. | |
187 | */ | |
188 | #define arch_clear_bit_unlock_is_negative_byte(nr, addr) \ | |
189 | (clear_bit_unlock_return_word(nr, addr) & BIT_MASK(7)) | |
d11914b2 NP |
190 | |
191 | #endif /* CONFIG_PPC64 */ | |
192 | ||
e779b2f9 | 193 | #include <asm-generic/bitops/non-atomic.h> |
a0e60b20 | 194 | |
5bece3d6 | 195 | static inline void arch___clear_bit_unlock(int nr, volatile unsigned long *addr) |
66ffb04c | 196 | { |
f10e2e5b | 197 | __asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory"); |
66ffb04c NP |
198 | __clear_bit(nr, addr); |
199 | } | |
200 | ||
a0e60b20 DG |
201 | /* |
202 | * Return the zero-based bit position (LE, not IBM bit numbering) of | |
203 | * the most significant 1-bit in a double word. | |
204 | */ | |
f782ddf2 | 205 | #define __ilog2(x) ilog2(x) |
ef55d53c | 206 | |
22ef33b3 | 207 | #include <asm-generic/bitops/ffz.h> |
a0e60b20 | 208 | |
f83647d6 | 209 | #include <asm-generic/bitops/builtin-__ffs.h> |
a0e60b20 | 210 | |
f83647d6 | 211 | #include <asm-generic/bitops/builtin-ffs.h> |
a0e60b20 DG |
212 | |
213 | /* | |
214 | * fls: find last (most-significant) bit set. | |
215 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. | |
216 | */ | |
5bece3d6 | 217 | static inline int fls(unsigned int x) |
a0e60b20 | 218 | { |
1891ef21 CL |
219 | int lz; |
220 | ||
221 | if (__builtin_constant_p(x)) | |
222 | return x ? 32 - __builtin_clz(x) : 0; | |
223 | asm("cntlzw %0,%1" : "=r" (lz) : "r" (x)); | |
224 | return 32 - lz; | |
a0e60b20 | 225 | } |
9f264be6 | 226 | |
2fcff790 | 227 | #include <asm-generic/bitops/builtin-__fls.h> |
56a6b1eb | 228 | |
1891ef21 CL |
229 | /* |
230 | * 64-bit can do this using one cntlzd (count leading zeroes doubleword) | |
231 | * instruction; for 32-bit we use the generic version, which does two | |
232 | * 32-bit fls calls. | |
233 | */ | |
234 | #ifdef CONFIG_PPC64 | |
5bece3d6 | 235 | static inline int fls64(__u64 x) |
9f264be6 | 236 | { |
1891ef21 CL |
237 | int lz; |
238 | ||
239 | if (__builtin_constant_p(x)) | |
240 | return x ? 64 - __builtin_clzll(x) : 0; | |
241 | asm("cntlzd %0,%1" : "=r" (lz) : "r" (x)); | |
242 | return 64 - lz; | |
9f264be6 | 243 | } |
1891ef21 CL |
244 | #else |
245 | #include <asm-generic/bitops/fls64.h> | |
246 | #endif | |
9f264be6 | 247 | |
64ff3128 AB |
248 | #ifdef CONFIG_PPC64 |
249 | unsigned int __arch_hweight8(unsigned int w); | |
250 | unsigned int __arch_hweight16(unsigned int w); | |
251 | unsigned int __arch_hweight32(unsigned int w); | |
252 | unsigned long __arch_hweight64(__u64 w); | |
253 | #include <asm-generic/bitops/const_hweight.h> | |
254 | #else | |
e779b2f9 | 255 | #include <asm-generic/bitops/hweight.h> |
64ff3128 AB |
256 | #endif |
257 | ||
47b9d9bd | 258 | #include <asm-generic/bitops/find.h> |
a0e60b20 | 259 | |
5bece3d6 DA |
260 | /* wrappers that deal with KASAN instrumentation */ |
261 | #include <asm-generic/bitops/instrumented-atomic.h> | |
262 | #include <asm-generic/bitops/instrumented-lock.h> | |
263 | ||
a0e60b20 | 264 | /* Little-endian versions */ |
79597be9 | 265 | #include <asm-generic/bitops/le.h> |
a0e60b20 | 266 | |
a0e60b20 DG |
267 | /* Bitmap functions for the ext2 filesystem */ |
268 | ||
148817ba | 269 | #include <asm-generic/bitops/ext2-atomic-setbit.h> |
a0e60b20 | 270 | |
e779b2f9 | 271 | #include <asm-generic/bitops/sched.h> |
a0e60b20 DG |
272 | |
273 | #endif /* __KERNEL__ */ | |
274 | ||
275 | #endif /* _ASM_POWERPC_BITOPS_H */ |