arch/s390/include/asm/bitops.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *    Copyright IBM Corp. 1999,2013
   4  *
   5  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
   6  *
   7  * The description below was taken in large parts from the powerpc
   8  * bitops header file:
   9  * Within a word, bits are numbered LSB first.  Lot's of places make
  10  * this assumption by directly testing bits with (val & (1<<nr)).
  11  * This can cause confusion for large (> 1 word) bitmaps on a
  12  * big-endian system because, unlike little endian, the number of each
  13  * bit depends on the word size.
  14  *
  15  * The bitop functions are defined to work on unsigned longs, so the bits
  16  * end up numbered:
  17  *   |63..............0|127............64|191...........128|255...........192|
  18  *
  19  * We also have special functions which work with an MSB0 encoding.
  20  * The bits are numbered:
  21  *   |0..............63|64............127|128...........191|192...........255|
  22  *
  23  * The main difference is that bit 0-63 in the bit number field needs to be
  24  * reversed compared to the LSB0 encoded bit fields. This can be achieved by
  25  * XOR with 0x3f.
  26  *
  27  */
  28
  29 #ifndef _S390_BITOPS_H
  30 #define _S390_BITOPS_H
  31
  32 #ifndef _LINUX_BITOPS_H
  33 #error only <linux/bitops.h> can be included directly
  34 #endif
  35
  36 #include <linux/typecheck.h>
  37 #include <linux/compiler.h>
  38 #include <asm/atomic_ops.h>
  39 #include <asm/barrier.h>
  40
  41 #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
  42
  43 static inline unsigned long *
  44 __bitops_word(unsigned long nr, volatile unsigned long *ptr)
  45 {
  46         unsigned long addr;
  47
  48         addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
  49         return (unsigned long *)addr;
  50 }
  51
  52 static inline unsigned char *
  53 __bitops_byte(unsigned long nr, volatile unsigned long *ptr)
  54 {
  55         return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
  56 }
  57
  58 static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
  59 {
  60         unsigned long *addr = __bitops_word(nr, ptr);
  61         unsigned long mask;
  62
  63 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
  64         if (__builtin_constant_p(nr)) {
  65                 unsigned char *caddr = __bitops_byte(nr, ptr);
  66
  67                 asm volatile(
  68                         "oi     %0,%b1\n"
  69                         : "+Q" (*caddr)
  70                         : "i" (1 << (nr & 7))
  71                         : "cc", "memory");
  72                 return;
  73         }
  74 #endif
  75         mask = 1UL << (nr & (BITS_PER_LONG - 1));
  76         __atomic64_or(mask, addr);
  77 }
  78
  79 static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
  80 {
  81         unsigned long *addr = __bitops_word(nr, ptr);
  82         unsigned long mask;
  83
  84 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
  85         if (__builtin_constant_p(nr)) {
  86                 unsigned char *caddr = __bitops_byte(nr, ptr);
  87
  88                 asm volatile(
  89                         "ni     %0,%b1\n"
  90                         : "+Q" (*caddr)
  91                         : "i" (~(1 << (nr & 7)))
  92                         : "cc", "memory");
  93                 return;
  94         }
  95 #endif
  96         mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
  97         __atomic64_and(mask, addr);
  98 }
  99
 100 static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
 101 {
 102         unsigned long *addr = __bitops_word(nr, ptr);
 103         unsigned long mask;
 104
 105 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
 106         if (__builtin_constant_p(nr)) {
 107                 unsigned char *caddr = __bitops_byte(nr, ptr);
 108
 109                 asm volatile(
 110                         "xi     %0,%b1\n"
 111                         : "+Q" (*caddr)
 112                         : "i" (1 << (nr & 7))
 113                         : "cc", "memory");
 114                 return;
 115         }
 116 #endif
 117         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 118         __atomic64_xor(mask, addr);
 119 }
 120
 121 static inline int
 122 test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 123 {
 124         unsigned long *addr = __bitops_word(nr, ptr);
 125         unsigned long old, mask;
 126
 127         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 128         old = __atomic64_or_barrier(mask, addr);
 129         return (old & mask) != 0;
 130 }
 131
 132 static inline int
 133 test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 134 {
 135         unsigned long *addr = __bitops_word(nr, ptr);
 136         unsigned long old, mask;
 137
 138         mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
 139         old = __atomic64_and_barrier(mask, addr);
 140         return (old & ~mask) != 0;
 141 }
 142
 143 static inline int
 144 test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 145 {
 146         unsigned long *addr = __bitops_word(nr, ptr);
 147         unsigned long old, mask;
 148
 149         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 150         old = __atomic64_xor_barrier(mask, addr);
 151         return (old & mask) != 0;
 152 }
 153
 154 static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
 155 {
 156         unsigned char *addr = __bitops_byte(nr, ptr);
 157
 158         *addr |= 1 << (nr & 7);
 159 }
 160
 161 static inline void
 162 __clear_bit(unsigned long nr, volatile unsigned long *ptr)
 163 {
 164         unsigned char *addr = __bitops_byte(nr, ptr);
 165
 166         *addr &= ~(1 << (nr & 7));
 167 }
 168
 169 static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
 170 {
 171         unsigned char *addr = __bitops_byte(nr, ptr);
 172
 173         *addr ^= 1 << (nr & 7);
 174 }
 175
 176 static inline int
 177 __test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 178 {
 179         unsigned char *addr = __bitops_byte(nr, ptr);
 180         unsigned char ch;
 181
 182         ch = *addr;
 183         *addr |= 1 << (nr & 7);
 184         return (ch >> (nr & 7)) & 1;
 185 }
 186
 187 static inline int
 188 __test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 189 {
 190         unsigned char *addr = __bitops_byte(nr, ptr);
 191         unsigned char ch;
 192
 193         ch = *addr;
 194         *addr &= ~(1 << (nr & 7));
 195         return (ch >> (nr & 7)) & 1;
 196 }
 197
 198 static inline int
 199 __test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 200 {
 201         unsigned char *addr = __bitops_byte(nr, ptr);
 202         unsigned char ch;
 203
 204         ch = *addr;
 205         *addr ^= 1 << (nr & 7);
 206         return (ch >> (nr & 7)) & 1;
 207 }
 208
 209 static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
 210 {
 211         const volatile unsigned char *addr;
 212
 213         addr = ((const volatile unsigned char *)ptr);
 214         addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
 215         return (*addr >> (nr & 7)) & 1;
 216 }
 217
 218 static inline int test_and_set_bit_lock(unsigned long nr,
 219                                         volatile unsigned long *ptr)
 220 {
 221         if (test_bit(nr, ptr))
 222                 return 1;
 223         return test_and_set_bit(nr, ptr);
 224 }
 225
 226 static inline void clear_bit_unlock(unsigned long nr,
 227                                     volatile unsigned long *ptr)
 228 {
 229         smp_mb__before_atomic();
 230         clear_bit(nr, ptr);
 231 }
 232
 233 static inline void __clear_bit_unlock(unsigned long nr,
 234                                       volatile unsigned long *ptr)
 235 {
 236         smp_mb();
 237         __clear_bit(nr, ptr);
 238 }
 239
 240 /*
 241  * Functions which use MSB0 bit numbering.
 242  * The bits are numbered:
 243  *   |0..............63|64............127|128...........191|192...........255|
 244  */
 245 unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
 246 unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
 247                                 unsigned long offset);
 248
 249 #define for_each_set_bit_inv(bit, addr, size)                           \
 250         for ((bit) = find_first_bit_inv((addr), (size));                \
 251              (bit) < (size);                                            \
 252              (bit) = find_next_bit_inv((addr), (size), (bit) + 1))
 253
 254 static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 255 {
 256         return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 257 }
 258
 259 static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 260 {
 261         return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 262 }
 263
 264 static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 265 {
 266         return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 267 }
 268
 269 static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 270 {
 271         return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 272 }
 273
 274 static inline int test_bit_inv(unsigned long nr,
 275                                const volatile unsigned long *ptr)
 276 {
 277         return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 278 }
 279
 280 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
 281
 282 /**
 283  * __flogr - find leftmost one
 284  * @word - The word to search
 285  *
 286  * Returns the bit number of the most significant bit set,
 287  * where the most significant bit has bit number 0.
 288  * If no bit is set this function returns 64.
 289  */
 290 static inline unsigned char __flogr(unsigned long word)
 291 {
 292         if (__builtin_constant_p(word)) {
 293                 unsigned long bit = 0;
 294
 295                 if (!word)
 296                         return 64;
 297                 if (!(word & 0xffffffff00000000UL)) {
 298                         word <<= 32;
 299                         bit += 32;
 300                 }
 301                 if (!(word & 0xffff000000000000UL)) {
 302                         word <<= 16;
 303                         bit += 16;
 304                 }
 305                 if (!(word & 0xff00000000000000UL)) {
 306                         word <<= 8;
 307                         bit += 8;
 308                 }
 309                 if (!(word & 0xf000000000000000UL)) {
 310                         word <<= 4;
 311                         bit += 4;
 312                 }
 313                 if (!(word & 0xc000000000000000UL)) {
 314                         word <<= 2;
 315                         bit += 2;
 316                 }
 317                 if (!(word & 0x8000000000000000UL)) {
 318                         word <<= 1;
 319                         bit += 1;
 320                 }
 321                 return bit;
 322         } else {
 323                 register unsigned long bit asm("4") = word;
 324                 register unsigned long out asm("5");
 325
 326                 asm volatile(
 327                         "       flogr   %[bit],%[bit]\n"
 328                         : [bit] "+d" (bit), [out] "=d" (out) : : "cc");
 329                 return bit;
 330         }
 331 }
 332
 333 /**
 334  * __ffs - find first bit in word.
 335  * @word: The word to search
 336  *
 337  * Undefined if no bit exists, so code should check against 0 first.
 338  */
 339 static inline unsigned long __ffs(unsigned long word)
 340 {
 341         return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
 342 }
 343
 344 /**
 345  * ffs - find first bit set
 346  * @word: the word to search
 347  *
 348  * This is defined the same way as the libc and
 349  * compiler builtin ffs routines (man ffs).
 350  */
 351 static inline int ffs(int word)
 352 {
 353         unsigned long mask = 2 * BITS_PER_LONG - 1;
 354         unsigned int val = (unsigned int)word;
 355
 356         return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
 357 }
 358
 359 /**
 360  * __fls - find last (most-significant) set bit in a long word
 361  * @word: the word to search
 362  *
 363  * Undefined if no set bit exists, so code should check against 0 first.
 364  */
 365 static inline unsigned long __fls(unsigned long word)
 366 {
 367         return __flogr(word) ^ (BITS_PER_LONG - 1);
 368 }
 369
 370 /**
 371  * fls64 - find last set bit in a 64-bit word
 372  * @word: the word to search
 373  *
 374  * This is defined in a similar way as the libc and compiler builtin
 375  * ffsll, but returns the position of the most significant set bit.
 376  *
 377  * fls64(value) returns 0 if value is 0 or the position of the last
 378  * set bit if value is nonzero. The last (most significant) bit is
 379  * at position 64.
 380  */
 381 static inline int fls64(unsigned long word)
 382 {
 383         unsigned long mask = 2 * BITS_PER_LONG - 1;
 384
 385         return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
 386 }
 387
 388 /**
 389  * fls - find last (most-significant) bit set
 390  * @word: the word to search
 391  *
 392  * This is defined the same way as ffs.
 393  * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 394  */
 395 static inline int fls(int word)
 396 {
 397         return fls64((unsigned int)word);
 398 }
 399
 400 #else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
 401
 402 #include <asm-generic/bitops/__ffs.h>
 403 #include <asm-generic/bitops/ffs.h>
 404 #include <asm-generic/bitops/__fls.h>
 405 #include <asm-generic/bitops/fls.h>
 406 #include <asm-generic/bitops/fls64.h>
 407
 408 #endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
 409
 410 #include <asm-generic/bitops/ffz.h>
 411 #include <asm-generic/bitops/find.h>
 412 #include <asm-generic/bitops/hweight.h>
 413 #include <asm-generic/bitops/sched.h>
 414 #include <asm-generic/bitops/le.h>
 415 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 416
 417 #endif /* _S390_BITOPS_H */