arch/s390/include/asm/bitops.h

   1 /*
   2  *    Copyright IBM Corp. 1999,2013
   3  *
   4  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
   5  *
   6  * The description below was taken in large parts from the powerpc
   7  * bitops header file:
   8  * Within a word, bits are numbered LSB first.  Lot's of places make
   9  * this assumption by directly testing bits with (val & (1<<nr)).
  10  * This can cause confusion for large (> 1 word) bitmaps on a
  11  * big-endian system because, unlike little endian, the number of each
  12  * bit depends on the word size.
  13  *
  14  * The bitop functions are defined to work on unsigned longs, so for an
  15  * s390x system the bits end up numbered:
  16  *   |63..............0|127............64|191...........128|255...........192|
  17  * and on s390:
  18  *   |31.....0|63....32|95....64|127...96|159..128|191..160|223..192|255..224|
  19  *
  20  * There are a few little-endian macros used mostly for filesystem
  21  * bitmaps, these work on similar bit arrays layouts, but
  22  * byte-oriented:
  23  *   |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56|
  24  *
  25  * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit
  26  * number field needs to be reversed compared to the big-endian bit
  27  * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b).
  28  *
  29  * We also have special functions which work with an MSB0 encoding:
  30  * on an s390x system the bits are numbered:
  31  *   |0..............63|64............127|128...........191|192...........255|
  32  * and on s390:
  33  *   |0.....31|32....63|64....95|96...127|128..159|160..191|192..223|224..255|
  34  *
  35  * The main difference is that bit 0-63 (64b) or 0-31 (32b) in the bit
  36  * number field needs to be reversed compared to the LSB0 encoded bit
  37  * fields. This can be achieved by XOR with 0x3f (64b) or 0x1f (32b).
  38  *
  39  */
  40
  41 #ifndef _S390_BITOPS_H
  42 #define _S390_BITOPS_H
  43
  44 #ifndef _LINUX_BITOPS_H
  45 #error only <linux/bitops.h> can be included directly
  46 #endif
  47
  48 #include <linux/typecheck.h>
  49 #include <linux/compiler.h>
  50 #include <asm/barrier.h>
  51
  52 #define __BITOPS_NO_BARRIER     "\n"
  53
  54 #ifndef CONFIG_64BIT
  55
  56 #define __BITOPS_OR             "or"
  57 #define __BITOPS_AND            "nr"
  58 #define __BITOPS_XOR            "xr"
  59 #define __BITOPS_BARRIER        "\n"
  60
  61 #define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)    \
  62 ({                                                              \
  63         unsigned long __old, __new;                             \
  64                                                                 \
  65         typecheck(unsigned long *, (__addr));                   \
  66         asm volatile(                                           \
  67                 "       l       %0,%2\n"                        \
  68                 "0:     lr      %1,%0\n"                        \
  69                 __op_string "   %1,%3\n"                        \
  70                 "       cs      %0,%1,%2\n"                     \
  71                 "       jl      0b"                             \
  72                 : "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
  73                 : "d" (__val)                                   \
  74                 : "cc", "memory");                              \
  75         __old;                                                  \
  76 })
  77
  78 #else /* CONFIG_64BIT */
  79
  80 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
  81
  82 #define __BITOPS_OR             "laog"
  83 #define __BITOPS_AND            "lang"
  84 #define __BITOPS_XOR            "laxg"
  85 #define __BITOPS_BARRIER        "bcr    14,0\n"
  86
  87 #define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)    \
  88 ({                                                              \
  89         unsigned long __old;                                    \
  90                                                                 \
  91         typecheck(unsigned long *, (__addr));                   \
  92         asm volatile(                                           \
  93                 __barrier                                       \
  94                 __op_string "   %0,%2,%1\n"                     \
  95                 __barrier                                       \
  96                 : "=d" (__old), "+Q" (*(__addr))                \
  97                 : "d" (__val)                                   \
  98                 : "cc", "memory");                              \
  99         __old;                                                  \
 100 })
 101
 102 #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 103
 104 #define __BITOPS_OR             "ogr"
 105 #define __BITOPS_AND            "ngr"
 106 #define __BITOPS_XOR            "xgr"
 107 #define __BITOPS_BARRIER        "\n"
 108
 109 #define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)    \
 110 ({                                                              \
 111         unsigned long __old, __new;                             \
 112                                                                 \
 113         typecheck(unsigned long *, (__addr));                   \
 114         asm volatile(                                           \
 115                 "       lg      %0,%2\n"                        \
 116                 "0:     lgr     %1,%0\n"                        \
 117                 __op_string "   %1,%3\n"                        \
 118                 "       csg     %0,%1,%2\n"                     \
 119                 "       jl      0b"                             \
 120                 : "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
 121                 : "d" (__val)                                   \
 122                 : "cc", "memory");                              \
 123         __old;                                                  \
 124 })
 125
 126 #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
 127
 128 #endif /* CONFIG_64BIT */
 129
 130 #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
 131
 132 static inline unsigned long *
 133 __bitops_word(unsigned long nr, volatile unsigned long *ptr)
 134 {
 135         unsigned long addr;
 136
 137         addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
 138         return (unsigned long *)addr;
 139 }
 140
 141 static inline unsigned char *
 142 __bitops_byte(unsigned long nr, volatile unsigned long *ptr)
 143 {
 144         return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
 145 }
 146
 147 static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
 148 {
 149         unsigned long *addr = __bitops_word(nr, ptr);
 150         unsigned long mask;
 151
 152 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
 153         if (__builtin_constant_p(nr)) {
 154                 unsigned char *caddr = __bitops_byte(nr, ptr);
 155
 156                 asm volatile(
 157                         "oi     %0,%b1\n"
 158                         : "+Q" (*caddr)
 159                         : "i" (1 << (nr & 7))
 160                         : "cc", "memory");
 161                 return;
 162         }
 163 #endif
 164         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 165         __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);
 166 }
 167
 168 static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
 169 {
 170         unsigned long *addr = __bitops_word(nr, ptr);
 171         unsigned long mask;
 172
 173 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
 174         if (__builtin_constant_p(nr)) {
 175                 unsigned char *caddr = __bitops_byte(nr, ptr);
 176
 177                 asm volatile(
 178                         "ni     %0,%b1\n"
 179                         : "+Q" (*caddr)
 180                         : "i" (~(1 << (nr & 7)))
 181                         : "cc", "memory");
 182                 return;
 183         }
 184 #endif
 185         mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
 186         __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);
 187 }
 188
 189 static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
 190 {
 191         unsigned long *addr = __bitops_word(nr, ptr);
 192         unsigned long mask;
 193
 194 #ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
 195         if (__builtin_constant_p(nr)) {
 196                 unsigned char *caddr = __bitops_byte(nr, ptr);
 197
 198                 asm volatile(
 199                         "xi     %0,%b1\n"
 200                         : "+Q" (*caddr)
 201                         : "i" (1 << (nr & 7))
 202                         : "cc", "memory");
 203                 return;
 204         }
 205 #endif
 206         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 207         __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);
 208 }
 209
 210 static inline int
 211 test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 212 {
 213         unsigned long *addr = __bitops_word(nr, ptr);
 214         unsigned long old, mask;
 215
 216         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 217         old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);
 218         return (old & mask) != 0;
 219 }
 220
 221 static inline int
 222 test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 223 {
 224         unsigned long *addr = __bitops_word(nr, ptr);
 225         unsigned long old, mask;
 226
 227         mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
 228         old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);
 229         return (old & ~mask) != 0;
 230 }
 231
 232 static inline int
 233 test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 234 {
 235         unsigned long *addr = __bitops_word(nr, ptr);
 236         unsigned long old, mask;
 237
 238         mask = 1UL << (nr & (BITS_PER_LONG - 1));
 239         old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);
 240         return (old & mask) != 0;
 241 }
 242
 243 static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
 244 {
 245         unsigned char *addr = __bitops_byte(nr, ptr);
 246
 247         *addr |= 1 << (nr & 7);
 248 }
 249
 250 static inline void
 251 __clear_bit(unsigned long nr, volatile unsigned long *ptr)
 252 {
 253         unsigned char *addr = __bitops_byte(nr, ptr);
 254
 255         *addr &= ~(1 << (nr & 7));
 256 }
 257
 258 static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
 259 {
 260         unsigned char *addr = __bitops_byte(nr, ptr);
 261
 262         *addr ^= 1 << (nr & 7);
 263 }
 264
 265 static inline int
 266 __test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 267 {
 268         unsigned char *addr = __bitops_byte(nr, ptr);
 269         unsigned char ch;
 270
 271         ch = *addr;
 272         *addr |= 1 << (nr & 7);
 273         return (ch >> (nr & 7)) & 1;
 274 }
 275
 276 static inline int
 277 __test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 278 {
 279         unsigned char *addr = __bitops_byte(nr, ptr);
 280         unsigned char ch;
 281
 282         ch = *addr;
 283         *addr &= ~(1 << (nr & 7));
 284         return (ch >> (nr & 7)) & 1;
 285 }
 286
 287 static inline int
 288 __test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 289 {
 290         unsigned char *addr = __bitops_byte(nr, ptr);
 291         unsigned char ch;
 292
 293         ch = *addr;
 294         *addr ^= 1 << (nr & 7);
 295         return (ch >> (nr & 7)) & 1;
 296 }
 297
 298 static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
 299 {
 300         const volatile unsigned char *addr;
 301
 302         addr = ((const volatile unsigned char *)ptr);
 303         addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
 304         return (*addr >> (nr & 7)) & 1;
 305 }
 306
 307 /*
 308  * Functions which use MSB0 bit numbering.
 309  * On an s390x system the bits are numbered:
 310  *   |0..............63|64............127|128...........191|192...........255|
 311  * and on s390:
 312  *   |0.....31|32....63|64....95|96...127|128..159|160..191|192..223|224..255|
 313  */
 314 unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
 315 unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
 316                                 unsigned long offset);
 317
 318 static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 319 {
 320         return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 321 }
 322
 323 static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 324 {
 325         return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 326 }
 327
 328 static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 329 {
 330         return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 331 }
 332
 333 static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 334 {
 335         return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 336 }
 337
 338 static inline int test_bit_inv(unsigned long nr,
 339                                const volatile unsigned long *ptr)
 340 {
 341         return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 342 }
 343
 344 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
 345
 346 /**
 347  * __flogr - find leftmost one
 348  * @word - The word to search
 349  *
 350  * Returns the bit number of the most significant bit set,
 351  * where the most significant bit has bit number 0.
 352  * If no bit is set this function returns 64.
 353  */
 354 static inline unsigned char __flogr(unsigned long word)
 355 {
 356         if (__builtin_constant_p(word)) {
 357                 unsigned long bit = 0;
 358
 359                 if (!word)
 360                         return 64;
 361                 if (!(word & 0xffffffff00000000UL)) {
 362                         word <<= 32;
 363                         bit += 32;
 364                 }
 365                 if (!(word & 0xffff000000000000UL)) {
 366                         word <<= 16;
 367                         bit += 16;
 368                 }
 369                 if (!(word & 0xff00000000000000UL)) {
 370                         word <<= 8;
 371                         bit += 8;
 372                 }
 373                 if (!(word & 0xf000000000000000UL)) {
 374                         word <<= 4;
 375                         bit += 4;
 376                 }
 377                 if (!(word & 0xc000000000000000UL)) {
 378                         word <<= 2;
 379                         bit += 2;
 380                 }
 381                 if (!(word & 0x8000000000000000UL)) {
 382                         word <<= 1;
 383                         bit += 1;
 384                 }
 385                 return bit;
 386         } else {
 387                 register unsigned long bit asm("4") = word;
 388                 register unsigned long out asm("5");
 389
 390                 asm volatile(
 391                         "       flogr   %[bit],%[bit]\n"
 392                         : [bit] "+d" (bit), [out] "=d" (out) : : "cc");
 393                 return bit;
 394         }
 395 }
 396
 397 /**
 398  * __ffs - find first bit in word.
 399  * @word: The word to search
 400  *
 401  * Undefined if no bit exists, so code should check against 0 first.
 402  */
 403 static inline unsigned long __ffs(unsigned long word)
 404 {
 405         return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
 406 }
 407
 408 /**
 409  * ffs - find first bit set
 410  * @word: the word to search
 411  *
 412  * This is defined the same way as the libc and
 413  * compiler builtin ffs routines (man ffs).
 414  */
 415 static inline int ffs(int word)
 416 {
 417         unsigned long mask = 2 * BITS_PER_LONG - 1;
 418         unsigned int val = (unsigned int)word;
 419
 420         return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
 421 }
 422
 423 /**
 424  * __fls - find last (most-significant) set bit in a long word
 425  * @word: the word to search
 426  *
 427  * Undefined if no set bit exists, so code should check against 0 first.
 428  */
 429 static inline unsigned long __fls(unsigned long word)
 430 {
 431         return __flogr(word) ^ (BITS_PER_LONG - 1);
 432 }
 433
 434 /**
 435  * fls64 - find last set bit in a 64-bit word
 436  * @word: the word to search
 437  *
 438  * This is defined in a similar way as the libc and compiler builtin
 439  * ffsll, but returns the position of the most significant set bit.
 440  *
 441  * fls64(value) returns 0 if value is 0 or the position of the last
 442  * set bit if value is nonzero. The last (most significant) bit is
 443  * at position 64.
 444  */
 445 static inline int fls64(unsigned long word)
 446 {
 447         unsigned long mask = 2 * BITS_PER_LONG - 1;
 448
 449         return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
 450 }
 451
 452 /**
 453  * fls - find last (most-significant) bit set
 454  * @word: the word to search
 455  *
 456  * This is defined the same way as ffs.
 457  * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 458  */
 459 static inline int fls(int word)
 460 {
 461         return fls64((unsigned int)word);
 462 }
 463
 464 #else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
 465
 466 #include <asm-generic/bitops/__ffs.h>
 467 #include <asm-generic/bitops/ffs.h>
 468 #include <asm-generic/bitops/__fls.h>
 469 #include <asm-generic/bitops/fls.h>
 470 #include <asm-generic/bitops/fls64.h>
 471
 472 #endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
 473
 474 #include <asm-generic/bitops/ffz.h>
 475 #include <asm-generic/bitops/find.h>
 476 #include <asm-generic/bitops/hweight.h>
 477 #include <asm-generic/bitops/lock.h>
 478 #include <asm-generic/bitops/sched.h>
 479 #include <asm-generic/bitops/le.h>
 480 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 481
 482 #endif /* _S390_BITOPS_H */