arch/arc/include/asm/bitops.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef _ASM_BITOPS_H
  10 #define _ASM_BITOPS_H
  11
  12 #ifndef _LINUX_BITOPS_H
  13 #error only <linux/bitops.h> can be included directly
  14 #endif
  15
  16 #ifndef __ASSEMBLY__
  17
  18 #include <linux/types.h>
  19 #include <linux/compiler.h>
  20 #include <asm/barrier.h>
  21 #ifndef CONFIG_ARC_HAS_LLSC
  22 #include <asm/smp.h>
  23 #endif
  24
  25 #ifdef CONFIG_ARC_HAS_LLSC
  26
  27 /*
  28  * Hardware assisted Atomic-R-M-W
  29  */
  30
  31 #define BIT_OP(op, c_op, asm_op)                                        \
  32 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
  33 {                                                                       \
  34         unsigned int temp;                                              \
  35                                                                         \
  36         m += nr >> 5;                                                   \
  37                                                                         \
  38         nr &= 0x1f;                                                     \
  39                                                                         \
  40         __asm__ __volatile__(                                           \
  41         "1:     llock       %0, [%1]            \n"                     \
  42         "       " #asm_op " %0, %0, %2  \n"                             \
  43         "       scond       %0, [%1]            \n"                     \
  44         "       bnz         1b                  \n"                     \
  45         : "=&r"(temp)   /* Early clobber, to prevent reg reuse */       \
  46         : "r"(m),       /* Not "m": llock only supports reg direct addr mode */ \
  47           "ir"(nr)                                                      \
  48         : "cc");                                                        \
  49 }
  50
  51 /*
  52  * Semantically:
  53  *    Test the bit
  54  *    if clear
  55  *        set it and return 0 (old value)
  56  *    else
  57  *        return 1 (old value).
  58  *
  59  * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally
  60  * and the old value of bit is returned
  61  */
  62 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
  63 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
  64 {                                                                       \
  65         unsigned long old, temp;                                        \
  66                                                                         \
  67         m += nr >> 5;                                                   \
  68                                                                         \
  69         nr &= 0x1f;                                                     \
  70                                                                         \
  71         /*                                                              \
  72          * Explicit full memory barrier needed before/after as          \
  73          * LLOCK/SCOND themselves don't provide any such smenatic       \
  74          */                                                             \
  75         smp_mb();                                                       \
  76                                                                         \
  77         __asm__ __volatile__(                                           \
  78         "1:     llock       %0, [%2]    \n"                             \
  79         "       " #asm_op " %1, %0, %3  \n"                             \
  80         "       scond       %1, [%2]    \n"                             \
  81         "       bnz         1b          \n"                             \
  82         : "=&r"(old), "=&r"(temp)                                       \
  83         : "r"(m), "ir"(nr)                                              \
  84         : "cc");                                                        \
  85                                                                         \
  86         smp_mb();                                                       \
  87                                                                         \
  88         return (old & (1 << nr)) != 0;                                  \
  89 }
  90
  91 #elif !defined(CONFIG_ARC_PLAT_EZNPS)
  92
  93 /*
  94  * Non hardware assisted Atomic-R-M-W
  95  * Locking would change to irq-disabling only (UP) and spinlocks (SMP)
  96  *
  97  * There's "significant" micro-optimization in writing our own variants of
  98  * bitops (over generic variants)
  99  *
 100  * (1) The generic APIs have "signed" @nr while we have it "unsigned"
 101  *     This avoids extra code to be generated for pointer arithmatic, since
 102  *     is "not sure" that index is NOT -ve
 103  * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc
 104  *     only consider bottom 5 bits of @nr, so NO need to mask them off.
 105  *     (GCC Quirk: however for constant @nr we still need to do the masking
 106  *             at compile time)
 107  */
 108
 109 #define BIT_OP(op, c_op, asm_op)                                        \
 110 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
 111 {                                                                       \
 112         unsigned long temp, flags;                                      \
 113         m += nr >> 5;                                                   \
 114                                                                         \
 115         /*                                                              \
 116          * spin lock/unlock provide the needed smp_mb() before/after    \
 117          */                                                             \
 118         bitops_lock(flags);                                             \
 119                                                                         \
 120         temp = *m;                                                      \
 121         *m = temp c_op (1UL << (nr & 0x1f));                                    \
 122                                                                         \
 123         bitops_unlock(flags);                                           \
 124 }
 125
 126 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 127 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 128 {                                                                       \
 129         unsigned long old, flags;                                       \
 130         m += nr >> 5;                                                   \
 131                                                                         \
 132         bitops_lock(flags);                                             \
 133                                                                         \
 134         old = *m;                                                       \
 135         *m = old c_op (1UL << (nr & 0x1f));                             \
 136                                                                         \
 137         bitops_unlock(flags);                                           \
 138                                                                         \
 139         return (old & (1UL << (nr & 0x1f))) != 0;                       \
 140 }
 141
 142 #else /* CONFIG_ARC_PLAT_EZNPS */
 143
 144 #define BIT_OP(op, c_op, asm_op)                                        \
 145 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\
 146 {                                                                       \
 147         m += nr >> 5;                                                   \
 148                                                                         \
 149         nr = (1UL << (nr & 0x1f));                                      \
 150         if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)                       \
 151                 nr = ~nr;                                               \
 152                                                                         \
 153         __asm__ __volatile__(                                           \
 154         "       mov r2, %0\n"                                           \
 155         "       mov r3, %1\n"                                           \
 156         "       .word %2\n"                                             \
 157         :                                                               \
 158         : "r"(nr), "r"(m), "i"(asm_op)                                  \
 159         : "r2", "r3", "memory");                                        \
 160 }
 161
 162 #define TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 163 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 164 {                                                                       \
 165         unsigned long old;                                              \
 166                                                                         \
 167         m += nr >> 5;                                                   \
 168                                                                         \
 169         nr = old = (1UL << (nr & 0x1f));                                \
 170         if (asm_op == CTOP_INST_AAND_DI_R2_R2_R3)                       \
 171                 old = ~old;                                             \
 172                                                                         \
 173         /* Explicit full memory barrier needed before/after */          \
 174         smp_mb();                                                       \
 175                                                                         \
 176         __asm__ __volatile__(                                           \
 177         "       mov r2, %0\n"                                           \
 178         "       mov r3, %1\n"                                           \
 179         "       .word %2\n"                                             \
 180         "       mov %0, r2"                                             \
 181         : "+r"(old)                                                     \
 182         : "r"(m), "i"(asm_op)                                           \
 183         : "r2", "r3", "memory");                                        \
 184                                                                         \
 185         smp_mb();                                                       \
 186                                                                         \
 187         return (old & nr) != 0;                                 \
 188 }
 189
 190 #endif /* CONFIG_ARC_PLAT_EZNPS */
 191
 192 /***************************************
 193  * Non atomic variants
 194  **************************************/
 195
 196 #define __BIT_OP(op, c_op, asm_op)                                      \
 197 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m)    \
 198 {                                                                       \
 199         unsigned long temp;                                             \
 200         m += nr >> 5;                                                   \
 201                                                                         \
 202         temp = *m;                                                      \
 203         *m = temp c_op (1UL << (nr & 0x1f));                            \
 204 }
 205
 206 #define __TEST_N_BIT_OP(op, c_op, asm_op)                               \
 207 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\
 208 {                                                                       \
 209         unsigned long old;                                              \
 210         m += nr >> 5;                                                   \
 211                                                                         \
 212         old = *m;                                                       \
 213         *m = old c_op (1UL << (nr & 0x1f));                             \
 214                                                                         \
 215         return (old & (1UL << (nr & 0x1f))) != 0;                       \
 216 }
 217
 218 #define BIT_OPS(op, c_op, asm_op)                                       \
 219                                                                         \
 220         /* set_bit(), clear_bit(), change_bit() */                      \
 221         BIT_OP(op, c_op, asm_op)                                        \
 222                                                                         \
 223         /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\
 224         TEST_N_BIT_OP(op, c_op, asm_op)                                 \
 225                                                                         \
 226         /* __set_bit(), __clear_bit(), __change_bit() */                \
 227         __BIT_OP(op, c_op, asm_op)                                      \
 228                                                                         \
 229         /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\
 230         __TEST_N_BIT_OP(op, c_op, asm_op)
 231
 232 #ifndef CONFIG_ARC_PLAT_EZNPS
 233 BIT_OPS(set, |, bset)
 234 BIT_OPS(clear, & ~, bclr)
 235 BIT_OPS(change, ^, bxor)
 236 #else
 237 BIT_OPS(set, |, CTOP_INST_AOR_DI_R2_R2_R3)
 238 BIT_OPS(clear, & ~, CTOP_INST_AAND_DI_R2_R2_R3)
 239 BIT_OPS(change, ^, CTOP_INST_AXOR_DI_R2_R2_R3)
 240 #endif
 241
 242 /*
 243  * This routine doesn't need to be atomic.
 244  */
 245 static inline int
 246 test_bit(unsigned int nr, const volatile unsigned long *addr)
 247 {
 248         unsigned long mask;
 249
 250         addr += nr >> 5;
 251
 252         mask = 1UL << (nr & 0x1f);
 253
 254         return ((mask & *addr) != 0);
 255 }
 256
 257 #ifdef CONFIG_ISA_ARCOMPACT
 258
 259 /*
 260  * Count the number of zeros, starting from MSB
 261  * Helper for fls( ) friends
 262  * This is a pure count, so (1-32) or (0-31) doesn't apply
 263  * It could be 0 to 32, based on num of 0's in there
 264  * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31
 265  */
 266 static inline __attribute__ ((const)) int clz(unsigned int x)
 267 {
 268         unsigned int res;
 269
 270         __asm__ __volatile__(
 271         "       norm.f  %0, %1          \n"
 272         "       mov.n   %0, 0           \n"
 273         "       add.p   %0, %0, 1       \n"
 274         : "=r"(res)
 275         : "r"(x)
 276         : "cc");
 277
 278         return res;
 279 }
 280
 281 static inline int constant_fls(unsigned int x)
 282 {
 283         int r = 32;
 284
 285         if (!x)
 286                 return 0;
 287         if (!(x & 0xffff0000u)) {
 288                 x <<= 16;
 289                 r -= 16;
 290         }
 291         if (!(x & 0xff000000u)) {
 292                 x <<= 8;
 293                 r -= 8;
 294         }
 295         if (!(x & 0xf0000000u)) {
 296                 x <<= 4;
 297                 r -= 4;
 298         }
 299         if (!(x & 0xc0000000u)) {
 300                 x <<= 2;
 301                 r -= 2;
 302         }
 303         if (!(x & 0x80000000u)) {
 304                 x <<= 1;
 305                 r -= 1;
 306         }
 307         return r;
 308 }
 309
 310 /*
 311  * fls = Find Last Set in word
 312  * @result: [1-32]
 313  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 314  */
 315 static inline __attribute__ ((const)) int fls(unsigned int x)
 316 {
 317         if (__builtin_constant_p(x))
 318                return constant_fls(x);
 319
 320         return 32 - clz(x);
 321 }
 322
 323 /*
 324  * __fls: Similar to fls, but zero based (0-31)
 325  */
 326 static inline __attribute__ ((const)) int __fls(unsigned long x)
 327 {
 328         if (!x)
 329                 return 0;
 330         else
 331                 return fls(x) - 1;
 332 }
 333
 334 /*
 335  * ffs = Find First Set in word (LSB to MSB)
 336  * @result: [1-32], 0 if all 0's
 337  */
 338 #define ffs(x)  ({ unsigned long __t = (x); fls(__t & -__t); })
 339
 340 /*
 341  * __ffs: Similar to ffs, but zero based (0-31)
 342  */
 343 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long word)
 344 {
 345         if (!word)
 346                 return word;
 347
 348         return ffs(word) - 1;
 349 }
 350
 351 #else   /* CONFIG_ISA_ARCV2 */
 352
 353 /*
 354  * fls = Find Last Set in word
 355  * @result: [1-32]
 356  * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0
 357  */
 358 static inline __attribute__ ((const)) int fls(unsigned long x)
 359 {
 360         int n;
 361
 362         asm volatile(
 363         "       fls.f   %0, %1          \n"  /* 0:31; 0(Z) if src 0 */
 364         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 365         : "=r"(n)       /* Early clobber not needed */
 366         : "r"(x)
 367         : "cc");
 368
 369         return n;
 370 }
 371
 372 /*
 373  * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set
 374  */
 375 static inline __attribute__ ((const)) int __fls(unsigned long x)
 376 {
 377         /* FLS insn has exactly same semantics as the API */
 378         return  __builtin_arc_fls(x);
 379 }
 380
 381 /*
 382  * ffs = Find First Set in word (LSB to MSB)
 383  * @result: [1-32], 0 if all 0's
 384  */
 385 static inline __attribute__ ((const)) int ffs(unsigned long x)
 386 {
 387         int n;
 388
 389         asm volatile(
 390         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 391         "       add.nz  %0, %0, 1       \n"  /* 0:31 -> 1:32 */
 392         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 393         : "=r"(n)       /* Early clobber not needed */
 394         : "r"(x)
 395         : "cc");
 396
 397         return n;
 398 }
 399
 400 /*
 401  * __ffs: Similar to ffs, but zero based (0-31)
 402  */
 403 static inline __attribute__ ((const)) unsigned long __ffs(unsigned long x)
 404 {
 405         unsigned long n;
 406
 407         asm volatile(
 408         "       ffs.f   %0, %1          \n"  /* 0:31; 31(Z) if src 0 */
 409         "       mov.z   %0, 0           \n"  /* 31(Z)-> 0 */
 410         : "=r"(n)
 411         : "r"(x)
 412         : "cc");
 413
 414         return n;
 415
 416 }
 417
 418 #endif  /* CONFIG_ISA_ARCOMPACT */
 419
 420 /*
 421  * ffz = Find First Zero in word.
 422  * @return:[0-31], 32 if all 1's
 423  */
 424 #define ffz(x)  __ffs(~(x))
 425
 426 #include <asm-generic/bitops/hweight.h>
 427 #include <asm-generic/bitops/fls64.h>
 428 #include <asm-generic/bitops/sched.h>
 429 #include <asm-generic/bitops/lock.h>
 430
 431 #include <asm-generic/bitops/find.h>
 432 #include <asm-generic/bitops/le.h>
 433 #include <asm-generic/bitops/ext2-atomic-setbit.h>
 434
 435 #endif /* !__ASSEMBLY__ */
 436
 437 #endif