[mirror_ubuntu-bionic-kernel.git] / arch / s390 / include / asm / bitops.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 *    Copyright IBM Corp. 1999,2013
 *
 *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
 *
 * The description below was taken in large parts from the powerpc
 * bitops header file:
 * Within a word, bits are numbered LSB first.  Lot's of places make
 * this assumption by directly testing bits with (val & (1<<nr)).
 * This can cause confusion for large (> 1 word) bitmaps on a
 * big-endian system because, unlike little endian, the number of each
 * bit depends on the word size.
 *
 * The bitop functions are defined to work on unsigned longs, so the bits
 * end up numbered:
 *   |63..............0|127............64|191...........128|255...........192|
 *
 * We also have special functions which work with an MSB0 encoding.
 * The bits are numbered:
 *   |0..............63|64............127|128...........191|192...........255|
 *
 * The main difference is that bit 0-63 in the bit number field needs to be
 * reversed compared to the LSB0 encoded bit fields. This can be achieved by
 * XOR with 0x3f.
 *
 */

#ifndef _S390_BITOPS_H
#define _S390_BITOPS_H

#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif

#include <linux/typecheck.h>
#include <linux/compiler.h>
#include <asm/atomic_ops.h>
#include <asm/barrier.h>

#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)

static inline unsigned long *
__bitops_word(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long addr;

	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
	return (unsigned long *)addr;
}

static inline unsigned char *
__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
{
	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
}

static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"oi	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (1 << (nr & 7))
			: "cc", "memory");
		return;
	}
#endif
	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	__atomic64_or(mask, addr);
}

static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"ni	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (~(1 << (nr & 7)))
			: "cc", "memory");
		return;
	}
#endif
	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	__atomic64_and(mask, addr);
}

static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"xi	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (1 << (nr & 7))
			: "cc", "memory");
		return;
	}
#endif
	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	__atomic64_xor(mask, addr);
}

static inline int
test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	old = __atomic64_or_barrier(mask, addr);
	return (old & mask) != 0;
}

static inline int
test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	old = __atomic64_and_barrier(mask, addr);
	return (old & ~mask) != 0;
}

static inline int
test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	old = __atomic64_xor_barrier(mask, addr);
	return (old & mask) != 0;
}

static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr |= 1 << (nr & 7);
}

static inline void 
__clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr &= ~(1 << (nr & 7));
}

static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr ^= 1 << (nr & 7);
}

static inline int
__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr |= 1 << (nr & 7);
	return (ch >> (nr & 7)) & 1;
}

static inline int
__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr &= ~(1 << (nr & 7));
	return (ch >> (nr & 7)) & 1;
}

static inline int
__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr ^= 1 << (nr & 7);
	return (ch >> (nr & 7)) & 1;
}

static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
{
	const volatile unsigned char *addr;

	addr = ((const volatile unsigned char *)ptr);
	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
	return (*addr >> (nr & 7)) & 1;
}

static inline int test_and_set_bit_lock(unsigned long nr,
					volatile unsigned long *ptr)
{
	if (test_bit(nr, ptr))
		return 1;
	return test_and_set_bit(nr, ptr);
}

static inline void clear_bit_unlock(unsigned long nr,
				    volatile unsigned long *ptr)
{
	smp_mb__before_atomic();
	clear_bit(nr, ptr);
}

static inline void __clear_bit_unlock(unsigned long nr,
				      volatile unsigned long *ptr)
{
	smp_mb();
	__clear_bit(nr, ptr);
}

/*
 * Functions which use MSB0 bit numbering.
 * The bits are numbered:
 *   |0..............63|64............127|128...........191|192...........255|
 */
unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
				unsigned long offset);

#define for_each_set_bit_inv(bit, addr, size)				\
	for ((bit) = find_first_bit_inv((addr), (size));		\
	     (bit) < (size);						\
	     (bit) = find_next_bit_inv((addr), (size), (bit) + 1))

static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline int test_bit_inv(unsigned long nr,
			       const volatile unsigned long *ptr)
{
	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES

/**
 * __flogr - find leftmost one
 * @word - The word to search
 *
 * Returns the bit number of the most significant bit set,
 * where the most significant bit has bit number 0.
 * If no bit is set this function returns 64.
 */
static inline unsigned char __flogr(unsigned long word)
{
	if (__builtin_constant_p(word)) {
		unsigned long bit = 0;

		if (!word)
			return 64;
		if (!(word & 0xffffffff00000000UL)) {
			word <<= 32;
			bit += 32;
		}
		if (!(word & 0xffff000000000000UL)) {
			word <<= 16;
			bit += 16;
		}
		if (!(word & 0xff00000000000000UL)) {
			word <<= 8;
			bit += 8;
		}
		if (!(word & 0xf000000000000000UL)) {
			word <<= 4;
			bit += 4;
		}
		if (!(word & 0xc000000000000000UL)) {
			word <<= 2;
			bit += 2;
		}
		if (!(word & 0x8000000000000000UL)) {
			word <<= 1;
			bit += 1;
		}
		return bit;
	} else {
		register unsigned long bit asm("4") = word;
		register unsigned long out asm("5");

		asm volatile(
			"       flogr   %[bit],%[bit]\n"
			: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
		return bit;
	}
}

/**
 * __ffs - find first bit in word.
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static inline unsigned long __ffs(unsigned long word)
{
	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
}

/**
 * ffs - find first bit set
 * @word: the word to search
 *
 * This is defined the same way as the libc and
 * compiler builtin ffs routines (man ffs).
 */
static inline int ffs(int word)
{
	unsigned long mask = 2 * BITS_PER_LONG - 1;
	unsigned int val = (unsigned int)word;

	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
}

/**
 * __fls - find last (most-significant) set bit in a long word
 * @word: the word to search
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
static inline unsigned long __fls(unsigned long word)
{
	return __flogr(word) ^ (BITS_PER_LONG - 1);
}

/**
 * fls64 - find last set bit in a 64-bit word
 * @word: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffsll, but returns the position of the most significant set bit.
 *
 * fls64(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 64.
 */
static inline int fls64(unsigned long word)
{
	unsigned long mask = 2 * BITS_PER_LONG - 1;

	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
}

/**
 * fls - find last (most-significant) bit set
 * @word: the word to search
 *
 * This is defined the same way as ffs.
 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 */
static inline int fls(int word)
{
	return fls64((unsigned int)word);
}

#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */

#include <asm-generic/bitops/__ffs.h>
#include <asm-generic/bitops/ffs.h>
#include <asm-generic/bitops/__fls.h>
#include <asm-generic/bitops/fls.h>
#include <asm-generic/bitops/fls64.h>

#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */

#include <asm-generic/bitops/ffz.h>
#include <asm-generic/bitops/find.h>
#include <asm-generic/bitops/hweight.h>
#include <asm-generic/bitops/sched.h>
#include <asm-generic/bitops/le.h>
#include <asm-generic/bitops/ext2-atomic-setbit.h>

#endif /* _S390_BITOPS_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4	2	/*
746479cd	3	* Copyright IBM Corp. 1999,2013
1da177e4	4	*
746479cd HC	5	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
	6	*
	7	* The description below was taken in large parts from the powerpc
	8	* bitops header file:
	9	* Within a word, bits are numbered LSB first. Lot's of places make
	10	* this assumption by directly testing bits with (val & (1<<nr)).
	11	* This can cause confusion for large (> 1 word) bitmaps on a
	12	* big-endian system because, unlike little endian, the number of each
	13	* bit depends on the word size.
	14	*
48002bd5 HC	15	* The bitop functions are defined to work on unsigned longs, so the bits
48002bd5 HC	16	* end up numbered:
db85eaeb	17	* \|63..............0\|127............64\|191...........128\|255...........192\|
746479cd	18	*
48002bd5 HC	19	* We also have special functions which work with an MSB0 encoding.
48002bd5 HC	20	* The bits are numbered:
746479cd	21	* \|0..............63\|64............127\|128...........191\|192...........255\|
746479cd	22	*
48002bd5 HC	23	* The main difference is that bit 0-63 in the bit number field needs to be
	24	* reversed compared to the LSB0 encoded bit fields. This can be achieved by
	25	* XOR with 0x3f.
1da177e4 LT	26	*
1da177e4 LT	27	*/
c406abd3	28
a53c8fab HC	29	#ifndef _S390_BITOPS_H
	30	#define _S390_BITOPS_H
	31
0624517d JS	32	#ifndef _LINUX_BITOPS_H
	33	#error only <linux/bitops.h> can be included directly
	34	#endif
	35
370b0b5f	36	#include <linux/typecheck.h>
1da177e4	37	#include <linux/compiler.h>
1993dbc7	38	#include <asm/atomic_ops.h>
0ccc8b7a HC	39	#include <asm/barrier.h>
0ccc8b7a HC	40
01c2475f	41	#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
1da177e4	42
370b0b5f HC	43	static inline unsigned long *
	44	__bitops_word(unsigned long nr, volatile unsigned long *ptr)
	45	{
	46	unsigned long addr;
	47
	48	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
	49	return (unsigned long *)addr;
	50	}
	51
	52	static inline unsigned char *
	53	__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
	54	{
	55	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
	56	}
	57
	58	static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	59	{
370b0b5f HC	60	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	61	unsigned long mask;
1da177e4	62
4ae80325 HC	63	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	64	if (__builtin_constant_p(nr)) {
	65	unsigned char *caddr = __bitops_byte(nr, ptr);
	66
	67	asm volatile(
	68	"oi %0,%b1\n"
	69	: "+Q" (*caddr)
	70	: "i" (1 << (nr & 7))
0ccc8b7a	71	: "cc", "memory");
4ae80325 HC	72	return;
	73	}
	74	#endif
01c2475f	75	mask = 1UL << (nr & (BITS_PER_LONG - 1));
1993dbc7	76	__atomic64_or(mask, addr);
1da177e4 LT	77	}
1da177e4 LT	78
370b0b5f	79	static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	80	{
370b0b5f HC	81	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	82	unsigned long mask;
1da177e4	83
4ae80325 HC	84	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	85	if (__builtin_constant_p(nr)) {
	86	unsigned char *caddr = __bitops_byte(nr, ptr);
	87
	88	asm volatile(
	89	"ni %0,%b1\n"
	90	: "+Q" (*caddr)
	91	: "i" (~(1 << (nr & 7)))
0ccc8b7a	92	: "cc", "memory");
4ae80325 HC	93	return;
	94	}
	95	#endif
01c2475f	96	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
1993dbc7	97	__atomic64_and(mask, addr);
1da177e4 LT	98	}
1da177e4 LT	99
370b0b5f	100	static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	101	{
370b0b5f HC	102	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	103	unsigned long mask;
1da177e4	104
4ae80325 HC	105	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	106	if (__builtin_constant_p(nr)) {
	107	unsigned char *caddr = __bitops_byte(nr, ptr);
	108
	109	asm volatile(
	110	"xi %0,%b1\n"
	111	: "+Q" (*caddr)
	112	: "i" (1 << (nr & 7))
0ccc8b7a	113	: "cc", "memory");
4ae80325 HC	114	return;
	115	}
	116	#endif
01c2475f	117	mask = 1UL << (nr & (BITS_PER_LONG - 1));
1993dbc7	118	__atomic64_xor(mask, addr);
1da177e4 LT	119	}
1da177e4 LT	120
1da177e4	121	static inline int
370b0b5f	122	test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	123	{
370b0b5f HC	124	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	125	unsigned long old, mask;
1da177e4	126
01c2475f	127	mask = 1UL << (nr & (BITS_PER_LONG - 1));
1993dbc7	128	old = __atomic64_or_barrier(mask, addr);
1da177e4 LT	129	return (old & mask) != 0;
	130	}
	131
1da177e4	132	static inline int
370b0b5f	133	test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	134	{
370b0b5f HC	135	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	136	unsigned long old, mask;
1da177e4	137
01c2475f	138	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
1993dbc7	139	old = __atomic64_and_barrier(mask, addr);
e344e52c	140	return (old & ~mask) != 0;
1da177e4 LT	141	}
1da177e4 LT	142
1da177e4	143	static inline int
370b0b5f	144	test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	145	{
370b0b5f HC	146	unsigned long *addr = __bitops_word(nr, ptr);
370b0b5f HC	147	unsigned long old, mask;
1da177e4	148
01c2475f	149	mask = 1UL << (nr & (BITS_PER_LONG - 1));
1993dbc7	150	old = __atomic64_xor_barrier(mask, addr);
1da177e4 LT	151	return (old & mask) != 0;
1da177e4 LT	152	}
1da177e4	153
1da177e4 LT	154	static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4 LT	155	{
370b0b5f	156	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4	157
370b0b5f	158	*addr \|= 1 << (nr & 7);
1da177e4 LT	159	}
1da177e4 LT	160
1da177e4 LT	161	static inline void
	162	__clear_bit(unsigned long nr, volatile unsigned long *ptr)
	163	{
370b0b5f	164	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4	165
370b0b5f	166	*addr &= ~(1 << (nr & 7));
1da177e4 LT	167	}
1da177e4 LT	168
1da177e4 LT	169	static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4 LT	170	{
370b0b5f	171	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4	172
370b0b5f	173	*addr ^= 1 << (nr & 7);
1da177e4 LT	174	}
1da177e4 LT	175
1da177e4	176	static inline int
370b0b5f	177	__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	178	{
370b0b5f	179	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4 LT	180	unsigned char ch;
1da177e4 LT	181
370b0b5f HC	182	ch = *addr;
370b0b5f HC	183	*addr \|= 1 << (nr & 7);
1da177e4 LT	184	return (ch >> (nr & 7)) & 1;
1da177e4 LT	185	}
1da177e4	186
1da177e4	187	static inline int
370b0b5f	188	__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	189	{
370b0b5f	190	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4 LT	191	unsigned char ch;
1da177e4 LT	192
370b0b5f HC	193	ch = *addr;
370b0b5f HC	194	*addr &= ~(1 << (nr & 7));
1da177e4 LT	195	return (ch >> (nr & 7)) & 1;
1da177e4 LT	196	}
1da177e4	197
1da177e4	198	static inline int
370b0b5f	199	__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
1da177e4	200	{
370b0b5f	201	unsigned char *addr = __bitops_byte(nr, ptr);
1da177e4 LT	202	unsigned char ch;
1da177e4 LT	203
370b0b5f HC	204	ch = *addr;
370b0b5f HC	205	*addr ^= 1 << (nr & 7);
1da177e4 LT	206	return (ch >> (nr & 7)) & 1;
1da177e4 LT	207	}
1da177e4	208
370b0b5f	209	static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
1da177e4	210	{
370b0b5f	211	const volatile unsigned char *addr;
1da177e4	212
370b0b5f HC	213	addr = ((const volatile unsigned char *)ptr);
	214	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
	215	return (*addr >> (nr & 7)) & 1;
1da177e4 LT	216	}
1da177e4 LT	217
acdc9fc9 MS	218	static inline int test_and_set_bit_lock(unsigned long nr,
	219	volatile unsigned long *ptr)
	220	{
	221	if (test_bit(nr, ptr))
	222	return 1;
	223	return test_and_set_bit(nr, ptr);
	224	}
	225
	226	static inline void clear_bit_unlock(unsigned long nr,
	227	volatile unsigned long *ptr)
	228	{
	229	smp_mb__before_atomic();
	230	clear_bit(nr, ptr);
	231	}
	232
	233	static inline void __clear_bit_unlock(unsigned long nr,
	234	volatile unsigned long *ptr)
	235	{
	236	smp_mb();
	237	__clear_bit(nr, ptr);
	238	}
	239
afff7e2b	240	/*
7d7c7b24	241	* Functions which use MSB0 bit numbering.
48002bd5	242	* The bits are numbered:
7d7c7b24	243	* \|0..............63\|64............127\|128...........191\|192...........255\|
e56e4e87	244	*/
7d7c7b24 HC	245	unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
	246	unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
	247	unsigned long offset);
	248
09214545 HC	249	#define for_each_set_bit_inv(bit, addr, size) \
	250	for ((bit) = find_first_bit_inv((addr), (size)); \
	251	(bit) < (size); \
	252	(bit) = find_next_bit_inv((addr), (size), (bit) + 1))
	253
7d7c7b24 HC	254	static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	255	{
	256	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	257	}
	258
	259	static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	260	{
	261	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	262	}
	263
	264	static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	265	{
	266	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	267	}
	268
	269	static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	270	{
	271	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	272	}
	273
	274	static inline int test_bit_inv(unsigned long nr,
	275	const volatile unsigned long *ptr)
	276	{
	277	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	278	}
e56e4e87	279
b1cb7e2b HC	280	#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
	281
	282	/**
	283	* __flogr - find leftmost one
	284	* @word - The word to search
	285	*
	286	* Returns the bit number of the most significant bit set,
	287	* where the most significant bit has bit number 0.
	288	* If no bit is set this function returns 64.
	289	*/
	290	static inline unsigned char __flogr(unsigned long word)
	291	{
	292	if (__builtin_constant_p(word)) {
	293	unsigned long bit = 0;
	294
	295	if (!word)
	296	return 64;
	297	if (!(word & 0xffffffff00000000UL)) {
	298	word <<= 32;
	299	bit += 32;
	300	}
	301	if (!(word & 0xffff000000000000UL)) {
	302	word <<= 16;
	303	bit += 16;
	304	}
	305	if (!(word & 0xff00000000000000UL)) {
	306	word <<= 8;
	307	bit += 8;
	308	}
	309	if (!(word & 0xf000000000000000UL)) {
	310	word <<= 4;
	311	bit += 4;
	312	}
	313	if (!(word & 0xc000000000000000UL)) {
	314	word <<= 2;
	315	bit += 2;
	316	}
	317	if (!(word & 0x8000000000000000UL)) {
	318	word <<= 1;
	319	bit += 1;
	320	}
	321	return bit;
	322	} else {
	323	register unsigned long bit asm("4") = word;
	324	register unsigned long out asm("5");
	325
	326	asm volatile(
	327	" flogr %[bit],%[bit]\n"
	328	: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
	329	return bit;
	330	}
	331	}
	332
	333	/**
	334	* __ffs - find first bit in word.
	335	* @word: The word to search
	336	*
	337	* Undefined if no bit exists, so code should check against 0 first.
	338	*/
	339	static inline unsigned long __ffs(unsigned long word)
	340	{
	341	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
	342	}
	343
344	/**
345	* ffs - find first bit set
346	* @word: the word to search
347	*
348	* This is defined the same way as the libc and
349	* compiler builtin ffs routines (man ffs).
350	*/
351	static inline int ffs(int word)
352	{
353	unsigned long mask = 2 * BITS_PER_LONG - 1;
354	unsigned int val = (unsigned int)word;
355
356	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
357	}
358
359	/**
360	* __fls - find last (most-significant) set bit in a long word
361	* @word: the word to search
362	*
363	* Undefined if no set bit exists, so code should check against 0 first.
364	*/
365	static inline unsigned long __fls(unsigned long word)
366	{
367	return __flogr(word) ^ (BITS_PER_LONG - 1);
368	}
369
370	/**
371	* fls64 - find last set bit in a 64-bit word
372	* @word: the word to search
373	*
374	* This is defined in a similar way as the libc and compiler builtin
375	* ffsll, but returns the position of the most significant set bit.
376	*
377	* fls64(value) returns 0 if value is 0 or the position of the last
378	* set bit if value is nonzero. The last (most significant) bit is
379	* at position 64.
380	*/
381	static inline int fls64(unsigned long word)
382	{
383	unsigned long mask = 2 * BITS_PER_LONG - 1;
384
385	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
386	}
387
388	/**
389	* fls - find last (most-significant) bit set
390	* @word: the word to search
391	*
392	* This is defined the same way as ffs.
393	* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
394	*/
395	static inline int fls(int word)
396	{
397	return fls64((unsigned int)word);
398	}
399
400	#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
401
746479cd HC	402	#include <asm-generic/bitops/__ffs.h>
746479cd HC	403	#include <asm-generic/bitops/ffs.h>
56a6b1eb	404	#include <asm-generic/bitops/__fls.h>
746479cd	405	#include <asm-generic/bitops/fls.h>
7e33db4e	406	#include <asm-generic/bitops/fls64.h>
b1cb7e2b HC	407
	408	#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
	409
746479cd HC	410	#include <asm-generic/bitops/ffz.h>
746479cd HC	411	#include <asm-generic/bitops/find.h>
7e33db4e	412	#include <asm-generic/bitops/hweight.h>
746479cd	413	#include <asm-generic/bitops/sched.h>
802caabb	414	#include <asm-generic/bitops/le.h>
148817ba	415	#include <asm-generic/bitops/ext2-atomic-setbit.h>
67fe9251	416
1da177e4	417	#endif /* _S390_BITOPS_H */