[mirror_ubuntu-bionic-kernel.git] / arch / s390 / include / asm / bitops.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 *    Copyright IBM Corp. 1999,2013
 *
 *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
 *
 * The description below was taken in large parts from the powerpc
 * bitops header file:
 * Within a word, bits are numbered LSB first.  Lot's of places make
 * this assumption by directly testing bits with (val & (1<<nr)).
 * This can cause confusion for large (> 1 word) bitmaps on a
 * big-endian system because, unlike little endian, the number of each
 * bit depends on the word size.
 *
 * The bitop functions are defined to work on unsigned longs, so the bits
 * end up numbered:
 *   |63..............0|127............64|191...........128|255...........192|
 *
 * We also have special functions which work with an MSB0 encoding.
 * The bits are numbered:
 *   |0..............63|64............127|128...........191|192...........255|
 *
 * The main difference is that bit 0-63 in the bit number field needs to be
 * reversed compared to the LSB0 encoded bit fields. This can be achieved by
 * XOR with 0x3f.
 *
 */

#ifndef _S390_BITOPS_H
#define _S390_BITOPS_H

#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif

#include <linux/typecheck.h>
#include <linux/compiler.h>
#include <asm/atomic_ops.h>
#include <asm/barrier.h>

#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)

static inline unsigned long *
__bitops_word(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long addr;

	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
	return (unsigned long *)addr;
}

static inline unsigned char *
__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
{
	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
}

static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"oi	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (1 << (nr & 7))
			: "cc", "memory");
		return;
	}
#endif
	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	__atomic64_or(mask, addr);
}

static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"ni	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (~(1 << (nr & 7)))
			: "cc", "memory");
		return;
	}
#endif
	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	__atomic64_and(mask, addr);
}

static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long mask;

#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	if (__builtin_constant_p(nr)) {
		unsigned char *caddr = __bitops_byte(nr, ptr);

		asm volatile(
			"xi	%0,%b1\n"
			: "+Q" (*caddr)
			: "i" (1 << (nr & 7))
			: "cc", "memory");
		return;
	}
#endif
	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	__atomic64_xor(mask, addr);
}

static inline int
test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	old = __atomic64_or_barrier(mask, addr);
	return (old & mask) != 0;
}

static inline int
test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	old = __atomic64_and_barrier(mask, addr);
	return (old & ~mask) != 0;
}

static inline int
test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned long *addr = __bitops_word(nr, ptr);
	unsigned long old, mask;

	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	old = __atomic64_xor_barrier(mask, addr);
	return (old & mask) != 0;
}

static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr |= 1 << (nr & 7);
}

static inline void 
__clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr &= ~(1 << (nr & 7));
}

static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);

	*addr ^= 1 << (nr & 7);
}

static inline int
__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr |= 1 << (nr & 7);
	return (ch >> (nr & 7)) & 1;
}

static inline int
__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr &= ~(1 << (nr & 7));
	return (ch >> (nr & 7)) & 1;
}

static inline int
__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
{
	unsigned char *addr = __bitops_byte(nr, ptr);
	unsigned char ch;

	ch = *addr;
	*addr ^= 1 << (nr & 7);
	return (ch >> (nr & 7)) & 1;
}

static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
{
	const volatile unsigned char *addr;

	addr = ((const volatile unsigned char *)ptr);
	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
	return (*addr >> (nr & 7)) & 1;
}

static inline int test_and_set_bit_lock(unsigned long nr,
					volatile unsigned long *ptr)
{
	if (test_bit(nr, ptr))
		return 1;
	return test_and_set_bit(nr, ptr);
}

static inline void clear_bit_unlock(unsigned long nr,
				    volatile unsigned long *ptr)
{
	smp_mb__before_atomic();
	clear_bit(nr, ptr);
}

static inline void __clear_bit_unlock(unsigned long nr,
				      volatile unsigned long *ptr)
{
	smp_mb();
	__clear_bit(nr, ptr);
}

/*
 * Functions which use MSB0 bit numbering.
 * The bits are numbered:
 *   |0..............63|64............127|128...........191|192...........255|
 */
unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
				unsigned long offset);

#define for_each_set_bit_inv(bit, addr, size)				\
	for ((bit) = find_first_bit_inv((addr), (size));		\
	     (bit) < (size);						\
	     (bit) = find_next_bit_inv((addr), (size), (bit) + 1))

static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
{
	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

static inline int test_bit_inv(unsigned long nr,
			       const volatile unsigned long *ptr)
{
	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
}

#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES

/**
 * __flogr - find leftmost one
 * @word - The word to search
 *
 * Returns the bit number of the most significant bit set,
 * where the most significant bit has bit number 0.
 * If no bit is set this function returns 64.
 */
static inline unsigned char __flogr(unsigned long word)
{
	if (__builtin_constant_p(word)) {
		unsigned long bit = 0;

		if (!word)
			return 64;
		if (!(word & 0xffffffff00000000UL)) {
			word <<= 32;
			bit += 32;
		}
		if (!(word & 0xffff000000000000UL)) {
			word <<= 16;
			bit += 16;
		}
		if (!(word & 0xff00000000000000UL)) {
			word <<= 8;
			bit += 8;
		}
		if (!(word & 0xf000000000000000UL)) {
			word <<= 4;
			bit += 4;
		}
		if (!(word & 0xc000000000000000UL)) {
			word <<= 2;
			bit += 2;
		}
		if (!(word & 0x8000000000000000UL)) {
			word <<= 1;
			bit += 1;
		}
		return bit;
	} else {
		register unsigned long bit asm("4") = word;
		register unsigned long out asm("5");

		asm volatile(
			"       flogr   %[bit],%[bit]\n"
			: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
		return bit;
	}
}

/**
 * __ffs - find first bit in word.
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static inline unsigned long __ffs(unsigned long word)
{
	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
}

/**
 * ffs - find first bit set
 * @word: the word to search
 *
 * This is defined the same way as the libc and
 * compiler builtin ffs routines (man ffs).
 */
static inline int ffs(int word)
{
	unsigned long mask = 2 * BITS_PER_LONG - 1;
	unsigned int val = (unsigned int)word;

	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
}

/**
 * __fls - find last (most-significant) set bit in a long word
 * @word: the word to search
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
static inline unsigned long __fls(unsigned long word)
{
	return __flogr(word) ^ (BITS_PER_LONG - 1);
}

/**
 * fls64 - find last set bit in a 64-bit word
 * @word: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffsll, but returns the position of the most significant set bit.
 *
 * fls64(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 64.
 */
static inline int fls64(unsigned long word)
{
	unsigned long mask = 2 * BITS_PER_LONG - 1;

	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
}

/**
 * fls - find last (most-significant) bit set
 * @word: the word to search
 *
 * This is defined the same way as ffs.
 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 */
static inline int fls(int word)
{
	return fls64((unsigned int)word);
}

#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */

#include <asm-generic/bitops/__ffs.h>
#include <asm-generic/bitops/ffs.h>
#include <asm-generic/bitops/__fls.h>
#include <asm-generic/bitops/fls.h>
#include <asm-generic/bitops/fls64.h>

#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */

#include <asm-generic/bitops/ffz.h>
#include <asm-generic/bitops/find.h>
#include <asm-generic/bitops/hweight.h>
#include <asm-generic/bitops/sched.h>
#include <asm-generic/bitops/le.h>
#include <asm-generic/bitops/ext2-atomic-setbit.h>

#endif /* _S390_BITOPS_H */
Commit	Line	Data
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Copyright IBM Corp. 1999,2013
	4	*
	5	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
	6	*
	7	* The description below was taken in large parts from the powerpc
	8	* bitops header file:
	9	* Within a word, bits are numbered LSB first. Lot's of places make
	10	* this assumption by directly testing bits with (val & (1<<nr)).
	11	* This can cause confusion for large (> 1 word) bitmaps on a
	12	* big-endian system because, unlike little endian, the number of each
	13	* bit depends on the word size.
	14	*
	15	* The bitop functions are defined to work on unsigned longs, so the bits
	16	* end up numbered:
	17	* \|63..............0\|127............64\|191...........128\|255...........192\|
	18	*
	19	* We also have special functions which work with an MSB0 encoding.
	20	* The bits are numbered:
	21	* \|0..............63\|64............127\|128...........191\|192...........255\|
	22	*
	23	* The main difference is that bit 0-63 in the bit number field needs to be
	24	* reversed compared to the LSB0 encoded bit fields. This can be achieved by
	25	* XOR with 0x3f.
	26	*
	27	*/
	28
	29	#ifndef _S390_BITOPS_H
	30	#define _S390_BITOPS_H
	31
	32	#ifndef _LINUX_BITOPS_H
	33	#error only <linux/bitops.h> can be included directly
	34	#endif
	35
	36	#include <linux/typecheck.h>
	37	#include <linux/compiler.h>
	38	#include <asm/atomic_ops.h>
	39	#include <asm/barrier.h>
	40
	41	#define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG)
	42
	43	static inline unsigned long *
	44	__bitops_word(unsigned long nr, volatile unsigned long *ptr)
	45	{
	46	unsigned long addr;
	47
	48	addr = (unsigned long)ptr + ((nr ^ (nr & (BITS_PER_LONG - 1))) >> 3);
	49	return (unsigned long *)addr;
	50	}
	51
	52	static inline unsigned char *
	53	__bitops_byte(unsigned long nr, volatile unsigned long *ptr)
	54	{
	55	return ((unsigned char *)ptr) + ((nr ^ (BITS_PER_LONG - 8)) >> 3);
	56	}
	57
	58	static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
	59	{
	60	unsigned long *addr = __bitops_word(nr, ptr);
	61	unsigned long mask;
	62
	63	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	64	if (__builtin_constant_p(nr)) {
	65	unsigned char *caddr = __bitops_byte(nr, ptr);
	66
	67	asm volatile(
	68	"oi %0,%b1\n"
	69	: "+Q" (*caddr)
	70	: "i" (1 << (nr & 7))
	71	: "cc", "memory");
	72	return;
	73	}
	74	#endif
	75	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	76	__atomic64_or(mask, addr);
	77	}
	78
	79	static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
	80	{
	81	unsigned long *addr = __bitops_word(nr, ptr);
	82	unsigned long mask;
	83
	84	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	85	if (__builtin_constant_p(nr)) {
	86	unsigned char *caddr = __bitops_byte(nr, ptr);
	87
	88	asm volatile(
	89	"ni %0,%b1\n"
	90	: "+Q" (*caddr)
	91	: "i" (~(1 << (nr & 7)))
	92	: "cc", "memory");
	93	return;
	94	}
	95	#endif
	96	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	97	__atomic64_and(mask, addr);
	98	}
	99
	100	static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
	101	{
	102	unsigned long *addr = __bitops_word(nr, ptr);
	103	unsigned long mask;
	104
	105	#ifdef CONFIG_HAVE_MARCH_ZEC12_FEATURES
	106	if (__builtin_constant_p(nr)) {
	107	unsigned char *caddr = __bitops_byte(nr, ptr);
	108
	109	asm volatile(
	110	"xi %0,%b1\n"
	111	: "+Q" (*caddr)
	112	: "i" (1 << (nr & 7))
	113	: "cc", "memory");
	114	return;
	115	}
	116	#endif
	117	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	118	__atomic64_xor(mask, addr);
	119	}
	120
	121	static inline int
	122	test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
	123	{
	124	unsigned long *addr = __bitops_word(nr, ptr);
	125	unsigned long old, mask;
	126
	127	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	128	old = __atomic64_or_barrier(mask, addr);
	129	return (old & mask) != 0;
	130	}
	131
	132	static inline int
	133	test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
	134	{
	135	unsigned long *addr = __bitops_word(nr, ptr);
	136	unsigned long old, mask;
	137
	138	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
	139	old = __atomic64_and_barrier(mask, addr);
	140	return (old & ~mask) != 0;
	141	}
	142
	143	static inline int
	144	test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
	145	{
	146	unsigned long *addr = __bitops_word(nr, ptr);
	147	unsigned long old, mask;
	148
	149	mask = 1UL << (nr & (BITS_PER_LONG - 1));
	150	old = __atomic64_xor_barrier(mask, addr);
	151	return (old & mask) != 0;
	152	}
	153
	154	static inline void __set_bit(unsigned long nr, volatile unsigned long *ptr)
	155	{
	156	unsigned char *addr = __bitops_byte(nr, ptr);
	157
	158	*addr \|= 1 << (nr & 7);
	159	}
	160
	161	static inline void
	162	__clear_bit(unsigned long nr, volatile unsigned long *ptr)
	163	{
	164	unsigned char *addr = __bitops_byte(nr, ptr);
	165
	166	*addr &= ~(1 << (nr & 7));
	167	}
	168
	169	static inline void __change_bit(unsigned long nr, volatile unsigned long *ptr)
	170	{
	171	unsigned char *addr = __bitops_byte(nr, ptr);
	172
	173	*addr ^= 1 << (nr & 7);
	174	}
	175
	176	static inline int
	177	__test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
	178	{
	179	unsigned char *addr = __bitops_byte(nr, ptr);
	180	unsigned char ch;
	181
	182	ch = *addr;
	183	*addr \|= 1 << (nr & 7);
	184	return (ch >> (nr & 7)) & 1;
	185	}
	186
	187	static inline int
	188	__test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
	189	{
	190	unsigned char *addr = __bitops_byte(nr, ptr);
	191	unsigned char ch;
	192
	193	ch = *addr;
	194	*addr &= ~(1 << (nr & 7));
	195	return (ch >> (nr & 7)) & 1;
	196	}
	197
	198	static inline int
	199	__test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
	200	{
	201	unsigned char *addr = __bitops_byte(nr, ptr);
	202	unsigned char ch;
	203
	204	ch = *addr;
	205	*addr ^= 1 << (nr & 7);
	206	return (ch >> (nr & 7)) & 1;
	207	}
	208
	209	static inline int test_bit(unsigned long nr, const volatile unsigned long *ptr)
	210	{
	211	const volatile unsigned char *addr;
	212
	213	addr = ((const volatile unsigned char *)ptr);
	214	addr += (nr ^ (BITS_PER_LONG - 8)) >> 3;
	215	return (*addr >> (nr & 7)) & 1;
	216	}
	217
	218	static inline int test_and_set_bit_lock(unsigned long nr,
	219	volatile unsigned long *ptr)
	220	{
	221	if (test_bit(nr, ptr))
	222	return 1;
	223	return test_and_set_bit(nr, ptr);
	224	}
	225
	226	static inline void clear_bit_unlock(unsigned long nr,
	227	volatile unsigned long *ptr)
	228	{
	229	smp_mb__before_atomic();
	230	clear_bit(nr, ptr);
	231	}
	232
	233	static inline void __clear_bit_unlock(unsigned long nr,
	234	volatile unsigned long *ptr)
	235	{
	236	smp_mb();
	237	__clear_bit(nr, ptr);
	238	}
	239
	240	/*
	241	* Functions which use MSB0 bit numbering.
	242	* The bits are numbered:
	243	* \|0..............63\|64............127\|128...........191\|192...........255\|
	244	*/
	245	unsigned long find_first_bit_inv(const unsigned long *addr, unsigned long size);
	246	unsigned long find_next_bit_inv(const unsigned long *addr, unsigned long size,
	247	unsigned long offset);
	248
	249	#define for_each_set_bit_inv(bit, addr, size) \
	250	for ((bit) = find_first_bit_inv((addr), (size)); \
	251	(bit) < (size); \
	252	(bit) = find_next_bit_inv((addr), (size), (bit) + 1))
	253
	254	static inline void set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	255	{
	256	return set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	257	}
	258
	259	static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	260	{
	261	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	262	}
	263
	264	static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	265	{
	266	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	267	}
	268
	269	static inline void __clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
	270	{
	271	return __clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	272	}
	273
	274	static inline int test_bit_inv(unsigned long nr,
	275	const volatile unsigned long *ptr)
	276	{
	277	return test_bit(nr ^ (BITS_PER_LONG - 1), ptr);
	278	}
	279
	280	#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
	281
	282	/**
	283	* __flogr - find leftmost one
	284	* @word - The word to search
	285	*
	286	* Returns the bit number of the most significant bit set,
	287	* where the most significant bit has bit number 0.
	288	* If no bit is set this function returns 64.
	289	*/
	290	static inline unsigned char __flogr(unsigned long word)
	291	{
	292	if (__builtin_constant_p(word)) {
	293	unsigned long bit = 0;
	294
	295	if (!word)
	296	return 64;
	297	if (!(word & 0xffffffff00000000UL)) {
	298	word <<= 32;
	299	bit += 32;
	300	}
	301	if (!(word & 0xffff000000000000UL)) {
	302	word <<= 16;
	303	bit += 16;
	304	}
	305	if (!(word & 0xff00000000000000UL)) {
	306	word <<= 8;
	307	bit += 8;
	308	}
	309	if (!(word & 0xf000000000000000UL)) {
	310	word <<= 4;
	311	bit += 4;
	312	}
	313	if (!(word & 0xc000000000000000UL)) {
	314	word <<= 2;
	315	bit += 2;
	316	}
	317	if (!(word & 0x8000000000000000UL)) {
	318	word <<= 1;
	319	bit += 1;
	320	}
	321	return bit;
	322	} else {
	323	register unsigned long bit asm("4") = word;
	324	register unsigned long out asm("5");
	325
	326	asm volatile(
	327	" flogr %[bit],%[bit]\n"
	328	: [bit] "+d" (bit), [out] "=d" (out) : : "cc");
	329	return bit;
	330	}
	331	}
	332
	333	/**
	334	* __ffs - find first bit in word.
	335	* @word: The word to search
	336	*
	337	* Undefined if no bit exists, so code should check against 0 first.
	338	*/
	339	static inline unsigned long __ffs(unsigned long word)
	340	{
	341	return __flogr(-word & word) ^ (BITS_PER_LONG - 1);
	342	}
	343
	344	/**
	345	* ffs - find first bit set
	346	* @word: the word to search
	347	*
	348	* This is defined the same way as the libc and
	349	* compiler builtin ffs routines (man ffs).
	350	*/
	351	static inline int ffs(int word)
	352	{
	353	unsigned long mask = 2 * BITS_PER_LONG - 1;
	354	unsigned int val = (unsigned int)word;
	355
	356	return (1 + (__flogr(-val & val) ^ (BITS_PER_LONG - 1))) & mask;
	357	}
	358
	359	/**
	360	* __fls - find last (most-significant) set bit in a long word
	361	* @word: the word to search
	362	*
	363	* Undefined if no set bit exists, so code should check against 0 first.
	364	*/
	365	static inline unsigned long __fls(unsigned long word)
	366	{
	367	return __flogr(word) ^ (BITS_PER_LONG - 1);
	368	}
	369
	370	/**
	371	* fls64 - find last set bit in a 64-bit word
	372	* @word: the word to search
	373	*
	374	* This is defined in a similar way as the libc and compiler builtin
	375	* ffsll, but returns the position of the most significant set bit.
	376	*
	377	* fls64(value) returns 0 if value is 0 or the position of the last
	378	* set bit if value is nonzero. The last (most significant) bit is
	379	* at position 64.
	380	*/
	381	static inline int fls64(unsigned long word)
	382	{
	383	unsigned long mask = 2 * BITS_PER_LONG - 1;
	384
	385	return (1 + (__flogr(word) ^ (BITS_PER_LONG - 1))) & mask;
	386	}
	387
	388	/**
	389	* fls - find last (most-significant) bit set
	390	* @word: the word to search
	391	*
	392	* This is defined the same way as ffs.
	393	* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
	394	*/
	395	static inline int fls(int word)
	396	{
	397	return fls64((unsigned int)word);
	398	}
	399
	400	#else /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
	401
	402	#include <asm-generic/bitops/__ffs.h>
	403	#include <asm-generic/bitops/ffs.h>
	404	#include <asm-generic/bitops/__fls.h>
	405	#include <asm-generic/bitops/fls.h>
	406	#include <asm-generic/bitops/fls64.h>
	407
	408	#endif /* CONFIG_HAVE_MARCH_Z9_109_FEATURES */
	409
	410	#include <asm-generic/bitops/ffz.h>
	411	#include <asm-generic/bitops/find.h>
	412	#include <asm-generic/bitops/hweight.h>
	413	#include <asm-generic/bitops/sched.h>
	414	#include <asm-generic/bitops/le.h>
	415	#include <asm-generic/bitops/ext2-atomic-setbit.h>
	416
	417	#endif /* _S390_BITOPS_H */