[mirror_ubuntu-focal-kernel.git] / arch / arm64 / include / asm / barrier.h

/*
 * Based on arch/arm/include/asm/barrier.h
 *
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_BARRIER_H
#define __ASM_BARRIER_H

#ifndef __ASSEMBLY__

#define sev()		asm volatile("sev" : : : "memory")
#define wfe()		asm volatile("wfe" : : : "memory")
#define wfi()		asm volatile("wfi" : : : "memory")

#define isb()		asm volatile("isb" : : : "memory")
#define dmb(opt)	asm volatile("dmb " #opt : : : "memory")
#define dsb(opt)	asm volatile("dsb " #opt : : : "memory")

#define mb()		dsb(sy)
#define rmb()		dsb(ld)
#define wmb()		dsb(st)

#ifndef CONFIG_SMP
#define smp_mb()	barrier()
#define smp_rmb()	barrier()
#define smp_wmb()	barrier()

#define smp_store_release(p, v)						\
do {									\
	compiletime_assert_atomic_type(*p);				\
	barrier();							\
	ACCESS_ONCE(*p) = (v);						\
} while (0)

#define smp_load_acquire(p)						\
({									\
	typeof(*p) ___p1 = ACCESS_ONCE(*p);				\
	compiletime_assert_atomic_type(*p);				\
	barrier();							\
	___p1;								\
})

#else

#define smp_mb()	dmb(ish)
#define smp_rmb()	dmb(ishld)
#define smp_wmb()	dmb(ishst)

#define smp_store_release(p, v)						\
do {									\
	compiletime_assert_atomic_type(*p);				\
	switch (sizeof(*p)) {						\
	case 4:								\
		asm volatile ("stlr %w1, %0"				\
				: "=Q" (*p) : "r" (v) : "memory");	\
		break;							\
	case 8:								\
		asm volatile ("stlr %1, %0"				\
				: "=Q" (*p) : "r" (v) : "memory");	\
		break;							\
	}								\
} while (0)

#define smp_load_acquire(p)						\
({									\
	typeof(*p) ___p1;						\
	compiletime_assert_atomic_type(*p);				\
	switch (sizeof(*p)) {						\
	case 4:								\
		asm volatile ("ldar %w0, %1"				\
			: "=r" (___p1) : "Q" (*p) : "memory");		\
		break;							\
	case 8:								\
		asm volatile ("ldar %0, %1"				\
			: "=r" (___p1) : "Q" (*p) : "memory");		\
		break;							\
	}								\
	___p1;								\
})

#endif

#define read_barrier_depends()		do { } while(0)
#define smp_read_barrier_depends()	do { } while(0)

#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
#define nop()		asm volatile("nop");

#define smp_mb__before_atomic()	smp_mb()
#define smp_mb__after_atomic()	smp_mb()

#endif	/* __ASSEMBLY__ */

#endif	/* __ASM_BARRIER_H */
Commit	Line	Data
10b663ae CM	1	/*
	2	* Based on arch/arm/include/asm/barrier.h
	3	*
	4	* Copyright (C) 2012 ARM Ltd.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18	#ifndef __ASM_BARRIER_H
	19	#define __ASM_BARRIER_H
	20
	21	#ifndef __ASSEMBLY__
	22
	23	#define sev() asm volatile("sev" : : : "memory")
	24	#define wfe() asm volatile("wfe" : : : "memory")
	25	#define wfi() asm volatile("wfi" : : : "memory")
	26
	27	#define isb() asm volatile("isb" : : : "memory")
493e6874 WD	28	#define dmb(opt) asm volatile("dmb " #opt : : : "memory")
493e6874 WD	29	#define dsb(opt) asm volatile("dsb " #opt : : : "memory")
10b663ae	30
98f7685e	31	#define mb() dsb(sy)
493e6874 WD	32	#define rmb() dsb(ld)
493e6874 WD	33	#define wmb() dsb(st)
10b663ae CM	34
	35	#ifndef CONFIG_SMP
	36	#define smp_mb() barrier()
	37	#define smp_rmb() barrier()
	38	#define smp_wmb() barrier()
47933ad4 PZ	39
	40	#define smp_store_release(p, v) \
	41	do { \
	42	compiletime_assert_atomic_type(*p); \
be6209a6	43	barrier(); \
47933ad4 PZ	44	ACCESS_ONCE(*p) = (v); \
	45	} while (0)
	46
	47	#define smp_load_acquire(p) \
	48	({ \
	49	typeof(p) ___p1 = ACCESS_ONCE(p); \
	50	compiletime_assert_atomic_type(*p); \
be6209a6	51	barrier(); \
47933ad4 PZ	52	___p1; \
	53	})
	54
10b663ae	55	#else
47933ad4	56
493e6874 WD	57	#define smp_mb() dmb(ish)
	58	#define smp_rmb() dmb(ishld)
	59	#define smp_wmb() dmb(ishst)
47933ad4 PZ	60
	61	#define smp_store_release(p, v) \
	62	do { \
	63	compiletime_assert_atomic_type(*p); \
	64	switch (sizeof(*p)) { \
	65	case 4: \
	66	asm volatile ("stlr %w1, %0" \
	67	: "=Q" (*p) : "r" (v) : "memory"); \
	68	break; \
	69	case 8: \
	70	asm volatile ("stlr %1, %0" \
	71	: "=Q" (*p) : "r" (v) : "memory"); \
	72	break; \
	73	} \
	74	} while (0)
	75
	76	#define smp_load_acquire(p) \
	77	({ \
	78	typeof(*p) ___p1; \
	79	compiletime_assert_atomic_type(*p); \
	80	switch (sizeof(*p)) { \
	81	case 4: \
	82	asm volatile ("ldar %w0, %1" \
	83	: "=r" (___p1) : "Q" (*p) : "memory"); \
	84	break; \
	85	case 8: \
	86	asm volatile ("ldar %0, %1" \
	87	: "=r" (___p1) : "Q" (*p) : "memory"); \
	88	break; \
	89	} \
	90	___p1; \
	91	})
	92
10b663ae CM	93	#endif
	94
	95	#define read_barrier_depends() do { } while(0)
	96	#define smp_read_barrier_depends() do { } while(0)
	97
	98	#define set_mb(var, value) do { var = value; smp_mb(); } while (0)
	99	#define nop() asm volatile("nop");
	100
8715466b PZ	101	#define smp_mb__before_atomic() smp_mb()
	102	#define smp_mb__after_atomic() smp_mb()
	103
10b663ae CM	104	#endif /* __ASSEMBLY__ */
	105
	106	#endif /* __ASM_BARRIER_H */