]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/x86/include/asm/barrier.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_BARRIER_H
3 #define _ASM_X86_BARRIER_H
5 #include <asm/alternative.h>
9 * Force strict CPU ordering.
10 * And yes, this might be required on UP too when we're talking
15 #define mb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "mfence", \
16 X86_FEATURE_XMM2) ::: "memory", "cc")
17 #define rmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "lfence", \
18 X86_FEATURE_XMM2) ::: "memory", "cc")
19 #define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \
20 X86_FEATURE_XMM2) ::: "memory", "cc")
22 #define mb() asm volatile("mfence":::"memory")
23 #define rmb() asm volatile("lfence":::"memory")
24 #define wmb() asm volatile("sfence" ::: "memory")
27 #ifdef CONFIG_X86_PPRO_FENCE
28 #define dma_rmb() rmb()
30 #define dma_rmb() barrier()
32 #define dma_wmb() barrier()
35 #define __smp_mb() asm volatile("lock; addl $0,-4(%%esp)" ::: "memory", "cc")
37 #define __smp_mb() asm volatile("lock; addl $0,-4(%%rsp)" ::: "memory", "cc")
39 #define __smp_rmb() dma_rmb()
40 #define __smp_wmb() barrier()
41 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
43 #if defined(CONFIG_X86_PPRO_FENCE)
46 * For this option x86 doesn't have a strong TSO memory
47 * model and we should fall back to full barriers.
50 #define __smp_store_release(p, v) \
52 compiletime_assert_atomic_type(*p); \
57 #define __smp_load_acquire(p) \
59 typeof(*p) ___p1 = READ_ONCE(*p); \
60 compiletime_assert_atomic_type(*p); \
65 #else /* regular x86 TSO memory ordering */
67 #define __smp_store_release(p, v) \
69 compiletime_assert_atomic_type(*p); \
74 #define __smp_load_acquire(p) \
76 typeof(*p) ___p1 = READ_ONCE(*p); \
77 compiletime_assert_atomic_type(*p); \
84 /* Atomic operations are already serializing on x86 */
85 #define __smp_mb__before_atomic() barrier()
86 #define __smp_mb__after_atomic() barrier()
88 #include <asm-generic/barrier.h>
90 #endif /* _ASM_X86_BARRIER_H */