]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/arm64/include/asm/spinlock.h
2 * Copyright (C) 2012 ARM Ltd.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #ifndef __ASM_SPINLOCK_H
17 #define __ASM_SPINLOCK_H
20 #include <asm/spinlock_types.h>
21 #include <asm/processor.h>
24 * Spinlock implementation.
26 * The memory barriers are implicit with the load-acquire and store-release
29 static inline void arch_spin_unlock_wait(arch_spinlock_t
*lock
)
32 arch_spinlock_t lockval
;
36 * Ensure prior spin_lock operations to other locks have completed
37 * on this CPU before we test whether "lock" is locked.
40 owner
= READ_ONCE(lock
->owner
) << 16;
46 /* Is the lock free? */
47 " eor %w1, %w0, %w0, ror #16\n"
49 /* Lock taken -- has there been a subsequent unlock->lock transition? */
50 " eor %w1, %w3, %w0, lsl #16\n"
53 * The owner has been updated, so there was an unlock->lock
54 * transition that we missed. That means we can rely on the
55 * store-release of the unlock operation paired with the
56 * load-acquire of the lock operation to publish any of our
57 * previous stores to the new lock owner and therefore don't
58 * need to bother with the writeback below.
63 * Serialise against any concurrent lockers by writing back the
66 ARM64_LSE_ATOMIC_INSN(
68 " stxr %w1, %w0, %2\n"
74 " eor %w1, %w1, %w0\n")
75 /* Somebody else wrote to the lock, GOTO 10 and reload the value */
78 : "=&r" (lockval
), "=&r" (tmp
), "+Q" (*lock
)
83 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
85 static inline void arch_spin_lock(arch_spinlock_t
*lock
)
88 arch_spinlock_t lockval
, newval
;
91 /* Atomically increment the next ticket. */
92 ARM64_LSE_ATOMIC_INSN(
94 " prfm pstl1strm, %3\n"
96 " add %w1, %w0, %w5\n"
97 " stxr %w2, %w1, %3\n"
101 " ldadda %w2, %w0, %3\n"
107 /* Did we get the lock? */
108 " eor %w1, %w0, %w0, ror #16\n"
111 * No: spin on the owner. Send a local event to avoid missing an
112 * unlock before the exclusive load.
117 " eor %w1, %w2, %w0, lsr #16\n"
119 /* We got the lock. Critical section starts here. */
121 : "=&r" (lockval
), "=&r" (newval
), "=&r" (tmp
), "+Q" (*lock
)
122 : "Q" (lock
->owner
), "I" (1 << TICKET_SHIFT
)
126 static inline int arch_spin_trylock(arch_spinlock_t
*lock
)
129 arch_spinlock_t lockval
;
131 asm volatile(ARM64_LSE_ATOMIC_INSN(
133 " prfm pstl1strm, %2\n"
135 " eor %w1, %w0, %w0, ror #16\n"
137 " add %w0, %w0, %3\n"
138 " stxr %w1, %w0, %2\n"
143 " eor %w1, %w0, %w0, ror #16\n"
145 " add %w1, %w0, %3\n"
146 " casa %w0, %w1, %2\n"
147 " and %w1, %w1, #0xffff\n"
148 " eor %w1, %w1, %w0, lsr #16\n"
150 : "=&r" (lockval
), "=&r" (tmp
), "+Q" (*lock
)
151 : "I" (1 << TICKET_SHIFT
)
157 static inline void arch_spin_unlock(arch_spinlock_t
*lock
)
161 asm volatile(ARM64_LSE_ATOMIC_INSN(
164 " add %w1, %w1, #1\n"
170 : "=Q" (lock
->owner
), "=&r" (tmp
)
175 static inline int arch_spin_value_unlocked(arch_spinlock_t lock
)
177 return lock
.owner
== lock
.next
;
180 static inline int arch_spin_is_locked(arch_spinlock_t
*lock
)
182 smp_mb(); /* See arch_spin_unlock_wait */
183 return !arch_spin_value_unlocked(READ_ONCE(*lock
));
186 static inline int arch_spin_is_contended(arch_spinlock_t
*lock
)
188 arch_spinlock_t lockval
= READ_ONCE(*lock
);
189 return (lockval
.next
- lockval
.owner
) > 1;
191 #define arch_spin_is_contended arch_spin_is_contended
194 * Write lock implementation.
196 * Write locks set bit 31. Unlocking, is done by writing 0 since the lock is
199 * The memory barriers are implicit with the load-acquire and store-release
203 static inline void arch_write_lock(arch_rwlock_t
*rw
)
207 asm volatile(ARM64_LSE_ATOMIC_INSN(
213 " stxr %w0, %w2, %1\n"
218 "2: casa %w0, %w2, %1\n"
225 : "=&r" (tmp
), "+Q" (rw
->lock
)
230 static inline int arch_write_trylock(arch_rwlock_t
*rw
)
234 asm volatile(ARM64_LSE_ATOMIC_INSN(
238 " stxr %w0, %w2, %1\n"
243 " casa %w0, %w2, %1\n"
246 : "=&r" (tmp
), "+Q" (rw
->lock
)
253 static inline void arch_write_unlock(arch_rwlock_t
*rw
)
255 asm volatile(ARM64_LSE_ATOMIC_INSN(
257 " swpl wzr, wzr, %0")
258 : "=Q" (rw
->lock
) :: "memory");
261 /* write_can_lock - would write_trylock() succeed? */
262 #define arch_write_can_lock(x) ((x)->lock == 0)
265 * Read lock implementation.
267 * It exclusively loads the lock value, increments it and stores the new value
268 * back if positive and the CPU still exclusively owns the location. If the
269 * value is negative, the lock is already held.
271 * During unlocking there may be multiple active read locks but no write lock.
273 * The memory barriers are implicit with the load-acquire and store-release
276 * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC
277 * and LSE implementations may exhibit different behaviour (although this
278 * will have no effect on lockdep).
280 static inline void arch_read_lock(arch_rwlock_t
*rw
)
282 unsigned int tmp
, tmp2
;
286 ARM64_LSE_ATOMIC_INSN(
290 " add %w0, %w0, #1\n"
291 " tbnz %w0, #31, 1b\n"
292 " stxr %w1, %w0, %2\n"
298 " adds %w1, %w0, #1\n"
299 " tbnz %w1, #31, 1b\n"
300 " casa %w0, %w1, %2\n"
301 " sbc %w0, %w1, %w0\n"
303 : "=&r" (tmp
), "=&r" (tmp2
), "+Q" (rw
->lock
)
308 static inline void arch_read_unlock(arch_rwlock_t
*rw
)
310 unsigned int tmp
, tmp2
;
312 asm volatile(ARM64_LSE_ATOMIC_INSN(
315 " sub %w0, %w0, #1\n"
316 " stlxr %w1, %w0, %2\n"
323 : "=&r" (tmp
), "=&r" (tmp2
), "+Q" (rw
->lock
)
328 static inline int arch_read_trylock(arch_rwlock_t
*rw
)
330 unsigned int tmp
, tmp2
;
332 asm volatile(ARM64_LSE_ATOMIC_INSN(
336 " add %w0, %w0, #1\n"
337 " tbnz %w0, #31, 2f\n"
338 " stxr %w1, %w0, %2\n"
343 " adds %w1, %w0, #1\n"
344 " tbnz %w1, #31, 1f\n"
345 " casa %w0, %w1, %2\n"
346 " sbc %w1, %w1, %w0\n"
349 : "=&r" (tmp
), "=&r" (tmp2
), "+Q" (rw
->lock
)
356 /* read_can_lock - would read_trylock() succeed? */
357 #define arch_read_can_lock(x) ((x)->lock < 0x80000000)
359 #define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
360 #define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
362 #define arch_spin_relax(lock) cpu_relax()
363 #define arch_read_relax(lock) cpu_relax()
364 #define arch_write_relax(lock) cpu_relax()
367 * Accesses appearing in program order before a spin_lock() operation
368 * can be reordered with accesses inside the critical section, by virtue
369 * of arch_spin_lock being constructed using acquire semantics.
371 * In cases where this is problematic (e.g. try_to_wake_up), an
372 * smp_mb__before_spinlock() can restore the required ordering.
374 #define smp_mb__before_spinlock() smp_mb()
376 #endif /* __ASM_SPINLOCK_H */