]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
19c5d690 WL |
2 | /* |
3 | * The owner field of the rw_semaphore structure will be set to | |
940a3b2b | 4 | * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear |
19c5d690 WL |
5 | * the owner field when it unlocks. A reader, on the other hand, will |
6 | * not touch the owner field when it unlocks. | |
7 | * | |
940a3b2b | 8 | * In essence, the owner field now has the following 4 states: |
19c5d690 WL |
9 | * 1) 0 |
10 | * - lock is free or the owner hasn't set the field yet | |
11 | * 2) RWSEM_READER_OWNED | |
12 | * - lock is currently or previously owned by readers (lock is free | |
13 | * or not set by owner yet) | |
940a3b2b WL |
14 | * 3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well |
15 | * - lock is owned by an anonymous writer, so spinning on the lock | |
16 | * owner should be disabled. | |
17 | * 4) Other non-zero value | |
18 | * - a writer owns the lock and other writers can spin on the lock owner. | |
19c5d690 | 19 | */ |
940a3b2b WL |
20 | #define RWSEM_ANONYMOUSLY_OWNED (1UL << 0) |
21 | #define RWSEM_READER_OWNED ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED) | |
19c5d690 | 22 | |
7a215f89 | 23 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
fb6a44f3 WL |
24 | /* |
25 | * All writes to owner are protected by WRITE_ONCE() to make sure that | |
26 | * store tearing can't happen as optimistic spinners may read and use | |
27 | * the owner value concurrently without lock. Read from owner, however, | |
28 | * may not need READ_ONCE() as long as the pointer value is only used | |
29 | * for comparison and isn't being dereferenced. | |
30 | */ | |
7a215f89 DB |
31 | static inline void rwsem_set_owner(struct rw_semaphore *sem) |
32 | { | |
fb6a44f3 | 33 | WRITE_ONCE(sem->owner, current); |
7a215f89 DB |
34 | } |
35 | ||
36 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | |
37 | { | |
fb6a44f3 | 38 | WRITE_ONCE(sem->owner, NULL); |
7a215f89 DB |
39 | } |
40 | ||
19c5d690 WL |
41 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) |
42 | { | |
43 | /* | |
44 | * We check the owner value first to make sure that we will only | |
45 | * do a write to the rwsem cacheline when it is really necessary | |
46 | * to minimize cacheline contention. | |
47 | */ | |
48 | if (sem->owner != RWSEM_READER_OWNED) | |
fb6a44f3 | 49 | WRITE_ONCE(sem->owner, RWSEM_READER_OWNED); |
19c5d690 WL |
50 | } |
51 | ||
940a3b2b WL |
52 | /* |
53 | * Return true if the a rwsem waiter can spin on the rwsem's owner | |
54 | * and steal the lock, i.e. the lock is not anonymously owned. | |
55 | * N.B. !owner is considered spinnable. | |
56 | */ | |
57 | static inline bool is_rwsem_owner_spinnable(struct task_struct *owner) | |
19c5d690 | 58 | { |
940a3b2b | 59 | return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED); |
19c5d690 WL |
60 | } |
61 | ||
940a3b2b WL |
62 | /* |
63 | * Return true if rwsem is owned by an anonymous writer or readers. | |
64 | */ | |
65 | static inline bool rwsem_has_anonymous_owner(struct task_struct *owner) | |
19c5d690 | 66 | { |
940a3b2b | 67 | return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED; |
19c5d690 | 68 | } |
7a215f89 DB |
69 | #else |
70 | static inline void rwsem_set_owner(struct rw_semaphore *sem) | |
71 | { | |
72 | } | |
73 | ||
74 | static inline void rwsem_clear_owner(struct rw_semaphore *sem) | |
75 | { | |
76 | } | |
19c5d690 WL |
77 | |
78 | static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) | |
79 | { | |
80 | } | |
7a215f89 | 81 | #endif |