]>
Commit | Line | Data |
---|---|---|
1 | /* SPDX-License-Identifier: GPL-2.0 */ | |
2 | #ifndef _LINUX_PERCPU_RWSEM_H | |
3 | #define _LINUX_PERCPU_RWSEM_H | |
4 | ||
5 | #include <linux/atomic.h> | |
6 | #include <linux/rwsem.h> | |
7 | #include <linux/percpu.h> | |
8 | #include <linux/rcuwait.h> | |
9 | #include <linux/rcu_sync.h> | |
10 | #include <linux/lockdep.h> | |
11 | ||
12 | struct percpu_rw_semaphore { | |
13 | struct rcu_sync rss; | |
14 | unsigned int __percpu *read_count; | |
15 | struct rw_semaphore rw_sem; /* slowpath */ | |
16 | struct rcuwait writer; /* blocked writer */ | |
17 | int readers_block; | |
18 | }; | |
19 | ||
20 | #define DEFINE_STATIC_PERCPU_RWSEM(name) \ | |
21 | static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ | |
22 | static struct percpu_rw_semaphore name = { \ | |
23 | .rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC), \ | |
24 | .read_count = &__percpu_rwsem_rc_##name, \ | |
25 | .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ | |
26 | .writer = __RCUWAIT_INITIALIZER(name.writer), \ | |
27 | } | |
28 | ||
29 | extern int __percpu_down_read(struct percpu_rw_semaphore *, int); | |
30 | extern void __percpu_up_read(struct percpu_rw_semaphore *); | |
31 | ||
32 | static inline void percpu_down_read_preempt_disable(struct percpu_rw_semaphore *sem) | |
33 | { | |
34 | might_sleep(); | |
35 | ||
36 | rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); | |
37 | ||
38 | preempt_disable(); | |
39 | /* | |
40 | * We are in an RCU-sched read-side critical section, so the writer | |
41 | * cannot both change sem->state from readers_fast and start checking | |
42 | * counters while we are here. So if we see !sem->state, we know that | |
43 | * the writer won't be checking until we're past the preempt_enable() | |
44 | * and that one the synchronize_sched() is done, the writer will see | |
45 | * anything we did within this RCU-sched read-size critical section. | |
46 | */ | |
47 | __this_cpu_inc(*sem->read_count); | |
48 | if (unlikely(!rcu_sync_is_idle(&sem->rss))) | |
49 | __percpu_down_read(sem, false); /* Unconditional memory barrier */ | |
50 | barrier(); | |
51 | /* | |
52 | * The barrier() prevents the compiler from | |
53 | * bleeding the critical section out. | |
54 | */ | |
55 | } | |
56 | ||
57 | static inline void percpu_down_read(struct percpu_rw_semaphore *sem) | |
58 | { | |
59 | percpu_down_read_preempt_disable(sem); | |
60 | preempt_enable(); | |
61 | } | |
62 | ||
63 | static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem) | |
64 | { | |
65 | int ret = 1; | |
66 | ||
67 | preempt_disable(); | |
68 | /* | |
69 | * Same as in percpu_down_read(). | |
70 | */ | |
71 | __this_cpu_inc(*sem->read_count); | |
72 | if (unlikely(!rcu_sync_is_idle(&sem->rss))) | |
73 | ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */ | |
74 | preempt_enable(); | |
75 | /* | |
76 | * The barrier() from preempt_enable() prevents the compiler from | |
77 | * bleeding the critical section out. | |
78 | */ | |
79 | ||
80 | if (ret) | |
81 | rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_); | |
82 | ||
83 | return ret; | |
84 | } | |
85 | ||
86 | static inline void percpu_up_read_preempt_enable(struct percpu_rw_semaphore *sem) | |
87 | { | |
88 | /* | |
89 | * The barrier() prevents the compiler from | |
90 | * bleeding the critical section out. | |
91 | */ | |
92 | barrier(); | |
93 | /* | |
94 | * Same as in percpu_down_read(). | |
95 | */ | |
96 | if (likely(rcu_sync_is_idle(&sem->rss))) | |
97 | __this_cpu_dec(*sem->read_count); | |
98 | else | |
99 | __percpu_up_read(sem); /* Unconditional memory barrier */ | |
100 | preempt_enable(); | |
101 | ||
102 | rwsem_release(&sem->rw_sem.dep_map, 1, _RET_IP_); | |
103 | } | |
104 | ||
105 | static inline void percpu_up_read(struct percpu_rw_semaphore *sem) | |
106 | { | |
107 | preempt_disable(); | |
108 | percpu_up_read_preempt_enable(sem); | |
109 | } | |
110 | ||
111 | extern void percpu_down_write(struct percpu_rw_semaphore *); | |
112 | extern void percpu_up_write(struct percpu_rw_semaphore *); | |
113 | ||
114 | extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, | |
115 | const char *, struct lock_class_key *); | |
116 | ||
117 | extern void percpu_free_rwsem(struct percpu_rw_semaphore *); | |
118 | ||
119 | #define percpu_init_rwsem(sem) \ | |
120 | ({ \ | |
121 | static struct lock_class_key rwsem_key; \ | |
122 | __percpu_init_rwsem(sem, #sem, &rwsem_key); \ | |
123 | }) | |
124 | ||
125 | #define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem) | |
126 | ||
127 | #define percpu_rwsem_assert_held(sem) \ | |
128 | lockdep_assert_held(&(sem)->rw_sem) | |
129 | ||
130 | static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, | |
131 | bool read, unsigned long ip) | |
132 | { | |
133 | lock_release(&sem->rw_sem.dep_map, 1, ip); | |
134 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | |
135 | if (!read) | |
136 | sem->rw_sem.owner = RWSEM_OWNER_UNKNOWN; | |
137 | #endif | |
138 | } | |
139 | ||
140 | static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, | |
141 | bool read, unsigned long ip) | |
142 | { | |
143 | lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip); | |
144 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | |
145 | if (!read) | |
146 | sem->rw_sem.owner = current; | |
147 | #endif | |
148 | } | |
149 | ||
150 | #endif |