]>
Commit | Line | Data |
---|---|---|
62ac665f MP |
1 | #ifndef _LINUX_PERCPU_RWSEM_H |
2 | #define _LINUX_PERCPU_RWSEM_H | |
3 | ||
9390ef0c | 4 | #include <linux/atomic.h> |
a1fd3e24 | 5 | #include <linux/rwsem.h> |
62ac665f | 6 | #include <linux/percpu.h> |
a1fd3e24 | 7 | #include <linux/wait.h> |
001dac62 | 8 | #include <linux/rcu_sync.h> |
8ebe3473 | 9 | #include <linux/lockdep.h> |
62ac665f MP |
10 | |
11 | struct percpu_rw_semaphore { | |
001dac62 | 12 | struct rcu_sync rss; |
80127a39 | 13 | unsigned int __percpu *read_count; |
a1fd3e24 | 14 | struct rw_semaphore rw_sem; |
80127a39 PZ |
15 | wait_queue_head_t writer; |
16 | int readers_block; | |
62ac665f MP |
17 | }; |
18 | ||
11d9684c PZ |
19 | #define DEFINE_STATIC_PERCPU_RWSEM(name) \ |
20 | static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_rc_##name); \ | |
21 | static struct percpu_rw_semaphore name = { \ | |
22 | .rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC), \ | |
23 | .read_count = &__percpu_rwsem_rc_##name, \ | |
24 | .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ | |
25 | .writer = __WAIT_QUEUE_HEAD_INITIALIZER(name.writer), \ | |
26 | } | |
27 | ||
80127a39 PZ |
28 | extern int __percpu_down_read(struct percpu_rw_semaphore *, int); |
29 | extern void __percpu_up_read(struct percpu_rw_semaphore *); | |
30 | ||
259d69b7 | 31 | static inline void percpu_down_read_preempt_disable(struct percpu_rw_semaphore *sem) |
80127a39 PZ |
32 | { |
33 | might_sleep(); | |
34 | ||
35 | rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); | |
36 | ||
37 | preempt_disable(); | |
38 | /* | |
39 | * We are in an RCU-sched read-side critical section, so the writer | |
40 | * cannot both change sem->state from readers_fast and start checking | |
41 | * counters while we are here. So if we see !sem->state, we know that | |
42 | * the writer won't be checking until we're past the preempt_enable() | |
43 | * and that one the synchronize_sched() is done, the writer will see | |
44 | * anything we did within this RCU-sched read-size critical section. | |
45 | */ | |
46 | __this_cpu_inc(*sem->read_count); | |
47 | if (unlikely(!rcu_sync_is_idle(&sem->rss))) | |
48 | __percpu_down_read(sem, false); /* Unconditional memory barrier */ | |
259d69b7 | 49 | barrier(); |
80127a39 | 50 | /* |
259d69b7 | 51 | * The barrier() prevents the compiler from |
80127a39 PZ |
52 | * bleeding the critical section out. |
53 | */ | |
54 | } | |
55 | ||
259d69b7 PZ |
56 | static inline void percpu_down_read(struct percpu_rw_semaphore *sem) |
57 | { | |
58 | percpu_down_read_preempt_disable(sem); | |
59 | preempt_enable(); | |
60 | } | |
61 | ||
80127a39 PZ |
62 | static inline int percpu_down_read_trylock(struct percpu_rw_semaphore *sem) |
63 | { | |
64 | int ret = 1; | |
65 | ||
66 | preempt_disable(); | |
67 | /* | |
68 | * Same as in percpu_down_read(). | |
69 | */ | |
70 | __this_cpu_inc(*sem->read_count); | |
71 | if (unlikely(!rcu_sync_is_idle(&sem->rss))) | |
72 | ret = __percpu_down_read(sem, true); /* Unconditional memory barrier */ | |
73 | preempt_enable(); | |
74 | /* | |
75 | * The barrier() from preempt_enable() prevents the compiler from | |
76 | * bleeding the critical section out. | |
77 | */ | |
78 | ||
79 | if (ret) | |
80 | rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 1, _RET_IP_); | |
81 | ||
82 | return ret; | |
83 | } | |
84 | ||
259d69b7 | 85 | static inline void percpu_up_read_preempt_enable(struct percpu_rw_semaphore *sem) |
80127a39 PZ |
86 | { |
87 | /* | |
259d69b7 | 88 | * The barrier() prevents the compiler from |
80127a39 PZ |
89 | * bleeding the critical section out. |
90 | */ | |
259d69b7 | 91 | barrier(); |
80127a39 PZ |
92 | /* |
93 | * Same as in percpu_down_read(). | |
94 | */ | |
95 | if (likely(rcu_sync_is_idle(&sem->rss))) | |
96 | __this_cpu_dec(*sem->read_count); | |
97 | else | |
98 | __percpu_up_read(sem); /* Unconditional memory barrier */ | |
99 | preempt_enable(); | |
100 | ||
101 | rwsem_release(&sem->rw_sem.dep_map, 1, _RET_IP_); | |
102 | } | |
5c1eabe6 | 103 | |
259d69b7 PZ |
104 | static inline void percpu_up_read(struct percpu_rw_semaphore *sem) |
105 | { | |
106 | preempt_disable(); | |
107 | percpu_up_read_preempt_enable(sem); | |
108 | } | |
109 | ||
a1fd3e24 ON |
110 | extern void percpu_down_write(struct percpu_rw_semaphore *); |
111 | extern void percpu_up_write(struct percpu_rw_semaphore *); | |
62ac665f | 112 | |
8ebe3473 ON |
113 | extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, |
114 | const char *, struct lock_class_key *); | |
80127a39 | 115 | |
a1fd3e24 | 116 | extern void percpu_free_rwsem(struct percpu_rw_semaphore *); |
62ac665f | 117 | |
80127a39 | 118 | #define percpu_init_rwsem(sem) \ |
8ebe3473 ON |
119 | ({ \ |
120 | static struct lock_class_key rwsem_key; \ | |
80127a39 | 121 | __percpu_init_rwsem(sem, #sem, &rwsem_key); \ |
8ebe3473 ON |
122 | }) |
123 | ||
55cc1565 ON |
124 | #define percpu_rwsem_is_held(sem) lockdep_is_held(&(sem)->rw_sem) |
125 | ||
11d9684c PZ |
126 | #define percpu_rwsem_assert_held(sem) \ |
127 | lockdep_assert_held(&(sem)->rw_sem) | |
128 | ||
55cc1565 ON |
129 | static inline void percpu_rwsem_release(struct percpu_rw_semaphore *sem, |
130 | bool read, unsigned long ip) | |
131 | { | |
132 | lock_release(&sem->rw_sem.dep_map, 1, ip); | |
133 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER | |
134 | if (!read) | |
135 | sem->rw_sem.owner = NULL; | |
136 | #endif | |
137 | } | |
138 | ||
139 | static inline void percpu_rwsem_acquire(struct percpu_rw_semaphore *sem, | |
140 | bool read, unsigned long ip) | |
141 | { | |
142 | lock_acquire(&sem->rw_sem.dep_map, 0, 1, read, 1, NULL, ip); | |
143 | } | |
144 | ||
62ac665f | 145 | #endif |