]>
Commit | Line | Data |
---|---|---|
4b393c50 | 1 | /* |
716154c5 BB |
2 | * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. |
3 | * Copyright (C) 2007 The Regents of the University of California. | |
4 | * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). | |
5 | * Written by Brian Behlendorf <behlendorf1@llnl.gov>. | |
715f6251 | 6 | * UCRL-CODE-235197 |
7 | * | |
716154c5 | 8 | * This file is part of the SPL, Solaris Porting Layer. |
3d6af2dd | 9 | * For details, see <http://zfsonlinux.org/>. |
716154c5 BB |
10 | * |
11 | * The SPL is free software; you can redistribute it and/or modify it | |
12 | * under the terms of the GNU General Public License as published by the | |
13 | * Free Software Foundation; either version 2 of the License, or (at your | |
14 | * option) any later version. | |
715f6251 | 15 | * |
716154c5 | 16 | * The SPL is distributed in the hope that it will be useful, but WITHOUT |
715f6251 | 17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
19 | * for more details. | |
20 | * | |
21 | * You should have received a copy of the GNU General Public License along | |
716154c5 | 22 | * with the SPL. If not, see <http://www.gnu.org/licenses/>. |
4b393c50 | 23 | */ |
715f6251 | 24 | |
09b414e8 | 25 | #ifndef _SPL_RWLOCK_H |
5461eefe | 26 | #define _SPL_RWLOCK_H |
f1ca4da6 | 27 | |
f4b37741 | 28 | #include <sys/types.h> |
d28db80f | 29 | #include <linux/rwsem.h> |
5712fade BB |
30 | |
31 | /* Linux kernel compatibility */ | |
32 | #if defined(CONFIG_PREEMPT_RT_FULL) | |
33 | #define SPL_RWSEM_SINGLE_READER_VALUE (1) | |
34 | #define SPL_RWSEM_SINGLE_WRITER_VALUE (0) | |
35 | #elif defined(CONFIG_RWSEM_GENERIC_SPINLOCK) | |
36 | #define SPL_RWSEM_SINGLE_READER_VALUE (1) | |
37 | #define SPL_RWSEM_SINGLE_WRITER_VALUE (-1) | |
38 | #else | |
39 | #define SPL_RWSEM_SINGLE_READER_VALUE (RWSEM_ACTIVE_READ_BIAS) | |
40 | #define SPL_RWSEM_SINGLE_WRITER_VALUE (RWSEM_ACTIVE_WRITE_BIAS) | |
41 | #endif | |
42 | ||
43 | /* Linux 3.16 changed activity to count for rwsem-spinlock */ | |
44 | #if defined(CONFIG_PREEMPT_RT_FULL) | |
45 | #define RWSEM_COUNT(sem) sem->read_depth | |
46 | #elif defined(HAVE_RWSEM_ACTIVITY) | |
47 | #define RWSEM_COUNT(sem) sem->activity | |
48 | /* Linux 4.8 changed count to an atomic_long_t for !rwsem-spinlock */ | |
49 | #elif defined(HAVE_RWSEM_ATOMIC_LONG_COUNT) | |
50 | #define RWSEM_COUNT(sem) atomic_long_read(&(sem)->count) | |
51 | #else | |
52 | #define RWSEM_COUNT(sem) sem->count | |
53 | #endif | |
54 | ||
55 | #if defined(RWSEM_SPINLOCK_IS_RAW) | |
56 | #define spl_rwsem_lock_irqsave(lk, fl) raw_spin_lock_irqsave(lk, fl) | |
57 | #define spl_rwsem_unlock_irqrestore(lk, fl) \ | |
58 | raw_spin_unlock_irqrestore(lk, fl) | |
59 | #define spl_rwsem_trylock_irqsave(lk, fl) raw_spin_trylock_irqsave(lk, fl) | |
60 | #else | |
61 | #define spl_rwsem_lock_irqsave(lk, fl) spin_lock_irqsave(lk, fl) | |
62 | #define spl_rwsem_unlock_irqrestore(lk, fl) spin_unlock_irqrestore(lk, fl) | |
63 | #define spl_rwsem_trylock_irqsave(lk, fl) spin_trylock_irqsave(lk, fl) | |
64 | #endif /* RWSEM_SPINLOCK_IS_RAW */ | |
65 | ||
66 | #define spl_rwsem_is_locked(rwsem) rwsem_is_locked(rwsem) | |
f1ca4da6 | 67 | |
68 | typedef enum { | |
374303a3 | 69 | RW_DRIVER = 2, |
692ae8d3 OF |
70 | RW_DEFAULT = 4, |
71 | RW_NOLOCKDEP = 5 | |
f1ca4da6 | 72 | } krw_type_t; |
73 | ||
74 | typedef enum { | |
374303a3 CC |
75 | RW_NONE = 0, |
76 | RW_WRITER = 1, | |
77 | RW_READER = 2 | |
f1ca4da6 | 78 | } krw_t; |
79 | ||
a00b3eb5 CC |
80 | /* |
81 | * If CONFIG_RWSEM_SPIN_ON_OWNER is defined, rw_semaphore will have an owner | |
82 | * field, so we don't need our own. | |
83 | */ | |
d28db80f | 84 | typedef struct { |
374303a3 | 85 | struct rw_semaphore rw_rwlock; |
a00b3eb5 | 86 | #ifndef CONFIG_RWSEM_SPIN_ON_OWNER |
374303a3 | 87 | kthread_t *rw_owner; |
a00b3eb5 | 88 | #endif |
692ae8d3 OF |
89 | #ifdef CONFIG_LOCKDEP |
90 | krw_type_t rw_type; | |
91 | #endif /* CONFIG_LOCKDEP */ | |
d28db80f | 92 | } krwlock_t; |
f1ca4da6 | 93 | |
5461eefe | 94 | #define SEM(rwp) (&(rwp)->rw_rwlock) |
d28db80f | 95 | |
d28db80f BB |
96 | static inline void |
97 | spl_rw_set_owner(krwlock_t *rwp) | |
98 | { | |
a00b3eb5 CC |
99 | /* |
100 | * If CONFIG_RWSEM_SPIN_ON_OWNER is defined, down_write, up_write, | |
101 | * downgrade_write and __init_rwsem will set/clear owner for us. | |
102 | */ | |
103 | #ifndef CONFIG_RWSEM_SPIN_ON_OWNER | |
4f8e643a | 104 | rwp->rw_owner = current; |
a00b3eb5 | 105 | #endif |
d28db80f BB |
106 | } |
107 | ||
108 | static inline void | |
109 | spl_rw_clear_owner(krwlock_t *rwp) | |
110 | { | |
a00b3eb5 | 111 | #ifndef CONFIG_RWSEM_SPIN_ON_OWNER |
4f8e643a | 112 | rwp->rw_owner = NULL; |
a00b3eb5 | 113 | #endif |
d28db80f BB |
114 | } |
115 | ||
116 | static inline kthread_t * | |
117 | rw_owner(krwlock_t *rwp) | |
118 | { | |
a00b3eb5 | 119 | #ifdef CONFIG_RWSEM_SPIN_ON_OWNER |
5461eefe | 120 | return (SEM(rwp)->owner); |
a00b3eb5 | 121 | #else |
5461eefe | 122 | return (rwp->rw_owner); |
a00b3eb5 | 123 | #endif |
d28db80f BB |
124 | } |
125 | ||
692ae8d3 OF |
126 | #ifdef CONFIG_LOCKDEP |
127 | static inline void | |
128 | spl_rw_set_type(krwlock_t *rwp, krw_type_t type) | |
129 | { | |
130 | rwp->rw_type = type; | |
131 | } | |
132 | static inline void | |
133 | spl_rw_lockdep_off_maybe(krwlock_t *rwp) \ | |
134 | { \ | |
135 | if (rwp && rwp->rw_type == RW_NOLOCKDEP) \ | |
136 | lockdep_off(); \ | |
137 | } | |
138 | static inline void | |
139 | spl_rw_lockdep_on_maybe(krwlock_t *rwp) \ | |
140 | { \ | |
141 | if (rwp && rwp->rw_type == RW_NOLOCKDEP) \ | |
142 | lockdep_on(); \ | |
143 | } | |
144 | #else /* CONFIG_LOCKDEP */ | |
5461eefe BB |
145 | #define spl_rw_set_type(rwp, type) |
146 | #define spl_rw_lockdep_off_maybe(rwp) | |
147 | #define spl_rw_lockdep_on_maybe(rwp) | |
692ae8d3 OF |
148 | #endif /* CONFIG_LOCKDEP */ |
149 | ||
d28db80f BB |
150 | static inline int |
151 | RW_READ_HELD(krwlock_t *rwp) | |
152 | { | |
2529b3a8 | 153 | /* |
154 | * Linux 4.8 will set owner to 1 when read held instead of leave it | |
155 | * NULL. So we check whether owner <= 1. | |
156 | */ | |
157 | return (spl_rwsem_is_locked(SEM(rwp)) && | |
158 | (unsigned long)rw_owner(rwp) <= 1); | |
d28db80f BB |
159 | } |
160 | ||
161 | static inline int | |
162 | RW_WRITE_HELD(krwlock_t *rwp) | |
163 | { | |
4f8e643a | 164 | return (rw_owner(rwp) == current); |
d28db80f BB |
165 | } |
166 | ||
167 | static inline int | |
168 | RW_LOCK_HELD(krwlock_t *rwp) | |
169 | { | |
5461eefe | 170 | return (spl_rwsem_is_locked(SEM(rwp))); |
d28db80f BB |
171 | } |
172 | ||
173 | /* | |
5461eefe | 174 | * The following functions must be a #define and not static inline. |
d28db80f BB |
175 | * This ensures that the native linux semaphore functions (down/up) |
176 | * will be correctly located in the users code which is important | |
177 | * for the built in kernel lock analysis tools | |
178 | */ | |
5461eefe BB |
179 | /* BEGIN CSTYLED */ |
180 | #define rw_init(rwp, name, type, arg) \ | |
374303a3 CC |
181 | ({ \ |
182 | static struct lock_class_key __key; \ | |
692ae8d3 | 183 | ASSERT(type == RW_DEFAULT || type == RW_NOLOCKDEP); \ |
374303a3 CC |
184 | \ |
185 | __init_rwsem(SEM(rwp), #rwp, &__key); \ | |
186 | spl_rw_clear_owner(rwp); \ | |
692ae8d3 | 187 | spl_rw_set_type(rwp, type); \ |
e811949a | 188 | }) |
f1ca4da6 | 189 | |
5461eefe | 190 | #define rw_destroy(rwp) \ |
374303a3 CC |
191 | ({ \ |
192 | VERIFY(!RW_LOCK_HELD(rwp)); \ | |
e811949a | 193 | }) |
ed61a7d0 | 194 | |
5461eefe | 195 | #define rw_tryenter(rwp, rw) \ |
374303a3 CC |
196 | ({ \ |
197 | int _rc_ = 0; \ | |
198 | \ | |
692ae8d3 | 199 | spl_rw_lockdep_off_maybe(rwp); \ |
374303a3 CC |
200 | switch (rw) { \ |
201 | case RW_READER: \ | |
202 | _rc_ = down_read_trylock(SEM(rwp)); \ | |
203 | break; \ | |
204 | case RW_WRITER: \ | |
205 | if ((_rc_ = down_write_trylock(SEM(rwp)))) \ | |
206 | spl_rw_set_owner(rwp); \ | |
207 | break; \ | |
208 | default: \ | |
209 | VERIFY(0); \ | |
210 | } \ | |
692ae8d3 | 211 | spl_rw_lockdep_on_maybe(rwp); \ |
374303a3 | 212 | _rc_; \ |
e8b31e84 | 213 | }) |
e8b31e84 | 214 | |
5461eefe | 215 | #define rw_enter(rwp, rw) \ |
374303a3 | 216 | ({ \ |
692ae8d3 | 217 | spl_rw_lockdep_off_maybe(rwp); \ |
374303a3 CC |
218 | switch (rw) { \ |
219 | case RW_READER: \ | |
220 | down_read(SEM(rwp)); \ | |
221 | break; \ | |
222 | case RW_WRITER: \ | |
223 | down_write(SEM(rwp)); \ | |
224 | spl_rw_set_owner(rwp); \ | |
225 | break; \ | |
226 | default: \ | |
227 | VERIFY(0); \ | |
228 | } \ | |
692ae8d3 | 229 | spl_rw_lockdep_on_maybe(rwp); \ |
e811949a | 230 | }) |
f1ca4da6 | 231 | |
5461eefe | 232 | #define rw_exit(rwp) \ |
374303a3 | 233 | ({ \ |
692ae8d3 | 234 | spl_rw_lockdep_off_maybe(rwp); \ |
374303a3 CC |
235 | if (RW_WRITE_HELD(rwp)) { \ |
236 | spl_rw_clear_owner(rwp); \ | |
237 | up_write(SEM(rwp)); \ | |
238 | } else { \ | |
239 | ASSERT(RW_READ_HELD(rwp)); \ | |
240 | up_read(SEM(rwp)); \ | |
241 | } \ | |
692ae8d3 | 242 | spl_rw_lockdep_on_maybe(rwp); \ |
d28db80f BB |
243 | }) |
244 | ||
5461eefe | 245 | #define rw_downgrade(rwp) \ |
374303a3 | 246 | ({ \ |
692ae8d3 | 247 | spl_rw_lockdep_off_maybe(rwp); \ |
374303a3 CC |
248 | spl_rw_clear_owner(rwp); \ |
249 | downgrade_write(SEM(rwp)); \ | |
692ae8d3 | 250 | spl_rw_lockdep_on_maybe(rwp); \ |
d28db80f BB |
251 | }) |
252 | ||
5461eefe | 253 | #define rw_tryupgrade(rwp) \ |
374303a3 | 254 | ({ \ |
374303a3 CC |
255 | int _rc_ = 0; \ |
256 | \ | |
a6ae97ca BB |
257 | if (RW_WRITE_HELD(rwp)) { \ |
258 | _rc_ = 1; \ | |
259 | } else { \ | |
f58040c0 CC |
260 | spl_rw_lockdep_off_maybe(rwp); \ |
261 | if ((_rc_ = rwsem_tryupgrade(SEM(rwp)))) \ | |
262 | spl_rw_set_owner(rwp); \ | |
263 | spl_rw_lockdep_on_maybe(rwp); \ | |
374303a3 | 264 | } \ |
374303a3 | 265 | _rc_; \ |
d28db80f | 266 | }) |
5461eefe | 267 | /* END CSTYLED */ |
d28db80f BB |
268 | |
269 | int spl_rw_init(void); | |
270 | void spl_rw_fini(void); | |
5712fade | 271 | int rwsem_tryupgrade(struct rw_semaphore *rwsem); |
d28db80f | 272 | |
e8b31e84 | 273 | #endif /* _SPL_RWLOCK_H */ |