]>
Commit | Line | Data |
---|---|---|
716154c5 BB |
1 | /*****************************************************************************\ |
2 | * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. | |
3 | * Copyright (C) 2007 The Regents of the University of California. | |
4 | * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). | |
5 | * Written by Brian Behlendorf <behlendorf1@llnl.gov>. | |
715f6251 | 6 | * UCRL-CODE-235197 |
7 | * | |
716154c5 | 8 | * This file is part of the SPL, Solaris Porting Layer. |
3d6af2dd | 9 | * For details, see <http://zfsonlinux.org/>. |
716154c5 BB |
10 | * |
11 | * The SPL is free software; you can redistribute it and/or modify it | |
12 | * under the terms of the GNU General Public License as published by the | |
13 | * Free Software Foundation; either version 2 of the License, or (at your | |
14 | * option) any later version. | |
715f6251 | 15 | * |
716154c5 | 16 | * The SPL is distributed in the hope that it will be useful, but WITHOUT |
715f6251 | 17 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
18 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
19 | * for more details. | |
20 | * | |
21 | * You should have received a copy of the GNU General Public License along | |
716154c5 BB |
22 | * with the SPL. If not, see <http://www.gnu.org/licenses/>. |
23 | ***************************************************************************** | |
24 | * Solaris Porting Layer (SPL) Credential Implementation. | |
25 | \*****************************************************************************/ | |
715f6251 | 26 | |
4efd4118 | 27 | #include <sys/condvar.h> |
10946b02 | 28 | #include <sys/time.h> |
4efd4118 | 29 | |
30 | void | |
31 | __cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg) | |
32 | { | |
4efd4118 | 33 | ASSERT(cvp); |
b29012b9 | 34 | ASSERT(name == NULL); |
4efd4118 | 35 | ASSERT(type == CV_DEFAULT); |
36 | ASSERT(arg == NULL); | |
37 | ||
38 | cvp->cv_magic = CV_MAGIC; | |
39 | init_waitqueue_head(&cvp->cv_event); | |
d599e4fa | 40 | init_waitqueue_head(&cvp->cv_destroy); |
4efd4118 | 41 | atomic_set(&cvp->cv_waiters, 0); |
d2733258 | 42 | atomic_set(&cvp->cv_refs, 1); |
4efd4118 | 43 | cvp->cv_mutex = NULL; |
4efd4118 | 44 | } |
45 | EXPORT_SYMBOL(__cv_init); | |
46 | ||
d599e4fa BB |
47 | static int |
48 | cv_destroy_wakeup(kcondvar_t *cvp) | |
49 | { | |
d2733258 BB |
50 | if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) { |
51 | ASSERT(cvp->cv_mutex == NULL); | |
52 | ASSERT(!waitqueue_active(&cvp->cv_event)); | |
53 | return 1; | |
54 | } | |
d599e4fa | 55 | |
d2733258 | 56 | return 0; |
d599e4fa BB |
57 | } |
58 | ||
4efd4118 | 59 | void |
60 | __cv_destroy(kcondvar_t *cvp) | |
61 | { | |
4efd4118 | 62 | ASSERT(cvp); |
63 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
d599e4fa | 64 | |
d2733258 BB |
65 | cvp->cv_magic = CV_DESTROY; |
66 | atomic_dec(&cvp->cv_refs); | |
67 | ||
68 | /* Block until all waiters are woken and references dropped. */ | |
d599e4fa BB |
69 | while (cv_destroy_wakeup(cvp) == 0) |
70 | wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1); | |
71 | ||
3c60f505 | 72 | ASSERT3P(cvp->cv_mutex, ==, NULL); |
d2733258 | 73 | ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0); |
3c60f505 BB |
74 | ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0); |
75 | ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0); | |
4efd4118 | 76 | } |
77 | EXPORT_SYMBOL(__cv_destroy); | |
78 | ||
f752b46e | 79 | static void |
46a75aad | 80 | cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io) |
4efd4118 | 81 | { |
82 | DEFINE_WAIT(wait); | |
4efd4118 | 83 | |
84 | ASSERT(cvp); | |
85 | ASSERT(mp); | |
86 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
4efd4118 | 87 | ASSERT(mutex_owned(mp)); |
d2733258 | 88 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 89 | |
90 | if (cvp->cv_mutex == NULL) | |
91 | cvp->cv_mutex = mp; | |
92 | ||
93 | /* Ensure the same mutex is used by all callers */ | |
94 | ASSERT(cvp->cv_mutex == mp); | |
4efd4118 | 95 | |
f752b46e | 96 | prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
4efd4118 | 97 | atomic_inc(&cvp->cv_waiters); |
98 | ||
99 | /* Mutex should be dropped after prepare_to_wait() this | |
100 | * ensures we're linked in to the waiters list and avoids the | |
101 | * race where 'cvp->cv_waiters > 0' but the list is empty. */ | |
102 | mutex_exit(mp); | |
46a75aad MJ |
103 | if (io) |
104 | io_schedule(); | |
105 | else | |
106 | schedule(); | |
4efd4118 | 107 | mutex_enter(mp); |
108 | ||
058de03c | 109 | /* No more waiters a different mutex could be used */ |
d599e4fa | 110 | if (atomic_dec_and_test(&cvp->cv_waiters)) { |
058de03c | 111 | cvp->cv_mutex = NULL; |
d599e4fa BB |
112 | wake_up(&cvp->cv_destroy); |
113 | } | |
058de03c | 114 | |
4efd4118 | 115 | finish_wait(&cvp->cv_event, &wait); |
d2733258 | 116 | atomic_dec(&cvp->cv_refs); |
4efd4118 | 117 | } |
f752b46e BB |
118 | |
119 | void | |
120 | __cv_wait(kcondvar_t *cvp, kmutex_t *mp) | |
121 | { | |
46a75aad | 122 | cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0); |
f752b46e | 123 | } |
4efd4118 | 124 | EXPORT_SYMBOL(__cv_wait); |
125 | ||
f752b46e BB |
126 | void |
127 | __cv_wait_interruptible(kcondvar_t *cvp, kmutex_t *mp) | |
128 | { | |
46a75aad | 129 | cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0); |
f752b46e BB |
130 | } |
131 | EXPORT_SYMBOL(__cv_wait_interruptible); | |
132 | ||
46a75aad MJ |
133 | void |
134 | __cv_wait_io(kcondvar_t *cvp, kmutex_t *mp) | |
135 | { | |
136 | cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1); | |
137 | } | |
138 | EXPORT_SYMBOL(__cv_wait_io); | |
139 | ||
4efd4118 | 140 | /* 'expire_time' argument is an absolute wall clock time in jiffies. |
141 | * Return value is time left (expire_time - now) or -1 if timeout occurred. | |
142 | */ | |
3f688a8c NK |
143 | static clock_t |
144 | __cv_timedwait_common(kcondvar_t *cvp, kmutex_t *mp, | |
145 | clock_t expire_time, int state) | |
4efd4118 | 146 | { |
147 | DEFINE_WAIT(wait); | |
148 | clock_t time_left; | |
4efd4118 | 149 | |
150 | ASSERT(cvp); | |
151 | ASSERT(mp); | |
152 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
4efd4118 | 153 | ASSERT(mutex_owned(mp)); |
d2733258 | 154 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 155 | |
156 | if (cvp->cv_mutex == NULL) | |
157 | cvp->cv_mutex = mp; | |
158 | ||
159 | /* Ensure the same mutex is used by all callers */ | |
160 | ASSERT(cvp->cv_mutex == mp); | |
4efd4118 | 161 | |
162 | /* XXX - Does not handle jiffie wrap properly */ | |
163 | time_left = expire_time - jiffies; | |
d2733258 BB |
164 | if (time_left <= 0) { |
165 | atomic_dec(&cvp->cv_refs); | |
10946b02 | 166 | return (-1); |
d2733258 | 167 | } |
4efd4118 | 168 | |
3f688a8c | 169 | prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
4efd4118 | 170 | atomic_inc(&cvp->cv_waiters); |
171 | ||
172 | /* Mutex should be dropped after prepare_to_wait() this | |
173 | * ensures we're linked in to the waiters list and avoids the | |
174 | * race where 'cvp->cv_waiters > 0' but the list is empty. */ | |
175 | mutex_exit(mp); | |
176 | time_left = schedule_timeout(time_left); | |
177 | mutex_enter(mp); | |
178 | ||
058de03c | 179 | /* No more waiters a different mutex could be used */ |
d599e4fa | 180 | if (atomic_dec_and_test(&cvp->cv_waiters)) { |
058de03c | 181 | cvp->cv_mutex = NULL; |
d599e4fa BB |
182 | wake_up(&cvp->cv_destroy); |
183 | } | |
058de03c | 184 | |
4efd4118 | 185 | finish_wait(&cvp->cv_event, &wait); |
d2733258 | 186 | atomic_dec(&cvp->cv_refs); |
4efd4118 | 187 | |
10946b02 | 188 | return (time_left > 0 ? time_left : -1); |
4efd4118 | 189 | } |
3f688a8c NK |
190 | |
191 | clock_t | |
192 | __cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) | |
193 | { | |
194 | return __cv_timedwait_common(cvp, mp, exp_time, TASK_UNINTERRUPTIBLE); | |
195 | } | |
4efd4118 | 196 | EXPORT_SYMBOL(__cv_timedwait); |
197 | ||
3f688a8c NK |
198 | clock_t |
199 | __cv_timedwait_interruptible(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) | |
200 | { | |
201 | return __cv_timedwait_common(cvp, mp, exp_time, TASK_INTERRUPTIBLE); | |
202 | } | |
203 | EXPORT_SYMBOL(__cv_timedwait_interruptible); | |
204 | ||
33a20369 LG |
205 | /* |
206 | *'expire_time' argument is an absolute clock time in nanoseconds. | |
207 | * Return value is time left (expire_time - now) or -1 if timeout occurred. | |
208 | */ | |
209 | static clock_t | |
210 | __cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, | |
211 | hrtime_t expire_time, int state) | |
212 | { | |
213 | DEFINE_WAIT(wait); | |
214 | hrtime_t time_left, now; | |
215 | unsigned long time_left_us; | |
33a20369 LG |
216 | |
217 | ASSERT(cvp); | |
218 | ASSERT(mp); | |
219 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
220 | ASSERT(mutex_owned(mp)); | |
221 | atomic_inc(&cvp->cv_refs); | |
222 | ||
223 | if (cvp->cv_mutex == NULL) | |
224 | cvp->cv_mutex = mp; | |
225 | ||
226 | /* Ensure the same mutex is used by all callers */ | |
227 | ASSERT(cvp->cv_mutex == mp); | |
228 | ||
229 | now = gethrtime(); | |
230 | time_left = expire_time - now; | |
231 | if (time_left <= 0) { | |
232 | atomic_dec(&cvp->cv_refs); | |
10946b02 | 233 | return (-1); |
33a20369 LG |
234 | } |
235 | time_left_us = time_left / NSEC_PER_USEC; | |
236 | ||
237 | prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); | |
238 | atomic_inc(&cvp->cv_waiters); | |
239 | ||
240 | /* Mutex should be dropped after prepare_to_wait() this | |
241 | * ensures we're linked in to the waiters list and avoids the | |
242 | * race where 'cvp->cv_waiters > 0' but the list is empty. */ | |
243 | mutex_exit(mp); | |
244 | /* Allow a 100 us range to give kernel an opportunity to coalesce | |
245 | * interrupts */ | |
246 | usleep_range(time_left_us, time_left_us + 100); | |
247 | mutex_enter(mp); | |
248 | ||
249 | /* No more waiters a different mutex could be used */ | |
250 | if (atomic_dec_and_test(&cvp->cv_waiters)) { | |
251 | cvp->cv_mutex = NULL; | |
252 | wake_up(&cvp->cv_destroy); | |
253 | } | |
254 | ||
255 | finish_wait(&cvp->cv_event, &wait); | |
256 | atomic_dec(&cvp->cv_refs); | |
257 | ||
258 | time_left = expire_time - gethrtime(); | |
10946b02 | 259 | return (time_left > 0 ? time_left : -1); |
33a20369 LG |
260 | } |
261 | ||
262 | /* | |
263 | * Compatibility wrapper for the cv_timedwait_hires() Illumos interface. | |
264 | */ | |
265 | clock_t | |
266 | cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, | |
267 | hrtime_t res, int flag) | |
268 | { | |
269 | if (res > 1) { | |
270 | /* | |
271 | * Align expiration to the specified resolution. | |
272 | */ | |
273 | if (flag & CALLOUT_FLAG_ROUNDUP) | |
274 | tim += res - 1; | |
275 | tim = (tim / res) * res; | |
276 | } | |
277 | ||
278 | if (!(flag & CALLOUT_FLAG_ABSOLUTE)) | |
279 | tim += gethrtime(); | |
280 | ||
281 | return __cv_timedwait_hires(cvp, mp, tim, TASK_UNINTERRUPTIBLE); | |
282 | } | |
283 | EXPORT_SYMBOL(cv_timedwait_hires); | |
284 | ||
4efd4118 | 285 | void |
286 | __cv_signal(kcondvar_t *cvp) | |
287 | { | |
4efd4118 | 288 | ASSERT(cvp); |
289 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
d2733258 | 290 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 291 | |
292 | /* All waiters are added with WQ_FLAG_EXCLUSIVE so only one | |
293 | * waiter will be set runable with each call to wake_up(). | |
294 | * Additionally wake_up() holds a spin_lock assoicated with | |
295 | * the wait queue to ensure we don't race waking up processes. */ | |
296 | if (atomic_read(&cvp->cv_waiters) > 0) | |
297 | wake_up(&cvp->cv_event); | |
298 | ||
d2733258 | 299 | atomic_dec(&cvp->cv_refs); |
4efd4118 | 300 | } |
301 | EXPORT_SYMBOL(__cv_signal); | |
302 | ||
303 | void | |
304 | __cv_broadcast(kcondvar_t *cvp) | |
305 | { | |
306 | ASSERT(cvp); | |
307 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
d2733258 | 308 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 309 | |
310 | /* Wake_up_all() will wake up all waiters even those which | |
311 | * have the WQ_FLAG_EXCLUSIVE flag set. */ | |
312 | if (atomic_read(&cvp->cv_waiters) > 0) | |
313 | wake_up_all(&cvp->cv_event); | |
314 | ||
d2733258 | 315 | atomic_dec(&cvp->cv_refs); |
4efd4118 | 316 | } |
317 | EXPORT_SYMBOL(__cv_broadcast); |