]>
Commit | Line | Data |
---|---|---|
23453686 | 1 | /* |
716154c5 BB |
2 | * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. |
3 | * Copyright (C) 2007 The Regents of the University of California. | |
4 | * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). | |
5 | * Written by Brian Behlendorf <behlendorf1@llnl.gov>. | |
715f6251 BB |
6 | * UCRL-CODE-235197 |
7 | * | |
716154c5 | 8 | * This file is part of the SPL, Solaris Porting Layer. |
716154c5 BB |
9 | * |
10 | * The SPL is free software; you can redistribute it and/or modify it | |
11 | * under the terms of the GNU General Public License as published by the | |
12 | * Free Software Foundation; either version 2 of the License, or (at your | |
13 | * option) any later version. | |
715f6251 | 14 | * |
716154c5 | 15 | * The SPL is distributed in the hope that it will be useful, but WITHOUT |
715f6251 BB |
16 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
17 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
18 | * for more details. | |
19 | * | |
20 | * You should have received a copy of the GNU General Public License along | |
716154c5 | 21 | * with the SPL. If not, see <http://www.gnu.org/licenses/>. |
23453686 | 22 | * |
e3d3d772 | 23 | * Solaris Porting Layer (SPL) Condition Variables Implementation. |
23453686 | 24 | */ |
715f6251 | 25 | |
4efd4118 | 26 | #include <sys/condvar.h> |
e5b9b344 | 27 | #include <sys/time.h> |
8d042842 | 28 | #include <sys/sysmacros.h> |
39cd90ef | 29 | #include <linux/hrtimer.h> |
93ce2b4c | 30 | #include <linux/compiler_compat.h> |
8d042842 | 31 | #include <linux/mod_compat.h> |
4efd4118 | 32 | |
186898bb DB |
33 | #include <linux/sched.h> |
34 | ||
35 | #ifdef HAVE_SCHED_SIGNAL_HEADER | |
36 | #include <linux/sched/signal.h> | |
37 | #endif | |
38 | ||
8d042842 | 39 | #define MAX_HRTIMEOUT_SLACK_US 1000 |
27218a32 | 40 | static unsigned int spl_schedule_hrtimeout_slack_us = 0; |
8d042842 TN |
41 | |
42 | static int | |
43 | param_set_hrtimeout_slack(const char *buf, zfs_kernel_param_t *kp) | |
44 | { | |
45 | unsigned long val; | |
46 | int error; | |
47 | ||
48 | error = kstrtoul(buf, 0, &val); | |
49 | if (error) | |
50 | return (error); | |
51 | ||
52 | if (val > MAX_HRTIMEOUT_SLACK_US) | |
53 | return (-EINVAL); | |
54 | ||
55 | error = param_set_uint(buf, kp); | |
56 | if (error < 0) | |
57 | return (error); | |
58 | ||
59 | return (0); | |
60 | } | |
61 | ||
62 | module_param_call(spl_schedule_hrtimeout_slack_us, param_set_hrtimeout_slack, | |
63 | param_get_uint, &spl_schedule_hrtimeout_slack_us, 0644); | |
64 | MODULE_PARM_DESC(spl_schedule_hrtimeout_slack_us, | |
65 | "schedule_hrtimeout_range() delta/slack value in us, default(0)"); | |
66 | ||
4efd4118 BB |
67 | void |
68 | __cv_init(kcondvar_t *cvp, char *name, kcv_type_t type, void *arg) | |
69 | { | |
4efd4118 | 70 | ASSERT(cvp); |
b29012b9 | 71 | ASSERT(name == NULL); |
4efd4118 BB |
72 | ASSERT(type == CV_DEFAULT); |
73 | ASSERT(arg == NULL); | |
74 | ||
75 | cvp->cv_magic = CV_MAGIC; | |
76 | init_waitqueue_head(&cvp->cv_event); | |
d599e4fa | 77 | init_waitqueue_head(&cvp->cv_destroy); |
4efd4118 | 78 | atomic_set(&cvp->cv_waiters, 0); |
d2733258 | 79 | atomic_set(&cvp->cv_refs, 1); |
4efd4118 | 80 | cvp->cv_mutex = NULL; |
4efd4118 BB |
81 | } |
82 | EXPORT_SYMBOL(__cv_init); | |
83 | ||
d599e4fa BB |
84 | static int |
85 | cv_destroy_wakeup(kcondvar_t *cvp) | |
86 | { | |
d2733258 BB |
87 | if (!atomic_read(&cvp->cv_waiters) && !atomic_read(&cvp->cv_refs)) { |
88 | ASSERT(cvp->cv_mutex == NULL); | |
89 | ASSERT(!waitqueue_active(&cvp->cv_event)); | |
23453686 | 90 | return (1); |
d2733258 | 91 | } |
d599e4fa | 92 | |
23453686 | 93 | return (0); |
d599e4fa BB |
94 | } |
95 | ||
4efd4118 BB |
96 | void |
97 | __cv_destroy(kcondvar_t *cvp) | |
98 | { | |
4efd4118 BB |
99 | ASSERT(cvp); |
100 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
d599e4fa | 101 | |
d2733258 BB |
102 | cvp->cv_magic = CV_DESTROY; |
103 | atomic_dec(&cvp->cv_refs); | |
104 | ||
105 | /* Block until all waiters are woken and references dropped. */ | |
d599e4fa BB |
106 | while (cv_destroy_wakeup(cvp) == 0) |
107 | wait_event_timeout(cvp->cv_destroy, cv_destroy_wakeup(cvp), 1); | |
108 | ||
3c60f505 | 109 | ASSERT3P(cvp->cv_mutex, ==, NULL); |
d2733258 | 110 | ASSERT3S(atomic_read(&cvp->cv_refs), ==, 0); |
3c60f505 BB |
111 | ASSERT3S(atomic_read(&cvp->cv_waiters), ==, 0); |
112 | ASSERT3S(waitqueue_active(&cvp->cv_event), ==, 0); | |
4efd4118 BB |
113 | } |
114 | EXPORT_SYMBOL(__cv_destroy); | |
115 | ||
f752b46e | 116 | static void |
46a75aad | 117 | cv_wait_common(kcondvar_t *cvp, kmutex_t *mp, int state, int io) |
4efd4118 BB |
118 | { |
119 | DEFINE_WAIT(wait); | |
e843553d | 120 | kmutex_t *m; |
4efd4118 BB |
121 | |
122 | ASSERT(cvp); | |
23453686 | 123 | ASSERT(mp); |
4efd4118 | 124 | ASSERT(cvp->cv_magic == CV_MAGIC); |
4efd4118 | 125 | ASSERT(mutex_owned(mp)); |
d2733258 | 126 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 127 | |
93ce2b4c | 128 | m = READ_ONCE(cvp->cv_mutex); |
e843553d CC |
129 | if (!m) |
130 | m = xchg(&cvp->cv_mutex, mp); | |
4efd4118 | 131 | /* Ensure the same mutex is used by all callers */ |
e843553d | 132 | ASSERT(m == NULL || m == mp); |
4efd4118 | 133 | |
f752b46e | 134 | prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
4efd4118 BB |
135 | atomic_inc(&cvp->cv_waiters); |
136 | ||
23453686 BB |
137 | /* |
138 | * Mutex should be dropped after prepare_to_wait() this | |
4efd4118 | 139 | * ensures we're linked in to the waiters list and avoids the |
23453686 BB |
140 | * race where 'cvp->cv_waiters > 0' but the list is empty. |
141 | */ | |
4efd4118 | 142 | mutex_exit(mp); |
46a75aad MJ |
143 | if (io) |
144 | io_schedule(); | |
145 | else | |
146 | schedule(); | |
4efd4118 | 147 | |
058de03c | 148 | /* No more waiters a different mutex could be used */ |
d599e4fa | 149 | if (atomic_dec_and_test(&cvp->cv_waiters)) { |
e843553d CC |
150 | /* |
151 | * This is set without any lock, so it's racy. But this is | |
152 | * just for debug anyway, so make it best-effort | |
153 | */ | |
058de03c | 154 | cvp->cv_mutex = NULL; |
d599e4fa BB |
155 | wake_up(&cvp->cv_destroy); |
156 | } | |
058de03c | 157 | |
4efd4118 | 158 | finish_wait(&cvp->cv_event, &wait); |
d2733258 | 159 | atomic_dec(&cvp->cv_refs); |
e843553d CC |
160 | |
161 | /* | |
162 | * Hold mutex after we release the cvp, otherwise we could dead lock | |
163 | * with a thread holding the mutex and call cv_destroy. | |
164 | */ | |
165 | mutex_enter(mp); | |
4efd4118 | 166 | } |
f752b46e BB |
167 | |
168 | void | |
169 | __cv_wait(kcondvar_t *cvp, kmutex_t *mp) | |
170 | { | |
46a75aad | 171 | cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 0); |
f752b46e | 172 | } |
4efd4118 BB |
173 | EXPORT_SYMBOL(__cv_wait); |
174 | ||
23602fdb BB |
175 | void |
176 | __cv_wait_io(kcondvar_t *cvp, kmutex_t *mp) | |
177 | { | |
178 | cv_wait_common(cvp, mp, TASK_UNINTERRUPTIBLE, 1); | |
179 | } | |
180 | EXPORT_SYMBOL(__cv_wait_io); | |
181 | ||
186898bb DB |
182 | int |
183 | __cv_wait_io_sig(kcondvar_t *cvp, kmutex_t *mp) | |
184 | { | |
185 | cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 1); | |
186 | ||
187 | return (signal_pending(current) ? 0 : 1); | |
188 | } | |
189 | EXPORT_SYMBOL(__cv_wait_io_sig); | |
190 | ||
191 | int | |
23453686 | 192 | __cv_wait_sig(kcondvar_t *cvp, kmutex_t *mp) |
f752b46e | 193 | { |
46a75aad | 194 | cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0); |
186898bb DB |
195 | |
196 | return (signal_pending(current) ? 0 : 1); | |
f752b46e | 197 | } |
23453686 | 198 | EXPORT_SYMBOL(__cv_wait_sig); |
f752b46e | 199 | |
ac6e5fb2 MM |
200 | void |
201 | __cv_wait_idle(kcondvar_t *cvp, kmutex_t *mp) | |
202 | { | |
203 | sigset_t blocked, saved; | |
204 | ||
205 | sigfillset(&blocked); | |
206 | (void) sigprocmask(SIG_BLOCK, &blocked, &saved); | |
207 | cv_wait_common(cvp, mp, TASK_INTERRUPTIBLE, 0); | |
208 | (void) sigprocmask(SIG_SETMASK, &saved, NULL); | |
209 | } | |
210 | EXPORT_SYMBOL(__cv_wait_idle); | |
211 | ||
23602fdb BB |
212 | #if defined(HAVE_IO_SCHEDULE_TIMEOUT) |
213 | #define spl_io_schedule_timeout(t) io_schedule_timeout(t) | |
214 | #else | |
8b8b44d0 RK |
215 | |
216 | struct spl_task_timer { | |
217 | struct timer_list timer; | |
218 | struct task_struct *task; | |
219 | }; | |
220 | ||
23602fdb | 221 | static void |
8b8b44d0 | 222 | __cv_wakeup(spl_timer_list_t t) |
46a75aad | 223 | { |
8b8b44d0 RK |
224 | struct timer_list *tmr = (struct timer_list *)t; |
225 | struct spl_task_timer *task_timer = from_timer(task_timer, tmr, timer); | |
226 | ||
227 | wake_up_process(task_timer->task); | |
46a75aad | 228 | } |
23602fdb BB |
229 | |
230 | static long | |
231 | spl_io_schedule_timeout(long time_left) | |
232 | { | |
233 | long expire_time = jiffies + time_left; | |
8b8b44d0 RK |
234 | struct spl_task_timer task_timer; |
235 | struct timer_list *timer = &task_timer.timer; | |
236 | ||
237 | task_timer.task = current; | |
23602fdb | 238 | |
8b8b44d0 RK |
239 | timer_setup(timer, __cv_wakeup, 0); |
240 | ||
241 | timer->expires = expire_time; | |
242 | add_timer(timer); | |
23602fdb BB |
243 | |
244 | io_schedule(); | |
245 | ||
8b8b44d0 RK |
246 | del_timer_sync(timer); |
247 | ||
23602fdb BB |
248 | time_left = expire_time - jiffies; |
249 | ||
250 | return (time_left < 0 ? 0 : time_left); | |
251 | } | |
252 | #endif | |
46a75aad | 253 | |
23453686 BB |
254 | /* |
255 | * 'expire_time' argument is an absolute wall clock time in jiffies. | |
4efd4118 BB |
256 | * Return value is time left (expire_time - now) or -1 if timeout occurred. |
257 | */ | |
3f688a8c | 258 | static clock_t |
23453686 | 259 | __cv_timedwait_common(kcondvar_t *cvp, kmutex_t *mp, clock_t expire_time, |
23602fdb | 260 | int state, int io) |
4efd4118 BB |
261 | { |
262 | DEFINE_WAIT(wait); | |
e843553d | 263 | kmutex_t *m; |
4efd4118 | 264 | clock_t time_left; |
4efd4118 BB |
265 | |
266 | ASSERT(cvp); | |
23453686 | 267 | ASSERT(mp); |
4efd4118 | 268 | ASSERT(cvp->cv_magic == CV_MAGIC); |
4efd4118 BB |
269 | ASSERT(mutex_owned(mp)); |
270 | ||
2ded1c7e BB |
271 | /* XXX - Does not handle jiffie wrap properly */ |
272 | time_left = expire_time - jiffies; | |
273 | if (time_left <= 0) | |
274 | return (-1); | |
275 | ||
276 | atomic_inc(&cvp->cv_refs); | |
93ce2b4c | 277 | m = READ_ONCE(cvp->cv_mutex); |
e843553d CC |
278 | if (!m) |
279 | m = xchg(&cvp->cv_mutex, mp); | |
4efd4118 | 280 | /* Ensure the same mutex is used by all callers */ |
e843553d | 281 | ASSERT(m == NULL || m == mp); |
4efd4118 | 282 | |
3f688a8c | 283 | prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
4efd4118 BB |
284 | atomic_inc(&cvp->cv_waiters); |
285 | ||
23453686 BB |
286 | /* |
287 | * Mutex should be dropped after prepare_to_wait() this | |
4efd4118 | 288 | * ensures we're linked in to the waiters list and avoids the |
23453686 BB |
289 | * race where 'cvp->cv_waiters > 0' but the list is empty. |
290 | */ | |
4efd4118 | 291 | mutex_exit(mp); |
23602fdb BB |
292 | if (io) |
293 | time_left = spl_io_schedule_timeout(time_left); | |
294 | else | |
295 | time_left = schedule_timeout(time_left); | |
4efd4118 | 296 | |
058de03c | 297 | /* No more waiters a different mutex could be used */ |
d599e4fa | 298 | if (atomic_dec_and_test(&cvp->cv_waiters)) { |
e843553d CC |
299 | /* |
300 | * This is set without any lock, so it's racy. But this is | |
301 | * just for debug anyway, so make it best-effort | |
302 | */ | |
058de03c | 303 | cvp->cv_mutex = NULL; |
d599e4fa BB |
304 | wake_up(&cvp->cv_destroy); |
305 | } | |
058de03c | 306 | |
4efd4118 | 307 | finish_wait(&cvp->cv_event, &wait); |
d2733258 | 308 | atomic_dec(&cvp->cv_refs); |
4efd4118 | 309 | |
e843553d CC |
310 | /* |
311 | * Hold mutex after we release the cvp, otherwise we could dead lock | |
312 | * with a thread holding the mutex and call cv_destroy. | |
313 | */ | |
314 | mutex_enter(mp); | |
8056a756 | 315 | return (time_left > 0 ? 1 : -1); |
4efd4118 | 316 | } |
3f688a8c | 317 | |
8056a756 | 318 | int |
3f688a8c NK |
319 | __cv_timedwait(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) |
320 | { | |
23602fdb BB |
321 | return (__cv_timedwait_common(cvp, mp, exp_time, |
322 | TASK_UNINTERRUPTIBLE, 0)); | |
3f688a8c | 323 | } |
4efd4118 BB |
324 | EXPORT_SYMBOL(__cv_timedwait); |
325 | ||
8056a756 | 326 | int |
23602fdb BB |
327 | __cv_timedwait_io(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) |
328 | { | |
329 | return (__cv_timedwait_common(cvp, mp, exp_time, | |
330 | TASK_UNINTERRUPTIBLE, 1)); | |
331 | } | |
332 | EXPORT_SYMBOL(__cv_timedwait_io); | |
333 | ||
8056a756 | 334 | int |
23453686 | 335 | __cv_timedwait_sig(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) |
3f688a8c | 336 | { |
8056a756 MM |
337 | int rc; |
338 | ||
339 | rc = __cv_timedwait_common(cvp, mp, exp_time, TASK_INTERRUPTIBLE, 0); | |
340 | return (signal_pending(current) ? 0 : rc); | |
3f688a8c | 341 | } |
23453686 | 342 | EXPORT_SYMBOL(__cv_timedwait_sig); |
3f688a8c | 343 | |
ac6e5fb2 MM |
344 | int |
345 | __cv_timedwait_idle(kcondvar_t *cvp, kmutex_t *mp, clock_t exp_time) | |
346 | { | |
347 | sigset_t blocked, saved; | |
348 | int rc; | |
349 | ||
350 | sigfillset(&blocked); | |
351 | (void) sigprocmask(SIG_BLOCK, &blocked, &saved); | |
352 | rc = __cv_timedwait_common(cvp, mp, exp_time, | |
353 | TASK_INTERRUPTIBLE, 0); | |
354 | (void) sigprocmask(SIG_SETMASK, &saved, NULL); | |
355 | ||
356 | return (rc); | |
357 | } | |
358 | EXPORT_SYMBOL(__cv_timedwait_idle); | |
184c6873 | 359 | /* |
23453686 | 360 | * 'expire_time' argument is an absolute clock time in nanoseconds. |
184c6873 NB |
361 | * Return value is time left (expire_time - now) or -1 if timeout occurred. |
362 | */ | |
363 | static clock_t | |
23453686 | 364 | __cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t expire_time, |
8d042842 | 365 | hrtime_t res, int state) |
184c6873 NB |
366 | { |
367 | DEFINE_WAIT(wait); | |
e843553d | 368 | kmutex_t *m; |
2ded1c7e | 369 | hrtime_t time_left; |
39cd90ef | 370 | ktime_t ktime_left; |
8d042842 | 371 | u64 slack = 0; |
8056a756 | 372 | int rc; |
184c6873 NB |
373 | |
374 | ASSERT(cvp); | |
375 | ASSERT(mp); | |
376 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
377 | ASSERT(mutex_owned(mp)); | |
184c6873 | 378 | |
2ded1c7e BB |
379 | time_left = expire_time - gethrtime(); |
380 | if (time_left <= 0) | |
381 | return (-1); | |
382 | ||
383 | atomic_inc(&cvp->cv_refs); | |
93ce2b4c | 384 | m = READ_ONCE(cvp->cv_mutex); |
e843553d CC |
385 | if (!m) |
386 | m = xchg(&cvp->cv_mutex, mp); | |
184c6873 | 387 | /* Ensure the same mutex is used by all callers */ |
e843553d | 388 | ASSERT(m == NULL || m == mp); |
184c6873 | 389 | |
184c6873 NB |
390 | prepare_to_wait_exclusive(&cvp->cv_event, &wait, state); |
391 | atomic_inc(&cvp->cv_waiters); | |
392 | ||
23453686 BB |
393 | /* |
394 | * Mutex should be dropped after prepare_to_wait() this | |
184c6873 | 395 | * ensures we're linked in to the waiters list and avoids the |
23453686 BB |
396 | * race where 'cvp->cv_waiters > 0' but the list is empty. |
397 | */ | |
184c6873 | 398 | mutex_exit(mp); |
8d042842 | 399 | |
39cd90ef | 400 | ktime_left = ktime_set(0, time_left); |
8d042842 TN |
401 | slack = MIN(MAX(res, spl_schedule_hrtimeout_slack_us * NSEC_PER_USEC), |
402 | MAX_HRTIMEOUT_SLACK_US * NSEC_PER_USEC); | |
8056a756 | 403 | rc = schedule_hrtimeout_range(&ktime_left, slack, HRTIMER_MODE_REL); |
184c6873 NB |
404 | |
405 | /* No more waiters a different mutex could be used */ | |
406 | if (atomic_dec_and_test(&cvp->cv_waiters)) { | |
e843553d CC |
407 | /* |
408 | * This is set without any lock, so it's racy. But this is | |
409 | * just for debug anyway, so make it best-effort | |
410 | */ | |
184c6873 NB |
411 | cvp->cv_mutex = NULL; |
412 | wake_up(&cvp->cv_destroy); | |
413 | } | |
414 | ||
415 | finish_wait(&cvp->cv_event, &wait); | |
416 | atomic_dec(&cvp->cv_refs); | |
417 | ||
e843553d | 418 | mutex_enter(mp); |
8056a756 | 419 | return (rc == -EINTR ? 1 : -1); |
184c6873 NB |
420 | } |
421 | ||
422 | /* | |
423 | * Compatibility wrapper for the cv_timedwait_hires() Illumos interface. | |
424 | */ | |
8056a756 | 425 | static int |
5461eefe BB |
426 | cv_timedwait_hires_common(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, |
427 | hrtime_t res, int flag, int state) | |
184c6873 | 428 | { |
872e0cc9 CC |
429 | if (!(flag & CALLOUT_FLAG_ABSOLUTE)) |
430 | tim += gethrtime(); | |
39cd90ef | 431 | |
8d042842 | 432 | return (__cv_timedwait_hires(cvp, mp, tim, res, state)); |
39cd90ef | 433 | } |
184c6873 | 434 | |
8056a756 | 435 | int |
39cd90ef CC |
436 | cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, hrtime_t res, |
437 | int flag) | |
438 | { | |
439 | return (cv_timedwait_hires_common(cvp, mp, tim, res, flag, | |
440 | TASK_UNINTERRUPTIBLE)); | |
184c6873 NB |
441 | } |
442 | EXPORT_SYMBOL(cv_timedwait_hires); | |
443 | ||
8056a756 | 444 | int |
5461eefe BB |
445 | cv_timedwait_sig_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, |
446 | hrtime_t res, int flag) | |
39cd90ef | 447 | { |
8056a756 MM |
448 | int rc; |
449 | ||
450 | rc = cv_timedwait_hires_common(cvp, mp, tim, res, flag, | |
451 | TASK_INTERRUPTIBLE); | |
452 | return (signal_pending(current) ? 0 : rc); | |
39cd90ef CC |
453 | } |
454 | EXPORT_SYMBOL(cv_timedwait_sig_hires); | |
455 | ||
ac6e5fb2 MM |
456 | int |
457 | cv_timedwait_idle_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim, | |
458 | hrtime_t res, int flag) | |
459 | { | |
460 | sigset_t blocked, saved; | |
461 | int rc; | |
462 | ||
463 | sigfillset(&blocked); | |
464 | (void) sigprocmask(SIG_BLOCK, &blocked, &saved); | |
465 | rc = cv_timedwait_hires_common(cvp, mp, tim, res, flag, | |
466 | TASK_INTERRUPTIBLE); | |
467 | (void) sigprocmask(SIG_SETMASK, &saved, NULL); | |
468 | ||
469 | return (rc); | |
470 | } | |
471 | EXPORT_SYMBOL(cv_timedwait_idle_hires); | |
472 | ||
4efd4118 BB |
473 | void |
474 | __cv_signal(kcondvar_t *cvp) | |
475 | { | |
4efd4118 BB |
476 | ASSERT(cvp); |
477 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
d2733258 | 478 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 479 | |
23453686 BB |
480 | /* |
481 | * All waiters are added with WQ_FLAG_EXCLUSIVE so only one | |
9f5c1bc6 AG |
482 | * waiter will be set runnable with each call to wake_up(). |
483 | * Additionally wake_up() holds a spin_lock associated with | |
23453686 BB |
484 | * the wait queue to ensure we don't race waking up processes. |
485 | */ | |
4efd4118 BB |
486 | if (atomic_read(&cvp->cv_waiters) > 0) |
487 | wake_up(&cvp->cv_event); | |
488 | ||
d2733258 | 489 | atomic_dec(&cvp->cv_refs); |
4efd4118 BB |
490 | } |
491 | EXPORT_SYMBOL(__cv_signal); | |
492 | ||
493 | void | |
494 | __cv_broadcast(kcondvar_t *cvp) | |
495 | { | |
496 | ASSERT(cvp); | |
497 | ASSERT(cvp->cv_magic == CV_MAGIC); | |
d2733258 | 498 | atomic_inc(&cvp->cv_refs); |
4efd4118 | 499 | |
23453686 BB |
500 | /* |
501 | * Wake_up_all() will wake up all waiters even those which | |
502 | * have the WQ_FLAG_EXCLUSIVE flag set. | |
503 | */ | |
4efd4118 BB |
504 | if (atomic_read(&cvp->cv_waiters) > 0) |
505 | wake_up_all(&cvp->cv_event); | |
506 | ||
d2733258 | 507 | atomic_dec(&cvp->cv_refs); |
4efd4118 BB |
508 | } |
509 | EXPORT_SYMBOL(__cv_broadcast); |