]>
Commit | Line | Data |
---|---|---|
34dc7c2f BB |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
9 | * or http://www.opensolaris.org/os/licensing. | |
10 | * See the License for the specific language governing permissions | |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | /* | |
428870ff | 22 | * Copyright 2010 Sun Microsystems, Inc. All rights reserved. |
34dc7c2f BB |
23 | * Use is subject to license terms. |
24 | */ | |
25 | ||
34dc7c2f BB |
26 | #include <sys/zfs_context.h> |
27 | ||
28 | int taskq_now; | |
b128c09f | 29 | taskq_t *system_taskq; |
34dc7c2f BB |
30 | |
31 | typedef struct task { | |
32 | struct task *task_next; | |
33 | struct task *task_prev; | |
34 | task_func_t *task_func; | |
35 | void *task_arg; | |
36 | } task_t; | |
37 | ||
38 | #define TASKQ_ACTIVE 0x00010000 | |
39 | ||
40 | struct taskq { | |
41 | kmutex_t tq_lock; | |
42 | krwlock_t tq_threadlock; | |
43 | kcondvar_t tq_dispatch_cv; | |
44 | kcondvar_t tq_wait_cv; | |
1e33ac1e | 45 | kthread_t **tq_threadlist; |
34dc7c2f BB |
46 | int tq_flags; |
47 | int tq_active; | |
48 | int tq_nthreads; | |
49 | int tq_nalloc; | |
50 | int tq_minalloc; | |
51 | int tq_maxalloc; | |
428870ff BB |
52 | kcondvar_t tq_maxalloc_cv; |
53 | int tq_maxalloc_wait; | |
34dc7c2f BB |
54 | task_t *tq_freelist; |
55 | task_t tq_task; | |
56 | }; | |
57 | ||
58 | static task_t * | |
59 | task_alloc(taskq_t *tq, int tqflags) | |
60 | { | |
61 | task_t *t; | |
428870ff | 62 | int rv; |
34dc7c2f | 63 | |
428870ff | 64 | again: if ((t = tq->tq_freelist) != NULL && tq->tq_nalloc >= tq->tq_minalloc) { |
34dc7c2f BB |
65 | tq->tq_freelist = t->task_next; |
66 | } else { | |
34dc7c2f | 67 | if (tq->tq_nalloc >= tq->tq_maxalloc) { |
428870ff | 68 | if (!(tqflags & KM_SLEEP)) |
34dc7c2f | 69 | return (NULL); |
428870ff | 70 | |
34dc7c2f BB |
71 | /* |
72 | * We don't want to exceed tq_maxalloc, but we can't | |
73 | * wait for other tasks to complete (and thus free up | |
74 | * task structures) without risking deadlock with | |
75 | * the caller. So, we just delay for one second | |
428870ff BB |
76 | * to throttle the allocation rate. If we have tasks |
77 | * complete before one second timeout expires then | |
78 | * taskq_ent_free will signal us and we will | |
79 | * immediately retry the allocation. | |
34dc7c2f | 80 | */ |
428870ff BB |
81 | tq->tq_maxalloc_wait++; |
82 | rv = cv_timedwait(&tq->tq_maxalloc_cv, | |
83 | &tq->tq_lock, ddi_get_lbolt() + hz); | |
84 | tq->tq_maxalloc_wait--; | |
85 | if (rv > 0) | |
86 | goto again; /* signaled */ | |
34dc7c2f | 87 | } |
428870ff BB |
88 | mutex_exit(&tq->tq_lock); |
89 | ||
34dc7c2f | 90 | t = kmem_alloc(sizeof (task_t), tqflags); |
428870ff | 91 | |
34dc7c2f BB |
92 | mutex_enter(&tq->tq_lock); |
93 | if (t != NULL) | |
94 | tq->tq_nalloc++; | |
95 | } | |
96 | return (t); | |
97 | } | |
98 | ||
99 | static void | |
100 | task_free(taskq_t *tq, task_t *t) | |
101 | { | |
102 | if (tq->tq_nalloc <= tq->tq_minalloc) { | |
103 | t->task_next = tq->tq_freelist; | |
104 | tq->tq_freelist = t; | |
105 | } else { | |
106 | tq->tq_nalloc--; | |
107 | mutex_exit(&tq->tq_lock); | |
108 | kmem_free(t, sizeof (task_t)); | |
109 | mutex_enter(&tq->tq_lock); | |
110 | } | |
428870ff BB |
111 | |
112 | if (tq->tq_maxalloc_wait) | |
113 | cv_signal(&tq->tq_maxalloc_cv); | |
34dc7c2f BB |
114 | } |
115 | ||
116 | taskqid_t | |
117 | taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t tqflags) | |
118 | { | |
119 | task_t *t; | |
120 | ||
121 | if (taskq_now) { | |
122 | func(arg); | |
123 | return (1); | |
124 | } | |
125 | ||
126 | mutex_enter(&tq->tq_lock); | |
127 | ASSERT(tq->tq_flags & TASKQ_ACTIVE); | |
128 | if ((t = task_alloc(tq, tqflags)) == NULL) { | |
129 | mutex_exit(&tq->tq_lock); | |
130 | return (0); | |
131 | } | |
428870ff BB |
132 | if (tqflags & TQ_FRONT) { |
133 | t->task_next = tq->tq_task.task_next; | |
134 | t->task_prev = &tq->tq_task; | |
135 | } else { | |
136 | t->task_next = &tq->tq_task; | |
137 | t->task_prev = tq->tq_task.task_prev; | |
138 | } | |
34dc7c2f BB |
139 | t->task_next->task_prev = t; |
140 | t->task_prev->task_next = t; | |
141 | t->task_func = func; | |
142 | t->task_arg = arg; | |
143 | cv_signal(&tq->tq_dispatch_cv); | |
144 | mutex_exit(&tq->tq_lock); | |
145 | return (1); | |
146 | } | |
147 | ||
148 | void | |
149 | taskq_wait(taskq_t *tq) | |
150 | { | |
151 | mutex_enter(&tq->tq_lock); | |
152 | while (tq->tq_task.task_next != &tq->tq_task || tq->tq_active != 0) | |
153 | cv_wait(&tq->tq_wait_cv, &tq->tq_lock); | |
154 | mutex_exit(&tq->tq_lock); | |
155 | } | |
156 | ||
1e33ac1e | 157 | static void |
34dc7c2f BB |
158 | taskq_thread(void *arg) |
159 | { | |
160 | taskq_t *tq = arg; | |
161 | task_t *t; | |
162 | ||
163 | mutex_enter(&tq->tq_lock); | |
164 | while (tq->tq_flags & TASKQ_ACTIVE) { | |
165 | if ((t = tq->tq_task.task_next) == &tq->tq_task) { | |
166 | if (--tq->tq_active == 0) | |
167 | cv_broadcast(&tq->tq_wait_cv); | |
168 | cv_wait(&tq->tq_dispatch_cv, &tq->tq_lock); | |
169 | tq->tq_active++; | |
170 | continue; | |
171 | } | |
172 | t->task_prev->task_next = t->task_next; | |
173 | t->task_next->task_prev = t->task_prev; | |
174 | mutex_exit(&tq->tq_lock); | |
175 | ||
176 | rw_enter(&tq->tq_threadlock, RW_READER); | |
177 | t->task_func(t->task_arg); | |
178 | rw_exit(&tq->tq_threadlock); | |
179 | ||
180 | mutex_enter(&tq->tq_lock); | |
181 | task_free(tq, t); | |
182 | } | |
183 | tq->tq_nthreads--; | |
184 | cv_broadcast(&tq->tq_wait_cv); | |
185 | mutex_exit(&tq->tq_lock); | |
1e33ac1e | 186 | thread_exit(); |
34dc7c2f BB |
187 | } |
188 | ||
189 | /*ARGSUSED*/ | |
190 | taskq_t * | |
191 | taskq_create(const char *name, int nthreads, pri_t pri, | |
192 | int minalloc, int maxalloc, uint_t flags) | |
193 | { | |
194 | taskq_t *tq = kmem_zalloc(sizeof (taskq_t), KM_SLEEP); | |
195 | int t; | |
196 | ||
9babb374 BB |
197 | if (flags & TASKQ_THREADS_CPU_PCT) { |
198 | int pct; | |
199 | ASSERT3S(nthreads, >=, 0); | |
200 | ASSERT3S(nthreads, <=, 100); | |
201 | pct = MIN(nthreads, 100); | |
202 | pct = MAX(pct, 0); | |
203 | ||
204 | nthreads = (sysconf(_SC_NPROCESSORS_ONLN) * pct) / 100; | |
205 | nthreads = MAX(nthreads, 1); /* need at least 1 thread */ | |
206 | } else { | |
207 | ASSERT3S(nthreads, >=, 1); | |
208 | } | |
209 | ||
34dc7c2f BB |
210 | rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL); |
211 | mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL); | |
212 | cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL); | |
213 | cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL); | |
428870ff | 214 | cv_init(&tq->tq_maxalloc_cv, NULL, CV_DEFAULT, NULL); |
34dc7c2f BB |
215 | tq->tq_flags = flags | TASKQ_ACTIVE; |
216 | tq->tq_active = nthreads; | |
217 | tq->tq_nthreads = nthreads; | |
218 | tq->tq_minalloc = minalloc; | |
219 | tq->tq_maxalloc = maxalloc; | |
220 | tq->tq_task.task_next = &tq->tq_task; | |
221 | tq->tq_task.task_prev = &tq->tq_task; | |
1e33ac1e | 222 | tq->tq_threadlist = kmem_alloc(nthreads*sizeof(kthread_t *), KM_SLEEP); |
34dc7c2f BB |
223 | |
224 | if (flags & TASKQ_PREPOPULATE) { | |
225 | mutex_enter(&tq->tq_lock); | |
226 | while (minalloc-- > 0) | |
227 | task_free(tq, task_alloc(tq, KM_SLEEP)); | |
228 | mutex_exit(&tq->tq_lock); | |
229 | } | |
230 | ||
231 | for (t = 0; t < nthreads; t++) | |
1e33ac1e BB |
232 | VERIFY((tq->tq_threadlist[t] = thread_create(NULL, 0, |
233 | taskq_thread, tq, TS_RUN, NULL, 0, 0)) != NULL); | |
34dc7c2f BB |
234 | |
235 | return (tq); | |
236 | } | |
237 | ||
238 | void | |
239 | taskq_destroy(taskq_t *tq) | |
240 | { | |
34dc7c2f BB |
241 | int nthreads = tq->tq_nthreads; |
242 | ||
243 | taskq_wait(tq); | |
244 | ||
245 | mutex_enter(&tq->tq_lock); | |
246 | ||
247 | tq->tq_flags &= ~TASKQ_ACTIVE; | |
248 | cv_broadcast(&tq->tq_dispatch_cv); | |
249 | ||
250 | while (tq->tq_nthreads != 0) | |
251 | cv_wait(&tq->tq_wait_cv, &tq->tq_lock); | |
252 | ||
253 | tq->tq_minalloc = 0; | |
254 | while (tq->tq_nalloc != 0) { | |
255 | ASSERT(tq->tq_freelist != NULL); | |
256 | task_free(tq, task_alloc(tq, KM_SLEEP)); | |
257 | } | |
258 | ||
259 | mutex_exit(&tq->tq_lock); | |
260 | ||
1e33ac1e | 261 | kmem_free(tq->tq_threadlist, nthreads * sizeof (kthread_t *)); |
34dc7c2f BB |
262 | |
263 | rw_destroy(&tq->tq_threadlock); | |
264 | mutex_destroy(&tq->tq_lock); | |
265 | cv_destroy(&tq->tq_dispatch_cv); | |
266 | cv_destroy(&tq->tq_wait_cv); | |
428870ff | 267 | cv_destroy(&tq->tq_maxalloc_cv); |
34dc7c2f BB |
268 | |
269 | kmem_free(tq, sizeof (taskq_t)); | |
270 | } | |
271 | ||
272 | int | |
1e33ac1e | 273 | taskq_member(taskq_t *tq, kthread_t *t) |
34dc7c2f BB |
274 | { |
275 | int i; | |
276 | ||
277 | if (taskq_now) | |
278 | return (1); | |
279 | ||
280 | for (i = 0; i < tq->tq_nthreads; i++) | |
1e33ac1e | 281 | if (tq->tq_threadlist[i] == t) |
34dc7c2f BB |
282 | return (1); |
283 | ||
284 | return (0); | |
285 | } | |
b128c09f BB |
286 | |
287 | void | |
288 | system_taskq_init(void) | |
289 | { | |
290 | system_taskq = taskq_create("system_taskq", 64, minclsyspri, 4, 512, | |
291 | TASKQ_DYNAMIC | TASKQ_PREPOPULATE); | |
292 | } | |
428870ff BB |
293 | |
294 | void | |
295 | system_taskq_fini(void) | |
296 | { | |
297 | taskq_destroy(system_taskq); | |
298 | system_taskq = NULL; /* defensive */ | |
299 | } |