]>
Commit | Line | Data |
---|---|---|
8b3d6663 AB |
1 | /* sched.c - SPU scheduler. |
2 | * | |
3 | * Copyright (C) IBM 2005 | |
4 | * Author: Mark Nutter <mnutter@us.ibm.com> | |
5 | * | |
6 | * SPU scheduler, based on Linux thread priority. For now use | |
7 | * a simple "cooperative" yield model with no preemption. SPU | |
8 | * scheduling will eventually be preemptive: When a thread with | |
9 | * a higher static priority gets ready to run, then an active SPU | |
10 | * context will be preempted and returned to the waitq. | |
11 | * | |
12 | * This program is free software; you can redistribute it and/or modify | |
13 | * it under the terms of the GNU General Public License as published by | |
14 | * the Free Software Foundation; either version 2, or (at your option) | |
15 | * any later version. | |
16 | * | |
17 | * This program is distributed in the hope that it will be useful, | |
18 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
20 | * GNU General Public License for more details. | |
21 | * | |
22 | * You should have received a copy of the GNU General Public License | |
23 | * along with this program; if not, write to the Free Software | |
24 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
25 | */ | |
26 | ||
3b3d22cb AB |
27 | #undef DEBUG |
28 | ||
8b3d6663 AB |
29 | #include <linux/config.h> |
30 | #include <linux/module.h> | |
31 | #include <linux/errno.h> | |
32 | #include <linux/sched.h> | |
33 | #include <linux/kernel.h> | |
34 | #include <linux/mm.h> | |
35 | #include <linux/completion.h> | |
36 | #include <linux/vmalloc.h> | |
37 | #include <linux/smp.h> | |
38 | #include <linux/smp_lock.h> | |
39 | #include <linux/stddef.h> | |
40 | #include <linux/unistd.h> | |
41 | ||
42 | #include <asm/io.h> | |
43 | #include <asm/mmu_context.h> | |
44 | #include <asm/spu.h> | |
45 | #include <asm/spu_csa.h> | |
46 | #include "spufs.h" | |
47 | ||
7945a4a2 | 48 | #define SPU_MIN_TIMESLICE (100 * HZ / 1000) |
2a911f0b | 49 | |
8b3d6663 AB |
50 | #define SPU_BITMAP_SIZE (((MAX_PRIO+BITS_PER_LONG)/BITS_PER_LONG)+1) |
51 | struct spu_prio_array { | |
52 | atomic_t nr_blocked; | |
53 | unsigned long bitmap[SPU_BITMAP_SIZE]; | |
54 | wait_queue_head_t waitq[MAX_PRIO]; | |
55 | }; | |
56 | ||
57 | /* spu_runqueue - This is the main runqueue data structure for SPUs. */ | |
58 | struct spu_runqueue { | |
59 | struct semaphore sem; | |
60 | unsigned long nr_active; | |
61 | unsigned long nr_idle; | |
62 | unsigned long nr_switches; | |
63 | struct list_head active_list; | |
64 | struct list_head idle_list; | |
65 | struct spu_prio_array prio; | |
66 | }; | |
67 | ||
68 | static struct spu_runqueue *spu_runqueues = NULL; | |
69 | ||
70 | static inline struct spu_runqueue *spu_rq(void) | |
71 | { | |
72 | /* Future: make this a per-NODE array, | |
73 | * and use cpu_to_node(smp_processor_id()) | |
74 | */ | |
75 | return spu_runqueues; | |
76 | } | |
77 | ||
78 | static inline struct spu *del_idle(struct spu_runqueue *rq) | |
79 | { | |
80 | struct spu *spu; | |
81 | ||
82 | BUG_ON(rq->nr_idle <= 0); | |
83 | BUG_ON(list_empty(&rq->idle_list)); | |
84 | /* Future: Move SPU out of low-power SRI state. */ | |
85 | spu = list_entry(rq->idle_list.next, struct spu, sched_list); | |
86 | list_del_init(&spu->sched_list); | |
87 | rq->nr_idle--; | |
88 | return spu; | |
89 | } | |
90 | ||
91 | static inline void del_active(struct spu_runqueue *rq, struct spu *spu) | |
92 | { | |
93 | BUG_ON(rq->nr_active <= 0); | |
94 | BUG_ON(list_empty(&rq->active_list)); | |
95 | list_del_init(&spu->sched_list); | |
96 | rq->nr_active--; | |
97 | } | |
98 | ||
99 | static inline void add_idle(struct spu_runqueue *rq, struct spu *spu) | |
100 | { | |
101 | /* Future: Put SPU into low-power SRI state. */ | |
102 | list_add_tail(&spu->sched_list, &rq->idle_list); | |
103 | rq->nr_idle++; | |
104 | } | |
105 | ||
106 | static inline void add_active(struct spu_runqueue *rq, struct spu *spu) | |
107 | { | |
108 | rq->nr_active++; | |
109 | rq->nr_switches++; | |
110 | list_add_tail(&spu->sched_list, &rq->active_list); | |
111 | } | |
112 | ||
113 | static void prio_wakeup(struct spu_runqueue *rq) | |
114 | { | |
115 | if (atomic_read(&rq->prio.nr_blocked) && rq->nr_idle) { | |
116 | int best = sched_find_first_bit(rq->prio.bitmap); | |
117 | if (best < MAX_PRIO) { | |
118 | wait_queue_head_t *wq = &rq->prio.waitq[best]; | |
119 | wake_up_interruptible_nr(wq, 1); | |
120 | } | |
121 | } | |
122 | } | |
123 | ||
5110459f AB |
124 | static void prio_wait(struct spu_runqueue *rq, struct spu_context *ctx, |
125 | u64 flags) | |
8b3d6663 AB |
126 | { |
127 | int prio = current->prio; | |
128 | wait_queue_head_t *wq = &rq->prio.waitq[prio]; | |
129 | DEFINE_WAIT(wait); | |
130 | ||
131 | __set_bit(prio, rq->prio.bitmap); | |
132 | atomic_inc(&rq->prio.nr_blocked); | |
133 | prepare_to_wait_exclusive(wq, &wait, TASK_INTERRUPTIBLE); | |
134 | if (!signal_pending(current)) { | |
135 | up(&rq->sem); | |
5110459f | 136 | up_write(&ctx->state_sema); |
8b3d6663 AB |
137 | pr_debug("%s: pid=%d prio=%d\n", __FUNCTION__, |
138 | current->pid, current->prio); | |
139 | schedule(); | |
5110459f | 140 | down_write(&ctx->state_sema); |
8b3d6663 AB |
141 | down(&rq->sem); |
142 | } | |
143 | finish_wait(wq, &wait); | |
144 | atomic_dec(&rq->prio.nr_blocked); | |
145 | if (!waitqueue_active(wq)) | |
146 | __clear_bit(prio, rq->prio.bitmap); | |
147 | } | |
148 | ||
149 | static inline int is_best_prio(struct spu_runqueue *rq) | |
150 | { | |
151 | int best_prio; | |
152 | ||
153 | best_prio = sched_find_first_bit(rq->prio.bitmap); | |
154 | return (current->prio < best_prio) ? 1 : 0; | |
155 | } | |
156 | ||
157 | static inline void mm_needs_global_tlbie(struct mm_struct *mm) | |
158 | { | |
159 | /* Global TLBIE broadcast required with SPEs. */ | |
160 | #if (NR_CPUS > 1) | |
161 | __cpus_setall(&mm->cpu_vm_mask, NR_CPUS); | |
162 | #else | |
163 | __cpus_setall(&mm->cpu_vm_mask, NR_CPUS+1); /* is this ok? */ | |
164 | #endif | |
165 | } | |
166 | ||
167 | static inline void bind_context(struct spu *spu, struct spu_context *ctx) | |
168 | { | |
169 | pr_debug("%s: pid=%d SPU=%d\n", __FUNCTION__, current->pid, | |
170 | spu->number); | |
171 | spu->ctx = ctx; | |
172 | spu->flags = 0; | |
2a911f0b | 173 | ctx->flags = 0; |
8b3d6663 AB |
174 | ctx->spu = spu; |
175 | ctx->ops = &spu_hw_ops; | |
176 | spu->pid = current->pid; | |
177 | spu->prio = current->prio; | |
178 | spu->mm = ctx->owner; | |
179 | mm_needs_global_tlbie(spu->mm); | |
180 | spu->ibox_callback = spufs_ibox_callback; | |
181 | spu->wbox_callback = spufs_wbox_callback; | |
5110459f | 182 | spu->stop_callback = spufs_stop_callback; |
8b3d6663 | 183 | mb(); |
5110459f | 184 | spu_unmap_mappings(ctx); |
8b3d6663 | 185 | spu_restore(&ctx->csa, spu); |
2a911f0b | 186 | spu->timestamp = jiffies; |
8b3d6663 AB |
187 | } |
188 | ||
189 | static inline void unbind_context(struct spu *spu, struct spu_context *ctx) | |
190 | { | |
191 | pr_debug("%s: unbind pid=%d SPU=%d\n", __FUNCTION__, | |
192 | spu->pid, spu->number); | |
5110459f | 193 | spu_unmap_mappings(ctx); |
8b3d6663 | 194 | spu_save(&ctx->csa, spu); |
2a911f0b | 195 | spu->timestamp = jiffies; |
8b3d6663 AB |
196 | ctx->state = SPU_STATE_SAVED; |
197 | spu->ibox_callback = NULL; | |
198 | spu->wbox_callback = NULL; | |
5110459f | 199 | spu->stop_callback = NULL; |
8b3d6663 AB |
200 | spu->mm = NULL; |
201 | spu->pid = 0; | |
202 | spu->prio = MAX_PRIO; | |
203 | ctx->ops = &spu_backing_ops; | |
204 | ctx->spu = NULL; | |
2a911f0b AB |
205 | ctx->flags = 0; |
206 | spu->flags = 0; | |
8b3d6663 AB |
207 | spu->ctx = NULL; |
208 | } | |
209 | ||
2a911f0b | 210 | static void spu_reaper(void *data) |
8b3d6663 | 211 | { |
2a911f0b AB |
212 | struct spu_context *ctx = data; |
213 | struct spu *spu; | |
8b3d6663 | 214 | |
2a911f0b AB |
215 | down_write(&ctx->state_sema); |
216 | spu = ctx->spu; | |
8837d921 | 217 | if (spu && test_bit(SPU_CONTEXT_PREEMPT, &ctx->flags)) { |
2a911f0b AB |
218 | if (atomic_read(&spu->rq->prio.nr_blocked)) { |
219 | pr_debug("%s: spu=%d\n", __func__, spu->number); | |
220 | ctx->ops->runcntl_stop(ctx); | |
221 | spu_deactivate(ctx); | |
222 | wake_up_all(&ctx->stop_wq); | |
223 | } else { | |
8837d921 | 224 | clear_bit(SPU_CONTEXT_PREEMPT, &ctx->flags); |
5110459f AB |
225 | } |
226 | } | |
2a911f0b AB |
227 | up_write(&ctx->state_sema); |
228 | put_spu_context(ctx); | |
229 | } | |
5110459f | 230 | |
2a911f0b AB |
231 | static void schedule_spu_reaper(struct spu_runqueue *rq, struct spu *spu) |
232 | { | |
233 | struct spu_context *ctx = get_spu_context(spu->ctx); | |
234 | unsigned long now = jiffies; | |
235 | unsigned long expire = spu->timestamp + SPU_MIN_TIMESLICE; | |
236 | ||
8837d921 | 237 | set_bit(SPU_CONTEXT_PREEMPT, &ctx->flags); |
2a911f0b AB |
238 | INIT_WORK(&ctx->reap_work, spu_reaper, ctx); |
239 | if (time_after(now, expire)) | |
240 | schedule_work(&ctx->reap_work); | |
241 | else | |
242 | schedule_delayed_work(&ctx->reap_work, expire - now); | |
243 | } | |
244 | ||
245 | static void check_preempt_active(struct spu_runqueue *rq) | |
246 | { | |
247 | struct list_head *p; | |
248 | struct spu *worst = NULL; | |
249 | ||
250 | list_for_each(p, &rq->active_list) { | |
251 | struct spu *spu = list_entry(p, struct spu, sched_list); | |
252 | struct spu_context *ctx = spu->ctx; | |
8837d921 | 253 | if (!test_bit(SPU_CONTEXT_PREEMPT, &ctx->flags)) { |
2a911f0b AB |
254 | if (!worst || (spu->prio > worst->prio)) { |
255 | worst = spu; | |
256 | } | |
8b3d6663 AB |
257 | } |
258 | } | |
2a911f0b AB |
259 | if (worst && (current->prio < worst->prio)) |
260 | schedule_spu_reaper(rq, worst); | |
8b3d6663 AB |
261 | } |
262 | ||
5110459f | 263 | static struct spu *get_idle_spu(struct spu_context *ctx, u64 flags) |
8b3d6663 AB |
264 | { |
265 | struct spu_runqueue *rq; | |
266 | struct spu *spu = NULL; | |
267 | ||
268 | rq = spu_rq(); | |
269 | down(&rq->sem); | |
270 | for (;;) { | |
271 | if (rq->nr_idle > 0) { | |
272 | if (is_best_prio(rq)) { | |
273 | /* Fall through. */ | |
274 | spu = del_idle(rq); | |
275 | break; | |
276 | } else { | |
277 | prio_wakeup(rq); | |
278 | up(&rq->sem); | |
279 | yield(); | |
280 | if (signal_pending(current)) { | |
281 | return NULL; | |
282 | } | |
283 | rq = spu_rq(); | |
284 | down(&rq->sem); | |
285 | continue; | |
286 | } | |
287 | } else { | |
2a911f0b | 288 | check_preempt_active(rq); |
5110459f | 289 | prio_wait(rq, ctx, flags); |
8b3d6663 AB |
290 | if (signal_pending(current)) { |
291 | prio_wakeup(rq); | |
292 | spu = NULL; | |
293 | break; | |
294 | } | |
295 | continue; | |
296 | } | |
297 | } | |
298 | up(&rq->sem); | |
299 | return spu; | |
300 | } | |
301 | ||
302 | static void put_idle_spu(struct spu *spu) | |
303 | { | |
304 | struct spu_runqueue *rq = spu->rq; | |
305 | ||
306 | down(&rq->sem); | |
307 | add_idle(rq, spu); | |
308 | prio_wakeup(rq); | |
309 | up(&rq->sem); | |
310 | } | |
311 | ||
312 | static int get_active_spu(struct spu *spu) | |
313 | { | |
314 | struct spu_runqueue *rq = spu->rq; | |
315 | struct list_head *p; | |
316 | struct spu *tmp; | |
317 | int rc = 0; | |
318 | ||
319 | down(&rq->sem); | |
320 | list_for_each(p, &rq->active_list) { | |
321 | tmp = list_entry(p, struct spu, sched_list); | |
322 | if (tmp == spu) { | |
323 | del_active(rq, spu); | |
324 | rc = 1; | |
325 | break; | |
326 | } | |
327 | } | |
328 | up(&rq->sem); | |
329 | return rc; | |
330 | } | |
331 | ||
332 | static void put_active_spu(struct spu *spu) | |
333 | { | |
334 | struct spu_runqueue *rq = spu->rq; | |
335 | ||
336 | down(&rq->sem); | |
337 | add_active(rq, spu); | |
338 | up(&rq->sem); | |
339 | } | |
340 | ||
341 | /* Lock order: | |
342 | * spu_activate() & spu_deactivate() require the | |
343 | * caller to have down_write(&ctx->state_sema). | |
344 | * | |
345 | * The rq->sem is breifly held (inside or outside a | |
346 | * given ctx lock) for list management, but is never | |
347 | * held during save/restore. | |
348 | */ | |
349 | ||
350 | int spu_activate(struct spu_context *ctx, u64 flags) | |
351 | { | |
352 | struct spu *spu; | |
353 | ||
354 | if (ctx->spu) | |
355 | return 0; | |
5110459f | 356 | spu = get_idle_spu(ctx, flags); |
8b3d6663 AB |
357 | if (!spu) |
358 | return (signal_pending(current)) ? -ERESTARTSYS : -EAGAIN; | |
359 | bind_context(spu, ctx); | |
2fb9d206 AB |
360 | /* |
361 | * We're likely to wait for interrupts on the same | |
362 | * CPU that we are now on, so send them here. | |
363 | */ | |
364 | spu_irq_setaffinity(spu, raw_smp_processor_id()); | |
8b3d6663 AB |
365 | put_active_spu(spu); |
366 | return 0; | |
367 | } | |
368 | ||
369 | void spu_deactivate(struct spu_context *ctx) | |
370 | { | |
371 | struct spu *spu; | |
372 | int needs_idle; | |
373 | ||
374 | spu = ctx->spu; | |
375 | if (!spu) | |
376 | return; | |
377 | needs_idle = get_active_spu(spu); | |
378 | unbind_context(spu, ctx); | |
379 | if (needs_idle) | |
380 | put_idle_spu(spu); | |
381 | } | |
382 | ||
383 | void spu_yield(struct spu_context *ctx) | |
384 | { | |
385 | struct spu *spu; | |
5110459f | 386 | int need_yield = 0; |
8b3d6663 | 387 | |
5110459f | 388 | down_write(&ctx->state_sema); |
8b3d6663 | 389 | spu = ctx->spu; |
5110459f | 390 | if (spu && (sched_find_first_bit(spu->rq->prio.bitmap) < MAX_PRIO)) { |
8b3d6663 AB |
391 | pr_debug("%s: yielding SPU %d\n", __FUNCTION__, spu->number); |
392 | spu_deactivate(ctx); | |
393 | ctx->state = SPU_STATE_SAVED; | |
5110459f | 394 | need_yield = 1; |
2a911f0b AB |
395 | } else if (spu) { |
396 | spu->prio = MAX_PRIO; | |
8b3d6663 AB |
397 | } |
398 | up_write(&ctx->state_sema); | |
5110459f AB |
399 | if (unlikely(need_yield)) |
400 | yield(); | |
8b3d6663 AB |
401 | } |
402 | ||
403 | int __init spu_sched_init(void) | |
404 | { | |
405 | struct spu_runqueue *rq; | |
406 | struct spu *spu; | |
407 | int i; | |
408 | ||
409 | rq = spu_runqueues = kmalloc(sizeof(struct spu_runqueue), GFP_KERNEL); | |
410 | if (!rq) { | |
411 | printk(KERN_WARNING "%s: Unable to allocate runqueues.\n", | |
412 | __FUNCTION__); | |
413 | return 1; | |
414 | } | |
415 | memset(rq, 0, sizeof(struct spu_runqueue)); | |
416 | init_MUTEX(&rq->sem); | |
417 | INIT_LIST_HEAD(&rq->active_list); | |
418 | INIT_LIST_HEAD(&rq->idle_list); | |
419 | rq->nr_active = 0; | |
420 | rq->nr_idle = 0; | |
421 | rq->nr_switches = 0; | |
422 | atomic_set(&rq->prio.nr_blocked, 0); | |
423 | for (i = 0; i < MAX_PRIO; i++) { | |
424 | init_waitqueue_head(&rq->prio.waitq[i]); | |
425 | __clear_bit(i, rq->prio.bitmap); | |
426 | } | |
427 | __set_bit(MAX_PRIO, rq->prio.bitmap); | |
428 | for (;;) { | |
429 | spu = spu_alloc(); | |
430 | if (!spu) | |
431 | break; | |
432 | pr_debug("%s: adding SPU[%d]\n", __FUNCTION__, spu->number); | |
433 | add_idle(rq, spu); | |
434 | spu->rq = rq; | |
2a911f0b | 435 | spu->timestamp = jiffies; |
8b3d6663 AB |
436 | } |
437 | if (!rq->nr_idle) { | |
438 | printk(KERN_WARNING "%s: No available SPUs.\n", __FUNCTION__); | |
439 | kfree(rq); | |
440 | return 1; | |
441 | } | |
442 | return 0; | |
443 | } | |
444 | ||
445 | void __exit spu_sched_exit(void) | |
446 | { | |
447 | struct spu_runqueue *rq = spu_rq(); | |
448 | struct spu *spu; | |
449 | ||
450 | if (!rq) { | |
451 | printk(KERN_WARNING "%s: no runqueues!\n", __FUNCTION__); | |
452 | return; | |
453 | } | |
454 | while (rq->nr_idle > 0) { | |
455 | spu = del_idle(rq); | |
456 | if (!spu) | |
457 | break; | |
458 | spu_free(spu); | |
459 | } | |
460 | kfree(rq); | |
461 | } |