]>
git.proxmox.com Git - mirror_ubuntu-focal-kernel.git/blob - arch/powerpc/platforms/cell/spufs/sched.c
91030b8abdca6fbc62ca05efad87367f102c6885
1 /* sched.c - SPU scheduler.
3 * Copyright (C) IBM 2005
4 * Author: Mark Nutter <mnutter@us.ibm.com>
6 * 2006-03-31 NUMA domains added.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 #include <linux/module.h>
26 #include <linux/errno.h>
27 #include <linux/sched.h>
28 #include <linux/kernel.h>
30 #include <linux/completion.h>
31 #include <linux/vmalloc.h>
32 #include <linux/smp.h>
33 #include <linux/smp_lock.h>
34 #include <linux/stddef.h>
35 #include <linux/unistd.h>
36 #include <linux/numa.h>
37 #include <linux/mutex.h>
38 #include <linux/notifier.h>
41 #include <asm/mmu_context.h>
43 #include <asm/spu_csa.h>
44 #include <asm/spu_priv1.h>
47 #define SPU_TIMESLICE (HZ)
49 struct spu_prio_array
{
50 DECLARE_BITMAP(bitmap
, MAX_PRIO
);
51 struct list_head runq
[MAX_PRIO
];
53 struct list_head active_list
[MAX_NUMNODES
];
54 struct mutex active_mutex
[MAX_NUMNODES
];
57 static struct spu_prio_array
*spu_prio
;
58 static struct workqueue_struct
*spu_sched_wq
;
60 static inline int node_allowed(int node
)
64 if (!nr_cpus_node(node
))
66 mask
= node_to_cpumask(node
);
67 if (!cpus_intersects(mask
, current
->cpus_allowed
))
72 void spu_start_tick(struct spu_context
*ctx
)
74 if (ctx
->policy
== SCHED_RR
) {
76 * Make sure the exiting bit is cleared.
78 clear_bit(SPU_SCHED_EXITING
, &ctx
->sched_flags
);
80 queue_delayed_work(spu_sched_wq
, &ctx
->sched_work
, SPU_TIMESLICE
);
84 void spu_stop_tick(struct spu_context
*ctx
)
86 if (ctx
->policy
== SCHED_RR
) {
88 * While the work can be rearming normally setting this flag
89 * makes sure it does not rearm itself anymore.
91 set_bit(SPU_SCHED_EXITING
, &ctx
->sched_flags
);
93 cancel_delayed_work(&ctx
->sched_work
);
97 void spu_sched_tick(struct work_struct
*work
)
99 struct spu_context
*ctx
=
100 container_of(work
, struct spu_context
, sched_work
.work
);
105 * If this context is being stopped avoid rescheduling from the
106 * scheduler tick because we would block on the state_mutex.
107 * The caller will yield the spu later on anyway.
109 if (test_bit(SPU_SCHED_EXITING
, &ctx
->sched_flags
))
112 mutex_lock(&ctx
->state_mutex
);
115 int best
= sched_find_first_bit(spu_prio
->bitmap
);
116 if (best
<= ctx
->prio
) {
121 mutex_unlock(&ctx
->state_mutex
);
125 * We need to break out of the wait loop in spu_run manually
126 * to ensure this context gets put on the runqueue again
129 wake_up(&ctx
->stop_wq
);
135 * spu_add_to_active_list - add spu to active list
136 * @spu: spu to add to the active list
138 static void spu_add_to_active_list(struct spu
*spu
)
140 mutex_lock(&spu_prio
->active_mutex
[spu
->node
]);
141 list_add_tail(&spu
->list
, &spu_prio
->active_list
[spu
->node
]);
142 mutex_unlock(&spu_prio
->active_mutex
[spu
->node
]);
146 * spu_remove_from_active_list - remove spu from active list
147 * @spu: spu to remove from the active list
149 static void spu_remove_from_active_list(struct spu
*spu
)
151 int node
= spu
->node
;
153 mutex_lock(&spu_prio
->active_mutex
[node
]);
154 list_del_init(&spu
->list
);
155 mutex_unlock(&spu_prio
->active_mutex
[node
]);
158 static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier
);
160 static void spu_switch_notify(struct spu
*spu
, struct spu_context
*ctx
)
162 blocking_notifier_call_chain(&spu_switch_notifier
,
163 ctx
? ctx
->object_id
: 0, spu
);
166 int spu_switch_event_register(struct notifier_block
* n
)
168 return blocking_notifier_chain_register(&spu_switch_notifier
, n
);
171 int spu_switch_event_unregister(struct notifier_block
* n
)
173 return blocking_notifier_chain_unregister(&spu_switch_notifier
, n
);
177 * spu_bind_context - bind spu context to physical spu
178 * @spu: physical spu to bind to
179 * @ctx: context to bind
181 static void spu_bind_context(struct spu
*spu
, struct spu_context
*ctx
)
183 pr_debug("%s: pid=%d SPU=%d NODE=%d\n", __FUNCTION__
, current
->pid
,
184 spu
->number
, spu
->node
);
188 ctx
->ops
= &spu_hw_ops
;
189 spu
->pid
= current
->pid
;
190 spu_associate_mm(spu
, ctx
->owner
);
191 spu
->ibox_callback
= spufs_ibox_callback
;
192 spu
->wbox_callback
= spufs_wbox_callback
;
193 spu
->stop_callback
= spufs_stop_callback
;
194 spu
->mfc_callback
= spufs_mfc_callback
;
195 spu
->dma_callback
= spufs_dma_callback
;
197 spu_unmap_mappings(ctx
);
198 spu_restore(&ctx
->csa
, spu
);
199 spu
->timestamp
= jiffies
;
200 spu_cpu_affinity_set(spu
, raw_smp_processor_id());
201 spu_switch_notify(spu
, ctx
);
202 spu_add_to_active_list(spu
);
203 ctx
->state
= SPU_STATE_RUNNABLE
;
207 * spu_unbind_context - unbind spu context from physical spu
208 * @spu: physical spu to unbind from
209 * @ctx: context to unbind
211 static void spu_unbind_context(struct spu
*spu
, struct spu_context
*ctx
)
213 pr_debug("%s: unbind pid=%d SPU=%d NODE=%d\n", __FUNCTION__
,
214 spu
->pid
, spu
->number
, spu
->node
);
216 spu_remove_from_active_list(spu
);
217 spu_switch_notify(spu
, NULL
);
218 spu_unmap_mappings(ctx
);
219 spu_save(&ctx
->csa
, spu
);
220 spu
->timestamp
= jiffies
;
221 ctx
->state
= SPU_STATE_SAVED
;
222 spu
->ibox_callback
= NULL
;
223 spu
->wbox_callback
= NULL
;
224 spu
->stop_callback
= NULL
;
225 spu
->mfc_callback
= NULL
;
226 spu
->dma_callback
= NULL
;
227 spu_associate_mm(spu
, NULL
);
229 ctx
->ops
= &spu_backing_ops
;
236 * spu_add_to_rq - add a context to the runqueue
237 * @ctx: context to add
239 static void __spu_add_to_rq(struct spu_context
*ctx
)
241 int prio
= ctx
->prio
;
243 list_add_tail(&ctx
->rq
, &spu_prio
->runq
[prio
]);
244 set_bit(prio
, spu_prio
->bitmap
);
247 static void __spu_del_from_rq(struct spu_context
*ctx
)
249 int prio
= ctx
->prio
;
251 if (!list_empty(&ctx
->rq
))
252 list_del_init(&ctx
->rq
);
253 if (list_empty(&spu_prio
->runq
[prio
]))
254 clear_bit(prio
, spu_prio
->bitmap
);
257 static void spu_prio_wait(struct spu_context
*ctx
)
261 spin_lock(&spu_prio
->runq_lock
);
262 prepare_to_wait_exclusive(&ctx
->stop_wq
, &wait
, TASK_INTERRUPTIBLE
);
263 if (!signal_pending(current
)) {
264 __spu_add_to_rq(ctx
);
265 spin_unlock(&spu_prio
->runq_lock
);
266 mutex_unlock(&ctx
->state_mutex
);
268 mutex_lock(&ctx
->state_mutex
);
269 spin_lock(&spu_prio
->runq_lock
);
270 __spu_del_from_rq(ctx
);
272 spin_unlock(&spu_prio
->runq_lock
);
273 __set_current_state(TASK_RUNNING
);
274 remove_wait_queue(&ctx
->stop_wq
, &wait
);
278 * spu_reschedule - try to find a runnable context for a spu
279 * @spu: spu available
281 * This function is called whenever a spu becomes idle. It looks for the
282 * most suitable runnable spu context and schedules it for execution.
284 static void spu_reschedule(struct spu
*spu
)
290 spin_lock(&spu_prio
->runq_lock
);
291 best
= sched_find_first_bit(spu_prio
->bitmap
);
292 if (best
< MAX_PRIO
) {
293 struct list_head
*rq
= &spu_prio
->runq
[best
];
294 struct spu_context
*ctx
;
296 BUG_ON(list_empty(rq
));
298 ctx
= list_entry(rq
->next
, struct spu_context
, rq
);
299 __spu_del_from_rq(ctx
);
300 wake_up(&ctx
->stop_wq
);
302 spin_unlock(&spu_prio
->runq_lock
);
305 static struct spu
*spu_get_idle(struct spu_context
*ctx
)
307 struct spu
*spu
= NULL
;
308 int node
= cpu_to_node(raw_smp_processor_id());
311 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
312 node
= (node
< MAX_NUMNODES
) ? node
: 0;
313 if (!node_allowed(node
))
315 spu
= spu_alloc_node(node
);
323 * find_victim - find a lower priority context to preempt
324 * @ctx: canidate context for running
326 * Returns the freed physical spu to run the new context on.
328 static struct spu
*find_victim(struct spu_context
*ctx
)
330 struct spu_context
*victim
= NULL
;
335 * Look for a possible preemption candidate on the local node first.
336 * If there is no candidate look at the other nodes. This isn't
337 * exactly fair, but so far the whole spu schedule tries to keep
338 * a strong node affinity. We might want to fine-tune this in
342 node
= cpu_to_node(raw_smp_processor_id());
343 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
344 node
= (node
< MAX_NUMNODES
) ? node
: 0;
345 if (!node_allowed(node
))
348 mutex_lock(&spu_prio
->active_mutex
[node
]);
349 list_for_each_entry(spu
, &spu_prio
->active_list
[node
], list
) {
350 struct spu_context
*tmp
= spu
->ctx
;
352 if (tmp
->rt_priority
< ctx
->rt_priority
&&
353 (!victim
|| tmp
->rt_priority
< victim
->rt_priority
))
356 mutex_unlock(&spu_prio
->active_mutex
[node
]);
360 * This nests ctx->state_mutex, but we always lock
361 * higher priority contexts before lower priority
362 * ones, so this is safe until we introduce
363 * priority inheritance schemes.
365 if (!mutex_trylock(&victim
->state_mutex
)) {
373 * This race can happen because we've dropped
374 * the active list mutex. No a problem, just
375 * restart the search.
377 mutex_unlock(&victim
->state_mutex
);
381 spu_unbind_context(spu
, victim
);
382 mutex_unlock(&victim
->state_mutex
);
384 * We need to break out of the wait loop in spu_run
385 * manually to ensure this context gets put on the
386 * runqueue again ASAP.
388 wake_up(&victim
->stop_wq
);
397 * spu_activate - find a free spu for a context and execute it
398 * @ctx: spu context to schedule
399 * @flags: flags (currently ignored)
401 * Tries to find a free spu to run @ctx. If no free spu is available
402 * add the context to the runqueue so it gets woken up once an spu
405 int spu_activate(struct spu_context
*ctx
, unsigned long flags
)
414 spu
= spu_get_idle(ctx
);
416 * If this is a realtime thread we try to get it running by
417 * preempting a lower priority thread.
419 if (!spu
&& ctx
->rt_priority
)
420 spu
= find_victim(ctx
);
422 spu_bind_context(spu
, ctx
);
427 } while (!signal_pending(current
));
433 * spu_deactivate - unbind a context from it's physical spu
434 * @ctx: spu context to unbind
436 * Unbind @ctx from the physical spu it is running on and schedule
437 * the highest priority context to run on the freed physical spu.
439 void spu_deactivate(struct spu_context
*ctx
)
441 struct spu
*spu
= ctx
->spu
;
444 spu_unbind_context(spu
, ctx
);
450 * spu_yield - yield a physical spu if others are waiting
451 * @ctx: spu context to yield
453 * Check if there is a higher priority context waiting and if yes
454 * unbind @ctx from the physical spu and schedule the highest
455 * priority context to run on the freed physical spu instead.
457 void spu_yield(struct spu_context
*ctx
)
461 if (mutex_trylock(&ctx
->state_mutex
)) {
462 if ((spu
= ctx
->spu
) != NULL
) {
463 int best
= sched_find_first_bit(spu_prio
->bitmap
);
464 if (best
< MAX_PRIO
) {
465 pr_debug("%s: yielding SPU %d NODE %d\n",
466 __FUNCTION__
, spu
->number
, spu
->node
);
470 mutex_unlock(&ctx
->state_mutex
);
474 int __init
spu_sched_init(void)
478 spu_sched_wq
= create_singlethread_workqueue("spusched");
482 spu_prio
= kzalloc(sizeof(struct spu_prio_array
), GFP_KERNEL
);
484 printk(KERN_WARNING
"%s: Unable to allocate priority queue.\n",
486 destroy_workqueue(spu_sched_wq
);
489 for (i
= 0; i
< MAX_PRIO
; i
++) {
490 INIT_LIST_HEAD(&spu_prio
->runq
[i
]);
491 __clear_bit(i
, spu_prio
->bitmap
);
493 __set_bit(MAX_PRIO
, spu_prio
->bitmap
);
494 for (i
= 0; i
< MAX_NUMNODES
; i
++) {
495 mutex_init(&spu_prio
->active_mutex
[i
]);
496 INIT_LIST_HEAD(&spu_prio
->active_list
[i
]);
498 spin_lock_init(&spu_prio
->runq_lock
);
502 void __exit
spu_sched_exit(void)
504 struct spu
*spu
, *tmp
;
507 for (node
= 0; node
< MAX_NUMNODES
; node
++) {
508 mutex_lock(&spu_prio
->active_mutex
[node
]);
509 list_for_each_entry_safe(spu
, tmp
, &spu_prio
->active_list
[node
],
511 list_del_init(&spu
->list
);
514 mutex_unlock(&spu_prio
->active_mutex
[node
]);
517 destroy_workqueue(spu_sched_wq
);