1 /* sched.c - SPU scheduler.
3 * Copyright (C) IBM 2005
4 * Author: Mark Nutter <mnutter@us.ibm.com>
6 * 2006-03-31 NUMA domains added.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 #include <linux/module.h>
26 #include <linux/errno.h>
27 #include <linux/sched.h>
28 #include <linux/kernel.h>
30 #include <linux/completion.h>
31 #include <linux/vmalloc.h>
32 #include <linux/smp.h>
33 #include <linux/stddef.h>
34 #include <linux/unistd.h>
35 #include <linux/numa.h>
36 #include <linux/mutex.h>
37 #include <linux/notifier.h>
38 #include <linux/kthread.h>
39 #include <linux/pid_namespace.h>
40 #include <linux/proc_fs.h>
41 #include <linux/seq_file.h>
42 #include <linux/marker.h>
45 #include <asm/mmu_context.h>
47 #include <asm/spu_csa.h>
48 #include <asm/spu_priv1.h>
51 struct spu_prio_array
{
52 DECLARE_BITMAP(bitmap
, MAX_PRIO
);
53 struct list_head runq
[MAX_PRIO
];
58 static unsigned long spu_avenrun
[3];
59 static struct spu_prio_array
*spu_prio
;
60 static struct task_struct
*spusched_task
;
61 static struct timer_list spusched_timer
;
62 static struct timer_list spuloadavg_timer
;
65 * Priority of a normal, non-rt, non-niced'd process (aka nice level 0).
67 #define NORMAL_PRIO 120
70 * Frequency of the spu scheduler tick. By default we do one SPU scheduler
71 * tick for every 10 CPU scheduler ticks.
73 #define SPUSCHED_TICK (10)
76 * These are the 'tuning knobs' of the scheduler:
78 * Minimum timeslice is 5 msecs (or 1 spu scheduler tick, whichever is
79 * larger), default timeslice is 100 msecs, maximum timeslice is 800 msecs.
81 #define MIN_SPU_TIMESLICE max(5 * HZ / (1000 * SPUSCHED_TICK), 1)
82 #define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK))
84 #define MAX_USER_PRIO (MAX_PRIO - MAX_RT_PRIO)
85 #define SCALE_PRIO(x, prio) \
86 max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_SPU_TIMESLICE)
89 * scale user-nice values [ -20 ... 0 ... 19 ] to time slice values:
90 * [800ms ... 100ms ... 5ms]
92 * The higher a thread's priority, the bigger timeslices
93 * it gets during one round of execution. But even the lowest
94 * priority thread gets MIN_TIMESLICE worth of execution time.
96 void spu_set_timeslice(struct spu_context
*ctx
)
98 if (ctx
->prio
< NORMAL_PRIO
)
99 ctx
->time_slice
= SCALE_PRIO(DEF_SPU_TIMESLICE
* 4, ctx
->prio
);
101 ctx
->time_slice
= SCALE_PRIO(DEF_SPU_TIMESLICE
, ctx
->prio
);
105 * Update scheduling information from the owning thread.
107 void __spu_update_sched_info(struct spu_context
*ctx
)
110 * assert that the context is not on the runqueue, so it is safe
111 * to change its scheduling parameters.
113 BUG_ON(!list_empty(&ctx
->rq
));
116 * 32-Bit assignments are atomic on powerpc, and we don't care about
117 * memory ordering here because retrieving the controlling thread is
118 * per definition racy.
120 ctx
->tid
= current
->pid
;
123 * We do our own priority calculations, so we normally want
124 * ->static_prio to start with. Unfortunately this field
125 * contains junk for threads with a realtime scheduling
126 * policy so we have to look at ->prio in this case.
128 if (rt_prio(current
->prio
))
129 ctx
->prio
= current
->prio
;
131 ctx
->prio
= current
->static_prio
;
132 ctx
->policy
= current
->policy
;
135 * TO DO: the context may be loaded, so we may need to activate
136 * it again on a different node. But it shouldn't hurt anything
137 * to update its parameters, because we know that the scheduler
138 * is not actively looking at this field, since it is not on the
139 * runqueue. The context will be rescheduled on the proper node
140 * if it is timesliced or preempted.
142 ctx
->cpus_allowed
= current
->cpus_allowed
;
144 /* Save the current cpu id for spu interrupt routing. */
145 ctx
->last_ran
= raw_smp_processor_id();
148 void spu_update_sched_info(struct spu_context
*ctx
)
152 if (ctx
->state
== SPU_STATE_RUNNABLE
) {
153 node
= ctx
->spu
->node
;
156 * Take list_mutex to sync with find_victim().
158 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
159 __spu_update_sched_info(ctx
);
160 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
162 __spu_update_sched_info(ctx
);
166 static int __node_allowed(struct spu_context
*ctx
, int node
)
168 if (nr_cpus_node(node
)) {
169 cpumask_t mask
= node_to_cpumask(node
);
171 if (cpus_intersects(mask
, ctx
->cpus_allowed
))
178 static int node_allowed(struct spu_context
*ctx
, int node
)
182 spin_lock(&spu_prio
->runq_lock
);
183 rval
= __node_allowed(ctx
, node
);
184 spin_unlock(&spu_prio
->runq_lock
);
189 void do_notify_spus_active(void)
194 * Wake up the active spu_contexts.
196 * When the awakened processes see their "notify_active" flag is set,
197 * they will call spu_switch_notify().
199 for_each_online_node(node
) {
202 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
203 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
204 if (spu
->alloc_state
!= SPU_FREE
) {
205 struct spu_context
*ctx
= spu
->ctx
;
206 set_bit(SPU_SCHED_NOTIFY_ACTIVE
,
209 wake_up_all(&ctx
->stop_wq
);
212 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
217 * spu_bind_context - bind spu context to physical spu
218 * @spu: physical spu to bind to
219 * @ctx: context to bind
221 static void spu_bind_context(struct spu
*spu
, struct spu_context
*ctx
)
223 spu_context_trace(spu_bind_context__enter
, ctx
, spu
);
225 spuctx_switch_state(ctx
, SPU_UTIL_SYSTEM
);
227 if (ctx
->flags
& SPU_CREATE_NOSCHED
)
228 atomic_inc(&cbe_spu_info
[spu
->node
].reserved_spus
);
230 ctx
->stats
.slb_flt_base
= spu
->stats
.slb_flt
;
231 ctx
->stats
.class2_intr_base
= spu
->stats
.class2_intr
;
233 spu_associate_mm(spu
, ctx
->owner
);
235 spin_lock_irq(&spu
->register_lock
);
239 ctx
->ops
= &spu_hw_ops
;
240 spu
->pid
= current
->pid
;
241 spu
->tgid
= current
->tgid
;
242 spu
->ibox_callback
= spufs_ibox_callback
;
243 spu
->wbox_callback
= spufs_wbox_callback
;
244 spu
->stop_callback
= spufs_stop_callback
;
245 spu
->mfc_callback
= spufs_mfc_callback
;
246 spin_unlock_irq(&spu
->register_lock
);
248 spu_unmap_mappings(ctx
);
250 spu_switch_log_notify(spu
, ctx
, SWITCH_LOG_START
, 0);
251 spu_restore(&ctx
->csa
, spu
);
252 spu
->timestamp
= jiffies
;
253 spu_switch_notify(spu
, ctx
);
254 ctx
->state
= SPU_STATE_RUNNABLE
;
256 spuctx_switch_state(ctx
, SPU_UTIL_USER
);
260 * Must be used with the list_mutex held.
262 static inline int sched_spu(struct spu
*spu
)
264 BUG_ON(!mutex_is_locked(&cbe_spu_info
[spu
->node
].list_mutex
));
266 return (!spu
->ctx
|| !(spu
->ctx
->flags
& SPU_CREATE_NOSCHED
));
269 static void aff_merge_remaining_ctxs(struct spu_gang
*gang
)
271 struct spu_context
*ctx
;
273 list_for_each_entry(ctx
, &gang
->aff_list_head
, aff_list
) {
274 if (list_empty(&ctx
->aff_list
))
275 list_add(&ctx
->aff_list
, &gang
->aff_list_head
);
277 gang
->aff_flags
|= AFF_MERGED
;
280 static void aff_set_offsets(struct spu_gang
*gang
)
282 struct spu_context
*ctx
;
286 list_for_each_entry_reverse(ctx
, &gang
->aff_ref_ctx
->aff_list
,
288 if (&ctx
->aff_list
== &gang
->aff_list_head
)
290 ctx
->aff_offset
= offset
--;
294 list_for_each_entry(ctx
, gang
->aff_ref_ctx
->aff_list
.prev
, aff_list
) {
295 if (&ctx
->aff_list
== &gang
->aff_list_head
)
297 ctx
->aff_offset
= offset
++;
300 gang
->aff_flags
|= AFF_OFFSETS_SET
;
303 static struct spu
*aff_ref_location(struct spu_context
*ctx
, int mem_aff
,
304 int group_size
, int lowest_offset
)
310 * TODO: A better algorithm could be used to find a good spu to be
311 * used as reference location for the ctxs chain.
313 node
= cpu_to_node(raw_smp_processor_id());
314 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
317 node
= (node
< MAX_NUMNODES
) ? node
: 0;
318 if (!node_allowed(ctx
, node
))
322 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
323 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
324 if (spu
->ctx
&& spu
->ctx
->gang
325 && spu
->ctx
->aff_offset
== 0)
327 (spu
->ctx
->gang
->contexts
- 1);
331 if (available_spus
< ctx
->gang
->contexts
) {
332 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
336 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
337 if ((!mem_aff
|| spu
->has_mem_affinity
) &&
339 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
343 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
348 static void aff_set_ref_point_location(struct spu_gang
*gang
)
350 int mem_aff
, gs
, lowest_offset
;
351 struct spu_context
*ctx
;
354 mem_aff
= gang
->aff_ref_ctx
->flags
& SPU_CREATE_AFFINITY_MEM
;
358 list_for_each_entry(tmp
, &gang
->aff_list_head
, aff_list
)
361 list_for_each_entry_reverse(ctx
, &gang
->aff_ref_ctx
->aff_list
,
363 if (&ctx
->aff_list
== &gang
->aff_list_head
)
365 lowest_offset
= ctx
->aff_offset
;
368 gang
->aff_ref_spu
= aff_ref_location(gang
->aff_ref_ctx
, mem_aff
, gs
,
372 static struct spu
*ctx_location(struct spu
*ref
, int offset
, int node
)
378 list_for_each_entry(spu
, ref
->aff_list
.prev
, aff_list
) {
379 BUG_ON(spu
->node
!= node
);
386 list_for_each_entry_reverse(spu
, ref
->aff_list
.next
, aff_list
) {
387 BUG_ON(spu
->node
!= node
);
399 * affinity_check is called each time a context is going to be scheduled.
400 * It returns the spu ptr on which the context must run.
402 static int has_affinity(struct spu_context
*ctx
)
404 struct spu_gang
*gang
= ctx
->gang
;
406 if (list_empty(&ctx
->aff_list
))
409 if (atomic_read(&ctx
->gang
->aff_sched_count
) == 0)
410 ctx
->gang
->aff_ref_spu
= NULL
;
412 if (!gang
->aff_ref_spu
) {
413 if (!(gang
->aff_flags
& AFF_MERGED
))
414 aff_merge_remaining_ctxs(gang
);
415 if (!(gang
->aff_flags
& AFF_OFFSETS_SET
))
416 aff_set_offsets(gang
);
417 aff_set_ref_point_location(gang
);
420 return gang
->aff_ref_spu
!= NULL
;
424 * spu_unbind_context - unbind spu context from physical spu
425 * @spu: physical spu to unbind from
426 * @ctx: context to unbind
428 static void spu_unbind_context(struct spu
*spu
, struct spu_context
*ctx
)
432 spu_context_trace(spu_unbind_context__enter
, ctx
, spu
);
434 spuctx_switch_state(ctx
, SPU_UTIL_SYSTEM
);
436 if (spu
->ctx
->flags
& SPU_CREATE_NOSCHED
)
437 atomic_dec(&cbe_spu_info
[spu
->node
].reserved_spus
);
440 atomic_dec_if_positive(&ctx
->gang
->aff_sched_count
);
442 spu_switch_notify(spu
, NULL
);
443 spu_unmap_mappings(ctx
);
444 spu_save(&ctx
->csa
, spu
);
445 spu_switch_log_notify(spu
, ctx
, SWITCH_LOG_STOP
, 0);
447 spin_lock_irq(&spu
->register_lock
);
448 spu
->timestamp
= jiffies
;
449 ctx
->state
= SPU_STATE_SAVED
;
450 spu
->ibox_callback
= NULL
;
451 spu
->wbox_callback
= NULL
;
452 spu
->stop_callback
= NULL
;
453 spu
->mfc_callback
= NULL
;
456 ctx
->ops
= &spu_backing_ops
;
459 spin_unlock_irq(&spu
->register_lock
);
461 spu_associate_mm(spu
, NULL
);
463 ctx
->stats
.slb_flt
+=
464 (spu
->stats
.slb_flt
- ctx
->stats
.slb_flt_base
);
465 ctx
->stats
.class2_intr
+=
466 (spu
->stats
.class2_intr
- ctx
->stats
.class2_intr_base
);
468 /* This maps the underlying spu state to idle */
469 spuctx_switch_state(ctx
, SPU_UTIL_IDLE_LOADED
);
472 if (spu_stopped(ctx
, &status
))
473 wake_up_all(&ctx
->stop_wq
);
477 * spu_add_to_rq - add a context to the runqueue
478 * @ctx: context to add
480 static void __spu_add_to_rq(struct spu_context
*ctx
)
483 * Unfortunately this code path can be called from multiple threads
484 * on behalf of a single context due to the way the problem state
485 * mmap support works.
487 * Fortunately we need to wake up all these threads at the same time
488 * and can simply skip the runqueue addition for every but the first
489 * thread getting into this codepath.
491 * It's still quite hacky, and long-term we should proxy all other
492 * threads through the owner thread so that spu_run is in control
493 * of all the scheduling activity for a given context.
495 if (list_empty(&ctx
->rq
)) {
496 list_add_tail(&ctx
->rq
, &spu_prio
->runq
[ctx
->prio
]);
497 set_bit(ctx
->prio
, spu_prio
->bitmap
);
498 if (!spu_prio
->nr_waiting
++)
499 __mod_timer(&spusched_timer
, jiffies
+ SPUSCHED_TICK
);
503 static void spu_add_to_rq(struct spu_context
*ctx
)
505 spin_lock(&spu_prio
->runq_lock
);
506 __spu_add_to_rq(ctx
);
507 spin_unlock(&spu_prio
->runq_lock
);
510 static void __spu_del_from_rq(struct spu_context
*ctx
)
512 int prio
= ctx
->prio
;
514 if (!list_empty(&ctx
->rq
)) {
515 if (!--spu_prio
->nr_waiting
)
516 del_timer(&spusched_timer
);
517 list_del_init(&ctx
->rq
);
519 if (list_empty(&spu_prio
->runq
[prio
]))
520 clear_bit(prio
, spu_prio
->bitmap
);
524 void spu_del_from_rq(struct spu_context
*ctx
)
526 spin_lock(&spu_prio
->runq_lock
);
527 __spu_del_from_rq(ctx
);
528 spin_unlock(&spu_prio
->runq_lock
);
531 static void spu_prio_wait(struct spu_context
*ctx
)
536 * The caller must explicitly wait for a context to be loaded
537 * if the nosched flag is set. If NOSCHED is not set, the caller
538 * queues the context and waits for an spu event or error.
540 BUG_ON(!(ctx
->flags
& SPU_CREATE_NOSCHED
));
542 spin_lock(&spu_prio
->runq_lock
);
543 prepare_to_wait_exclusive(&ctx
->stop_wq
, &wait
, TASK_INTERRUPTIBLE
);
544 if (!signal_pending(current
)) {
545 __spu_add_to_rq(ctx
);
546 spin_unlock(&spu_prio
->runq_lock
);
547 mutex_unlock(&ctx
->state_mutex
);
549 mutex_lock(&ctx
->state_mutex
);
550 spin_lock(&spu_prio
->runq_lock
);
551 __spu_del_from_rq(ctx
);
553 spin_unlock(&spu_prio
->runq_lock
);
554 __set_current_state(TASK_RUNNING
);
555 remove_wait_queue(&ctx
->stop_wq
, &wait
);
558 static struct spu
*spu_get_idle(struct spu_context
*ctx
)
560 struct spu
*spu
, *aff_ref_spu
;
563 spu_context_nospu_trace(spu_get_idle__enter
, ctx
);
566 mutex_lock(&ctx
->gang
->aff_mutex
);
567 if (has_affinity(ctx
)) {
568 aff_ref_spu
= ctx
->gang
->aff_ref_spu
;
569 atomic_inc(&ctx
->gang
->aff_sched_count
);
570 mutex_unlock(&ctx
->gang
->aff_mutex
);
571 node
= aff_ref_spu
->node
;
573 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
574 spu
= ctx_location(aff_ref_spu
, ctx
->aff_offset
, node
);
575 if (spu
&& spu
->alloc_state
== SPU_FREE
)
577 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
579 atomic_dec(&ctx
->gang
->aff_sched_count
);
582 mutex_unlock(&ctx
->gang
->aff_mutex
);
584 node
= cpu_to_node(raw_smp_processor_id());
585 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
586 node
= (node
< MAX_NUMNODES
) ? node
: 0;
587 if (!node_allowed(ctx
, node
))
590 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
591 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
592 if (spu
->alloc_state
== SPU_FREE
)
595 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
599 spu_context_nospu_trace(spu_get_idle__not_found
, ctx
);
603 spu
->alloc_state
= SPU_USED
;
604 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
605 spu_context_trace(spu_get_idle__found
, ctx
, spu
);
606 spu_init_channels(spu
);
611 * find_victim - find a lower priority context to preempt
612 * @ctx: canidate context for running
614 * Returns the freed physical spu to run the new context on.
616 static struct spu
*find_victim(struct spu_context
*ctx
)
618 struct spu_context
*victim
= NULL
;
622 spu_context_nospu_trace(spu_find_victim__enter
, ctx
);
625 * Look for a possible preemption candidate on the local node first.
626 * If there is no candidate look at the other nodes. This isn't
627 * exactly fair, but so far the whole spu scheduler tries to keep
628 * a strong node affinity. We might want to fine-tune this in
632 node
= cpu_to_node(raw_smp_processor_id());
633 for (n
= 0; n
< MAX_NUMNODES
; n
++, node
++) {
634 node
= (node
< MAX_NUMNODES
) ? node
: 0;
635 if (!node_allowed(ctx
, node
))
638 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
639 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
) {
640 struct spu_context
*tmp
= spu
->ctx
;
642 if (tmp
&& tmp
->prio
> ctx
->prio
&&
643 !(tmp
->flags
& SPU_CREATE_NOSCHED
) &&
644 (!victim
|| tmp
->prio
> victim
->prio
)) {
649 get_spu_context(victim
);
650 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
654 * This nests ctx->state_mutex, but we always lock
655 * higher priority contexts before lower priority
656 * ones, so this is safe until we introduce
657 * priority inheritance schemes.
659 * XXX if the highest priority context is locked,
660 * this can loop a long time. Might be better to
661 * look at another context or give up after X retries.
663 if (!mutex_trylock(&victim
->state_mutex
)) {
664 put_spu_context(victim
);
670 if (!spu
|| victim
->prio
<= ctx
->prio
) {
672 * This race can happen because we've dropped
673 * the active list mutex. Not a problem, just
674 * restart the search.
676 mutex_unlock(&victim
->state_mutex
);
677 put_spu_context(victim
);
682 spu_context_trace(__spu_deactivate__unload
, ctx
, spu
);
684 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
685 cbe_spu_info
[node
].nr_active
--;
686 spu_unbind_context(spu
, victim
);
687 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
689 victim
->stats
.invol_ctx_switch
++;
690 spu
->stats
.invol_ctx_switch
++;
691 if (test_bit(SPU_SCHED_SPU_RUN
, &victim
->sched_flags
))
692 spu_add_to_rq(victim
);
694 mutex_unlock(&victim
->state_mutex
);
695 put_spu_context(victim
);
704 static void __spu_schedule(struct spu
*spu
, struct spu_context
*ctx
)
706 int node
= spu
->node
;
709 spu_set_timeslice(ctx
);
711 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
712 if (spu
->ctx
== NULL
) {
713 spu_bind_context(spu
, ctx
);
714 cbe_spu_info
[node
].nr_active
++;
715 spu
->alloc_state
= SPU_USED
;
718 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
721 wake_up_all(&ctx
->run_wq
);
726 static void spu_schedule(struct spu
*spu
, struct spu_context
*ctx
)
728 /* not a candidate for interruptible because it's called either
729 from the scheduler thread or from spu_deactivate */
730 mutex_lock(&ctx
->state_mutex
);
731 __spu_schedule(spu
, ctx
);
735 static void spu_unschedule(struct spu
*spu
, struct spu_context
*ctx
)
737 int node
= spu
->node
;
739 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
740 cbe_spu_info
[node
].nr_active
--;
741 spu
->alloc_state
= SPU_FREE
;
742 spu_unbind_context(spu
, ctx
);
743 ctx
->stats
.invol_ctx_switch
++;
744 spu
->stats
.invol_ctx_switch
++;
745 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);
749 * spu_activate - find a free spu for a context and execute it
750 * @ctx: spu context to schedule
751 * @flags: flags (currently ignored)
753 * Tries to find a free spu to run @ctx. If no free spu is available
754 * add the context to the runqueue so it gets woken up once an spu
757 int spu_activate(struct spu_context
*ctx
, unsigned long flags
)
762 * If there are multiple threads waiting for a single context
763 * only one actually binds the context while the others will
764 * only be able to acquire the state_mutex once the context
765 * already is in runnable state.
771 if (signal_pending(current
))
774 spu
= spu_get_idle(ctx
);
776 * If this is a realtime thread we try to get it running by
777 * preempting a lower priority thread.
779 if (!spu
&& rt_prio(ctx
->prio
))
780 spu
= find_victim(ctx
);
782 unsigned long runcntl
;
784 runcntl
= ctx
->ops
->runcntl_read(ctx
);
785 __spu_schedule(spu
, ctx
);
786 if (runcntl
& SPU_RUNCNTL_RUNNABLE
)
787 spuctx_switch_state(ctx
, SPU_UTIL_USER
);
792 if (ctx
->flags
& SPU_CREATE_NOSCHED
) {
794 goto spu_activate_top
;
803 * grab_runnable_context - try to find a runnable context
805 * Remove the highest priority context on the runqueue and return it
806 * to the caller. Returns %NULL if no runnable context was found.
808 static struct spu_context
*grab_runnable_context(int prio
, int node
)
810 struct spu_context
*ctx
;
813 spin_lock(&spu_prio
->runq_lock
);
814 best
= find_first_bit(spu_prio
->bitmap
, prio
);
815 while (best
< prio
) {
816 struct list_head
*rq
= &spu_prio
->runq
[best
];
818 list_for_each_entry(ctx
, rq
, rq
) {
819 /* XXX(hch): check for affinity here aswell */
820 if (__node_allowed(ctx
, node
)) {
821 __spu_del_from_rq(ctx
);
829 spin_unlock(&spu_prio
->runq_lock
);
833 static int __spu_deactivate(struct spu_context
*ctx
, int force
, int max_prio
)
835 struct spu
*spu
= ctx
->spu
;
836 struct spu_context
*new = NULL
;
839 new = grab_runnable_context(max_prio
, spu
->node
);
841 spu_unschedule(spu
, ctx
);
843 if (new->flags
& SPU_CREATE_NOSCHED
)
844 wake_up(&new->stop_wq
);
847 spu_schedule(spu
, new);
848 /* this one can't easily be made
850 mutex_lock(&ctx
->state_mutex
);
860 * spu_deactivate - unbind a context from it's physical spu
861 * @ctx: spu context to unbind
863 * Unbind @ctx from the physical spu it is running on and schedule
864 * the highest priority context to run on the freed physical spu.
866 void spu_deactivate(struct spu_context
*ctx
)
868 spu_context_nospu_trace(spu_deactivate__enter
, ctx
);
869 __spu_deactivate(ctx
, 1, MAX_PRIO
);
873 * spu_yield - yield a physical spu if others are waiting
874 * @ctx: spu context to yield
876 * Check if there is a higher priority context waiting and if yes
877 * unbind @ctx from the physical spu and schedule the highest
878 * priority context to run on the freed physical spu instead.
880 void spu_yield(struct spu_context
*ctx
)
882 spu_context_nospu_trace(spu_yield__enter
, ctx
);
883 if (!(ctx
->flags
& SPU_CREATE_NOSCHED
)) {
884 mutex_lock(&ctx
->state_mutex
);
885 __spu_deactivate(ctx
, 0, MAX_PRIO
);
886 mutex_unlock(&ctx
->state_mutex
);
890 static noinline
void spusched_tick(struct spu_context
*ctx
)
892 struct spu_context
*new = NULL
;
893 struct spu
*spu
= NULL
;
895 if (spu_acquire(ctx
))
896 BUG(); /* a kernel thread never has signals pending */
898 if (ctx
->state
!= SPU_STATE_RUNNABLE
)
900 if (ctx
->flags
& SPU_CREATE_NOSCHED
)
902 if (ctx
->policy
== SCHED_FIFO
)
905 if (--ctx
->time_slice
&& test_bit(SPU_SCHED_SPU_RUN
, &ctx
->sched_flags
))
910 spu_context_trace(spusched_tick__preempt
, ctx
, spu
);
912 new = grab_runnable_context(ctx
->prio
+ 1, spu
->node
);
914 spu_unschedule(spu
, ctx
);
915 if (test_bit(SPU_SCHED_SPU_RUN
, &ctx
->sched_flags
))
918 spu_context_nospu_trace(spusched_tick__newslice
, ctx
);
919 if (!ctx
->time_slice
)
926 spu_schedule(spu
, new);
930 * count_active_contexts - count nr of active tasks
932 * Return the number of tasks currently running or waiting to run.
934 * Note that we don't take runq_lock / list_mutex here. Reading
935 * a single 32bit value is atomic on powerpc, and we don't care
936 * about memory ordering issues here.
938 static unsigned long count_active_contexts(void)
940 int nr_active
= 0, node
;
942 for (node
= 0; node
< MAX_NUMNODES
; node
++)
943 nr_active
+= cbe_spu_info
[node
].nr_active
;
944 nr_active
+= spu_prio
->nr_waiting
;
950 * spu_calc_load - update the avenrun load estimates.
952 * No locking against reading these values from userspace, as for
953 * the CPU loadavg code.
955 static void spu_calc_load(void)
957 unsigned long active_tasks
; /* fixed-point */
959 active_tasks
= count_active_contexts() * FIXED_1
;
960 CALC_LOAD(spu_avenrun
[0], EXP_1
, active_tasks
);
961 CALC_LOAD(spu_avenrun
[1], EXP_5
, active_tasks
);
962 CALC_LOAD(spu_avenrun
[2], EXP_15
, active_tasks
);
965 static void spusched_wake(unsigned long data
)
967 mod_timer(&spusched_timer
, jiffies
+ SPUSCHED_TICK
);
968 wake_up_process(spusched_task
);
971 static void spuloadavg_wake(unsigned long data
)
973 mod_timer(&spuloadavg_timer
, jiffies
+ LOAD_FREQ
);
977 static int spusched_thread(void *unused
)
982 while (!kthread_should_stop()) {
983 set_current_state(TASK_INTERRUPTIBLE
);
985 for (node
= 0; node
< MAX_NUMNODES
; node
++) {
986 struct mutex
*mtx
= &cbe_spu_info
[node
].list_mutex
;
989 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
,
991 struct spu_context
*ctx
= spu
->ctx
;
994 get_spu_context(ctx
);
998 put_spu_context(ctx
);
1008 void spuctx_switch_state(struct spu_context
*ctx
,
1009 enum spu_utilization_state new_state
)
1011 unsigned long long curtime
;
1012 signed long long delta
;
1015 enum spu_utilization_state old_state
;
1019 curtime
= timespec_to_ns(&ts
);
1020 delta
= curtime
- ctx
->stats
.tstamp
;
1022 WARN_ON(!mutex_is_locked(&ctx
->state_mutex
));
1026 old_state
= ctx
->stats
.util_state
;
1027 ctx
->stats
.util_state
= new_state
;
1028 ctx
->stats
.tstamp
= curtime
;
1031 * Update the physical SPU utilization statistics.
1034 ctx
->stats
.times
[old_state
] += delta
;
1035 spu
->stats
.times
[old_state
] += delta
;
1036 spu
->stats
.util_state
= new_state
;
1037 spu
->stats
.tstamp
= curtime
;
1039 if (old_state
== SPU_UTIL_USER
)
1040 atomic_dec(&cbe_spu_info
[node
].busy_spus
);
1041 if (new_state
== SPU_UTIL_USER
)
1042 atomic_inc(&cbe_spu_info
[node
].busy_spus
);
1046 #define LOAD_INT(x) ((x) >> FSHIFT)
1047 #define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100)
1049 static int show_spu_loadavg(struct seq_file
*s
, void *private)
1053 a
= spu_avenrun
[0] + (FIXED_1
/200);
1054 b
= spu_avenrun
[1] + (FIXED_1
/200);
1055 c
= spu_avenrun
[2] + (FIXED_1
/200);
1058 * Note that last_pid doesn't really make much sense for the
1059 * SPU loadavg (it even seems very odd on the CPU side...),
1060 * but we include it here to have a 100% compatible interface.
1062 seq_printf(s
, "%d.%02d %d.%02d %d.%02d %ld/%d %d\n",
1063 LOAD_INT(a
), LOAD_FRAC(a
),
1064 LOAD_INT(b
), LOAD_FRAC(b
),
1065 LOAD_INT(c
), LOAD_FRAC(c
),
1066 count_active_contexts(),
1067 atomic_read(&nr_spu_contexts
),
1068 current
->nsproxy
->pid_ns
->last_pid
);
1072 static int spu_loadavg_open(struct inode
*inode
, struct file
*file
)
1074 return single_open(file
, show_spu_loadavg
, NULL
);
1077 static const struct file_operations spu_loadavg_fops
= {
1078 .open
= spu_loadavg_open
,
1080 .llseek
= seq_lseek
,
1081 .release
= single_release
,
1084 int __init
spu_sched_init(void)
1086 struct proc_dir_entry
*entry
;
1087 int err
= -ENOMEM
, i
;
1089 spu_prio
= kzalloc(sizeof(struct spu_prio_array
), GFP_KERNEL
);
1093 for (i
= 0; i
< MAX_PRIO
; i
++) {
1094 INIT_LIST_HEAD(&spu_prio
->runq
[i
]);
1095 __clear_bit(i
, spu_prio
->bitmap
);
1097 spin_lock_init(&spu_prio
->runq_lock
);
1099 setup_timer(&spusched_timer
, spusched_wake
, 0);
1100 setup_timer(&spuloadavg_timer
, spuloadavg_wake
, 0);
1102 spusched_task
= kthread_run(spusched_thread
, NULL
, "spusched");
1103 if (IS_ERR(spusched_task
)) {
1104 err
= PTR_ERR(spusched_task
);
1105 goto out_free_spu_prio
;
1108 mod_timer(&spuloadavg_timer
, 0);
1110 entry
= proc_create("spu_loadavg", 0, NULL
, &spu_loadavg_fops
);
1112 goto out_stop_kthread
;
1114 pr_debug("spusched: tick: %d, min ticks: %d, default ticks: %d\n",
1115 SPUSCHED_TICK
, MIN_SPU_TIMESLICE
, DEF_SPU_TIMESLICE
);
1119 kthread_stop(spusched_task
);
1126 void spu_sched_exit(void)
1131 remove_proc_entry("spu_loadavg", NULL
);
1133 del_timer_sync(&spusched_timer
);
1134 del_timer_sync(&spuloadavg_timer
);
1135 kthread_stop(spusched_task
);
1137 for (node
= 0; node
< MAX_NUMNODES
; node
++) {
1138 mutex_lock(&cbe_spu_info
[node
].list_mutex
);
1139 list_for_each_entry(spu
, &cbe_spu_info
[node
].spus
, cbe_list
)
1140 if (spu
->alloc_state
!= SPU_FREE
)
1141 spu
->alloc_state
= SPU_FREE
;
1142 mutex_unlock(&cbe_spu_info
[node
].list_mutex
);