1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Scheduler internal types and methods:
5 #include <linux/sched.h>
7 #include <linux/sched/autogroup.h>
8 #include <linux/sched/clock.h>
9 #include <linux/sched/coredump.h>
10 #include <linux/sched/cpufreq.h>
11 #include <linux/sched/cputime.h>
12 #include <linux/sched/deadline.h>
13 #include <linux/sched/debug.h>
14 #include <linux/sched/hotplug.h>
15 #include <linux/sched/idle.h>
16 #include <linux/sched/init.h>
17 #include <linux/sched/isolation.h>
18 #include <linux/sched/jobctl.h>
19 #include <linux/sched/loadavg.h>
20 #include <linux/sched/mm.h>
21 #include <linux/sched/nohz.h>
22 #include <linux/sched/numa_balancing.h>
23 #include <linux/sched/prio.h>
24 #include <linux/sched/rt.h>
25 #include <linux/sched/signal.h>
26 #include <linux/sched/smt.h>
27 #include <linux/sched/stat.h>
28 #include <linux/sched/sysctl.h>
29 #include <linux/sched/task.h>
30 #include <linux/sched/task_stack.h>
31 #include <linux/sched/topology.h>
32 #include <linux/sched/user.h>
33 #include <linux/sched/wake_q.h>
34 #include <linux/sched/xacct.h>
36 #include <uapi/linux/sched/types.h>
38 #include <linux/binfmts.h>
39 #include <linux/blkdev.h>
40 #include <linux/compat.h>
41 #include <linux/context_tracking.h>
42 #include <linux/cpufreq.h>
43 #include <linux/cpuidle.h>
44 #include <linux/cpuset.h>
45 #include <linux/ctype.h>
46 #include <linux/debugfs.h>
47 #include <linux/delayacct.h>
48 #include <linux/energy_model.h>
49 #include <linux/init_task.h>
50 #include <linux/kprobes.h>
51 #include <linux/kthread.h>
52 #include <linux/membarrier.h>
53 #include <linux/migrate.h>
54 #include <linux/mmu_context.h>
55 #include <linux/nmi.h>
56 #include <linux/proc_fs.h>
57 #include <linux/prefetch.h>
58 #include <linux/profile.h>
59 #include <linux/psi.h>
60 #include <linux/rcupdate_wait.h>
61 #include <linux/security.h>
62 #include <linux/stop_machine.h>
63 #include <linux/suspend.h>
64 #include <linux/swait.h>
65 #include <linux/syscalls.h>
66 #include <linux/task_work.h>
67 #include <linux/tsacct_kern.h>
71 #ifdef CONFIG_PARAVIRT
72 # include <asm/paravirt.h>
76 #include "cpudeadline.h"
78 #ifdef CONFIG_SCHED_DEBUG
79 # define SCHED_WARN_ON(x) WARN_ONCE(x, #x)
81 # define SCHED_WARN_ON(x) ({ (void)(x), 0; })
87 /* task_struct::on_rq states: */
88 #define TASK_ON_RQ_QUEUED 1
89 #define TASK_ON_RQ_MIGRATING 2
91 extern __read_mostly
int scheduler_running
;
93 extern unsigned long calc_load_update
;
94 extern atomic_long_t calc_load_tasks
;
96 extern void calc_global_load_tick(struct rq
*this_rq
);
97 extern long calc_load_fold_active(struct rq
*this_rq
, long adjust
);
100 * Helpers for converting nanosecond timing to jiffy resolution
102 #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
105 * Increase resolution of nice-level calculations for 64-bit architectures.
106 * The extra resolution improves shares distribution and load balancing of
107 * low-weight task groups (eg. nice +19 on an autogroup), deeper taskgroup
108 * hierarchies, especially on larger systems. This is not a user-visible change
109 * and does not change the user-interface for setting shares/weights.
111 * We increase resolution only if we have enough bits to allow this increased
112 * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit
113 * are pretty high and the returns do not justify the increased costs.
115 * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to
116 * increase coverage and consistency always enable it on 64-bit platforms.
119 # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)
120 # define scale_load(w) ((w) << SCHED_FIXEDPOINT_SHIFT)
121 # define scale_load_down(w) \
123 unsigned long __w = (w); \
125 __w = max(2UL, __w >> SCHED_FIXEDPOINT_SHIFT); \
129 # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT)
130 # define scale_load(w) (w)
131 # define scale_load_down(w) (w)
135 * Task weight (visible to users) and its load (invisible to users) have
136 * independent resolution, but they should be well calibrated. We use
137 * scale_load() and scale_load_down(w) to convert between them. The
138 * following must be true:
140 * scale_load(sched_prio_to_weight[USER_PRIO(NICE_TO_PRIO(0))]) == NICE_0_LOAD
143 #define NICE_0_LOAD (1L << NICE_0_LOAD_SHIFT)
146 * Single value that decides SCHED_DEADLINE internal math precision.
147 * 10 -> just above 1us
148 * 9 -> just above 0.5us
153 * Single value that denotes runtime == period, ie unlimited time.
155 #define RUNTIME_INF ((u64)~0ULL)
157 static inline int idle_policy(int policy
)
159 return policy
== SCHED_IDLE
;
161 static inline int fair_policy(int policy
)
163 return policy
== SCHED_NORMAL
|| policy
== SCHED_BATCH
;
166 static inline int rt_policy(int policy
)
168 return policy
== SCHED_FIFO
|| policy
== SCHED_RR
;
171 static inline int dl_policy(int policy
)
173 return policy
== SCHED_DEADLINE
;
175 static inline bool valid_policy(int policy
)
177 return idle_policy(policy
) || fair_policy(policy
) ||
178 rt_policy(policy
) || dl_policy(policy
);
181 static inline int task_has_idle_policy(struct task_struct
*p
)
183 return idle_policy(p
->policy
);
186 static inline int task_has_rt_policy(struct task_struct
*p
)
188 return rt_policy(p
->policy
);
191 static inline int task_has_dl_policy(struct task_struct
*p
)
193 return dl_policy(p
->policy
);
196 #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT)
198 static inline void update_avg(u64
*avg
, u64 sample
)
200 s64 diff
= sample
- *avg
;
205 * !! For sched_setattr_nocheck() (kernel) only !!
207 * This is actually gross. :(
209 * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE
210 * tasks, but still be able to sleep. We need this on platforms that cannot
211 * atomically change clock frequency. Remove once fast switching will be
212 * available on such platforms.
214 * SUGOV stands for SchedUtil GOVernor.
216 #define SCHED_FLAG_SUGOV 0x10000000
218 static inline bool dl_entity_is_special(struct sched_dl_entity
*dl_se
)
220 #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
221 return unlikely(dl_se
->flags
& SCHED_FLAG_SUGOV
);
228 * Tells if entity @a should preempt entity @b.
231 dl_entity_preempt(struct sched_dl_entity
*a
, struct sched_dl_entity
*b
)
233 return dl_entity_is_special(a
) ||
234 dl_time_before(a
->deadline
, b
->deadline
);
238 * This is the priority-queue data structure of the RT scheduling class:
240 struct rt_prio_array
{
241 DECLARE_BITMAP(bitmap
, MAX_RT_PRIO
+1); /* include 1 bit for delimiter */
242 struct list_head queue
[MAX_RT_PRIO
];
245 struct rt_bandwidth
{
246 /* nests inside the rq lock: */
247 raw_spinlock_t rt_runtime_lock
;
250 struct hrtimer rt_period_timer
;
251 unsigned int rt_period_active
;
254 void __dl_clear_params(struct task_struct
*p
);
257 * To keep the bandwidth of -deadline tasks and groups under control
258 * we need some place where:
259 * - store the maximum -deadline bandwidth of the system (the group);
260 * - cache the fraction of that bandwidth that is currently allocated.
262 * This is all done in the data structure below. It is similar to the
263 * one used for RT-throttling (rt_bandwidth), with the main difference
264 * that, since here we are only interested in admission control, we
265 * do not decrease any runtime while the group "executes", neither we
266 * need a timer to replenish it.
268 * With respect to SMP, the bandwidth is given on a per-CPU basis,
270 * - dl_bw (< 100%) is the bandwidth of the system (group) on each CPU;
271 * - dl_total_bw array contains, in the i-eth element, the currently
272 * allocated bandwidth on the i-eth CPU.
273 * Moreover, groups consume bandwidth on each CPU, while tasks only
274 * consume bandwidth on the CPU they're running on.
275 * Finally, dl_total_bw_cpu is used to cache the index of dl_total_bw
276 * that will be shown the next time the proc or cgroup controls will
277 * be red. It on its turn can be changed by writing on its own
280 struct dl_bandwidth
{
281 raw_spinlock_t dl_runtime_lock
;
286 static inline int dl_bandwidth_enabled(void)
288 return sysctl_sched_rt_runtime
>= 0;
297 static inline void __dl_update(struct dl_bw
*dl_b
, s64 bw
);
300 void __dl_sub(struct dl_bw
*dl_b
, u64 tsk_bw
, int cpus
)
302 dl_b
->total_bw
-= tsk_bw
;
303 __dl_update(dl_b
, (s32
)tsk_bw
/ cpus
);
307 void __dl_add(struct dl_bw
*dl_b
, u64 tsk_bw
, int cpus
)
309 dl_b
->total_bw
+= tsk_bw
;
310 __dl_update(dl_b
, -((s32
)tsk_bw
/ cpus
));
314 bool __dl_overflow(struct dl_bw
*dl_b
, int cpus
, u64 old_bw
, u64 new_bw
)
316 return dl_b
->bw
!= -1 &&
317 dl_b
->bw
* cpus
< dl_b
->total_bw
- old_bw
+ new_bw
;
320 extern void init_dl_bw(struct dl_bw
*dl_b
);
321 extern int sched_dl_global_validate(void);
322 extern void sched_dl_do_global(void);
323 extern int sched_dl_overflow(struct task_struct
*p
, int policy
, const struct sched_attr
*attr
);
324 extern void __setparam_dl(struct task_struct
*p
, const struct sched_attr
*attr
);
325 extern void __getparam_dl(struct task_struct
*p
, struct sched_attr
*attr
);
326 extern bool __checkparam_dl(const struct sched_attr
*attr
);
327 extern bool dl_param_changed(struct task_struct
*p
, const struct sched_attr
*attr
);
328 extern int dl_task_can_attach(struct task_struct
*p
, const struct cpumask
*cs_cpus_allowed
);
329 extern int dl_cpuset_cpumask_can_shrink(const struct cpumask
*cur
, const struct cpumask
*trial
);
330 extern bool dl_cpu_busy(unsigned int cpu
);
332 #ifdef CONFIG_CGROUP_SCHED
334 #include <linux/cgroup.h>
335 #include <linux/psi.h>
340 extern struct list_head task_groups
;
342 struct cfs_bandwidth
{
343 #ifdef CONFIG_CFS_BANDWIDTH
348 s64 hierarchical_quota
;
352 u8 distribute_running
;
354 struct hrtimer period_timer
;
355 struct hrtimer slack_timer
;
356 struct list_head throttled_cfs_rq
;
365 /* Task group related information */
367 struct cgroup_subsys_state css
;
369 #ifdef CONFIG_FAIR_GROUP_SCHED
370 /* schedulable entities of this group on each CPU */
371 struct sched_entity
**se
;
372 /* runqueue "owned" by this group on each CPU */
373 struct cfs_rq
**cfs_rq
;
374 unsigned long shares
;
378 * load_avg can be heavily contended at clock tick time, so put
379 * it in its own cacheline separated from the fields above which
380 * will also be accessed at each tick.
382 atomic_long_t load_avg ____cacheline_aligned
;
386 #ifdef CONFIG_RT_GROUP_SCHED
387 struct sched_rt_entity
**rt_se
;
388 struct rt_rq
**rt_rq
;
390 struct rt_bandwidth rt_bandwidth
;
394 struct list_head list
;
396 struct task_group
*parent
;
397 struct list_head siblings
;
398 struct list_head children
;
400 #ifdef CONFIG_SCHED_AUTOGROUP
401 struct autogroup
*autogroup
;
404 struct cfs_bandwidth cfs_bandwidth
;
406 #ifdef CONFIG_UCLAMP_TASK_GROUP
407 /* The two decimal precision [%] value requested from user-space */
408 unsigned int uclamp_pct
[UCLAMP_CNT
];
409 /* Clamp values requested for a task group */
410 struct uclamp_se uclamp_req
[UCLAMP_CNT
];
411 /* Effective clamp values used for a task group */
412 struct uclamp_se uclamp
[UCLAMP_CNT
];
417 #ifdef CONFIG_FAIR_GROUP_SCHED
418 #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD
421 * A weight of 0 or 1 can cause arithmetics problems.
422 * A weight of a cfs_rq is the sum of weights of which entities
423 * are queued on this cfs_rq, so a weight of a entity should not be
424 * too large, so as the shares value of a task group.
425 * (The default weight is 1024 - so there's no practical
426 * limitation from this.)
428 #define MIN_SHARES (1UL << 1)
429 #define MAX_SHARES (1UL << 18)
432 typedef int (*tg_visitor
)(struct task_group
*, void *);
434 extern int walk_tg_tree_from(struct task_group
*from
,
435 tg_visitor down
, tg_visitor up
, void *data
);
438 * Iterate the full tree, calling @down when first entering a node and @up when
439 * leaving it for the final time.
441 * Caller must hold rcu_lock or sufficient equivalent.
443 static inline int walk_tg_tree(tg_visitor down
, tg_visitor up
, void *data
)
445 return walk_tg_tree_from(&root_task_group
, down
, up
, data
);
448 extern int tg_nop(struct task_group
*tg
, void *data
);
450 extern void free_fair_sched_group(struct task_group
*tg
);
451 extern int alloc_fair_sched_group(struct task_group
*tg
, struct task_group
*parent
);
452 extern void online_fair_sched_group(struct task_group
*tg
);
453 extern void unregister_fair_sched_group(struct task_group
*tg
);
454 extern void init_tg_cfs_entry(struct task_group
*tg
, struct cfs_rq
*cfs_rq
,
455 struct sched_entity
*se
, int cpu
,
456 struct sched_entity
*parent
);
457 extern void init_cfs_bandwidth(struct cfs_bandwidth
*cfs_b
);
459 extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth
*cfs_b
);
460 extern void start_cfs_bandwidth(struct cfs_bandwidth
*cfs_b
);
461 extern void unthrottle_cfs_rq(struct cfs_rq
*cfs_rq
);
463 extern void free_rt_sched_group(struct task_group
*tg
);
464 extern int alloc_rt_sched_group(struct task_group
*tg
, struct task_group
*parent
);
465 extern void init_tg_rt_entry(struct task_group
*tg
, struct rt_rq
*rt_rq
,
466 struct sched_rt_entity
*rt_se
, int cpu
,
467 struct sched_rt_entity
*parent
);
468 extern int sched_group_set_rt_runtime(struct task_group
*tg
, long rt_runtime_us
);
469 extern int sched_group_set_rt_period(struct task_group
*tg
, u64 rt_period_us
);
470 extern long sched_group_rt_runtime(struct task_group
*tg
);
471 extern long sched_group_rt_period(struct task_group
*tg
);
472 extern int sched_rt_can_attach(struct task_group
*tg
, struct task_struct
*tsk
);
474 extern struct task_group
*sched_create_group(struct task_group
*parent
);
475 extern void sched_online_group(struct task_group
*tg
,
476 struct task_group
*parent
);
477 extern void sched_destroy_group(struct task_group
*tg
);
478 extern void sched_offline_group(struct task_group
*tg
);
480 extern void sched_move_task(struct task_struct
*tsk
);
482 #ifdef CONFIG_FAIR_GROUP_SCHED
483 extern int sched_group_set_shares(struct task_group
*tg
, unsigned long shares
);
486 extern void set_task_rq_fair(struct sched_entity
*se
,
487 struct cfs_rq
*prev
, struct cfs_rq
*next
);
488 #else /* !CONFIG_SMP */
489 static inline void set_task_rq_fair(struct sched_entity
*se
,
490 struct cfs_rq
*prev
, struct cfs_rq
*next
) { }
491 #endif /* CONFIG_SMP */
492 #endif /* CONFIG_FAIR_GROUP_SCHED */
494 #else /* CONFIG_CGROUP_SCHED */
496 struct cfs_bandwidth
{ };
498 #endif /* CONFIG_CGROUP_SCHED */
500 /* CFS-related fields in a runqueue */
502 struct load_weight load
;
503 unsigned int nr_running
;
504 unsigned int h_nr_running
; /* SCHED_{NORMAL,BATCH,IDLE} */
505 unsigned int idle_h_nr_running
; /* SCHED_IDLE */
510 u64 min_vruntime_copy
;
513 struct rb_root_cached tasks_timeline
;
516 * 'curr' points to currently running entity on this cfs_rq.
517 * It is set to NULL otherwise (i.e when none are currently running).
519 struct sched_entity
*curr
;
520 struct sched_entity
*next
;
521 struct sched_entity
*last
;
522 struct sched_entity
*skip
;
524 #ifdef CONFIG_SCHED_DEBUG
525 unsigned int nr_spread_over
;
532 struct sched_avg avg
;
534 u64 load_last_update_time_copy
;
537 raw_spinlock_t lock ____cacheline_aligned
;
539 unsigned long load_avg
;
540 unsigned long util_avg
;
541 unsigned long runnable_avg
;
544 #ifdef CONFIG_FAIR_GROUP_SCHED
545 unsigned long tg_load_avg_contrib
;
547 long prop_runnable_sum
;
550 * h_load = weight * f(tg)
552 * Where f(tg) is the recursive weight fraction assigned to
555 unsigned long h_load
;
556 u64 last_h_load_update
;
557 struct sched_entity
*h_load_next
;
558 #endif /* CONFIG_FAIR_GROUP_SCHED */
559 #endif /* CONFIG_SMP */
561 #ifdef CONFIG_FAIR_GROUP_SCHED
562 struct rq
*rq
; /* CPU runqueue to which this cfs_rq is attached */
565 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
566 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
567 * (like users, containers etc.)
569 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a CPU.
570 * This list is used during load balance.
573 struct list_head leaf_cfs_rq_list
;
574 struct task_group
*tg
; /* group that "owns" this runqueue */
576 #ifdef CONFIG_CFS_BANDWIDTH
578 s64 runtime_remaining
;
581 u64 throttled_clock_task
;
582 u64 throttled_clock_task_time
;
585 struct list_head throttled_list
;
586 #endif /* CONFIG_CFS_BANDWIDTH */
587 #endif /* CONFIG_FAIR_GROUP_SCHED */
590 static inline int rt_bandwidth_enabled(void)
592 return sysctl_sched_rt_runtime
>= 0;
595 /* RT IPI pull logic requires IRQ_WORK */
596 #if defined(CONFIG_IRQ_WORK) && defined(CONFIG_SMP)
597 # define HAVE_RT_PUSH_IPI
600 /* Real-Time classes' related field in a runqueue: */
602 struct rt_prio_array active
;
603 unsigned int rt_nr_running
;
604 unsigned int rr_nr_running
;
605 #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
607 int curr
; /* highest queued rt task prio */
609 int next
; /* next highest */
614 unsigned long rt_nr_migratory
;
615 unsigned long rt_nr_total
;
617 struct plist_head pushable_tasks
;
619 #endif /* CONFIG_SMP */
625 /* Nests inside the rq lock: */
626 raw_spinlock_t rt_runtime_lock
;
628 #ifdef CONFIG_RT_GROUP_SCHED
629 unsigned long rt_nr_boosted
;
632 struct task_group
*tg
;
636 static inline bool rt_rq_is_runnable(struct rt_rq
*rt_rq
)
638 return rt_rq
->rt_queued
&& rt_rq
->rt_nr_running
;
641 /* Deadline class' related fields in a runqueue */
643 /* runqueue is an rbtree, ordered by deadline */
644 struct rb_root_cached root
;
646 unsigned long dl_nr_running
;
650 * Deadline values of the currently executing and the
651 * earliest ready task on this rq. Caching these facilitates
652 * the decision whether or not a ready but not running task
653 * should migrate somewhere else.
660 unsigned long dl_nr_migratory
;
664 * Tasks on this rq that can be pushed away. They are kept in
665 * an rb-tree, ordered by tasks' deadlines, with caching
666 * of the leftmost (earliest deadline) element.
668 struct rb_root_cached pushable_dl_tasks_root
;
673 * "Active utilization" for this runqueue: increased when a
674 * task wakes up (becomes TASK_RUNNING) and decreased when a
680 * Utilization of the tasks "assigned" to this runqueue (including
681 * the tasks that are in runqueue and the tasks that executed on this
682 * CPU and blocked). Increased when a task moves to this runqueue, and
683 * decreased when the task moves away (migrates, changes scheduling
684 * policy, or terminates).
685 * This is needed to compute the "inactive utilization" for the
686 * runqueue (inactive utilization = this_bw - running_bw).
692 * Inverse of the fraction of CPU utilization that can be reclaimed
693 * by the GRUB algorithm.
698 #ifdef CONFIG_FAIR_GROUP_SCHED
699 /* An entity is a task if it doesn't "own" a runqueue */
700 #define entity_is_task(se) (!se->my_q)
702 static inline void se_update_runnable(struct sched_entity
*se
)
704 if (!entity_is_task(se
))
705 se
->runnable_weight
= se
->my_q
->h_nr_running
;
708 static inline long se_runnable(struct sched_entity
*se
)
710 if (entity_is_task(se
))
713 return se
->runnable_weight
;
717 #define entity_is_task(se) 1
719 static inline void se_update_runnable(struct sched_entity
*se
) {}
721 static inline long se_runnable(struct sched_entity
*se
)
729 * XXX we want to get rid of these helpers and use the full load resolution.
731 static inline long se_weight(struct sched_entity
*se
)
733 return scale_load_down(se
->load
.weight
);
737 static inline bool sched_asym_prefer(int a
, int b
)
739 return arch_asym_cpu_priority(a
) > arch_asym_cpu_priority(b
);
743 struct em_perf_domain
*em_pd
;
744 struct perf_domain
*next
;
748 /* Scheduling group status flags */
749 #define SG_OVERLOAD 0x1 /* More than one runnable task on a CPU. */
750 #define SG_OVERUTILIZED 0x2 /* One or more CPUs are over-utilized. */
753 * We add the notion of a root-domain which will be used to define per-domain
754 * variables. Each exclusive cpuset essentially defines an island domain by
755 * fully partitioning the member CPUs from any other cpuset. Whenever a new
756 * exclusive cpuset is created, we also create and attach a new root-domain
765 cpumask_var_t online
;
768 * Indicate pullable load on at least one CPU, e.g:
769 * - More than one runnable task
770 * - Running task is misfit
774 /* Indicate one or more cpus over-utilized (tipping point) */
778 * The bit corresponding to a CPU gets set here if such CPU has more
779 * than one runnable -deadline task (as it is below for RT tasks).
781 cpumask_var_t dlo_mask
;
786 #ifdef HAVE_RT_PUSH_IPI
788 * For IPI pull requests, loop across the rto_mask.
790 struct irq_work rto_push_work
;
791 raw_spinlock_t rto_lock
;
792 /* These are only updated and read within rto_lock */
795 /* These atomics are updated outside of a lock */
796 atomic_t rto_loop_next
;
797 atomic_t rto_loop_start
;
800 * The "RT overload" flag: it gets set if a CPU has more than
801 * one runnable RT task.
803 cpumask_var_t rto_mask
;
804 struct cpupri cpupri
;
806 unsigned long max_cpu_capacity
;
809 * NULL-terminated list of performance domains intersecting with the
810 * CPUs of the rd. Protected by RCU.
812 struct perf_domain __rcu
*pd
;
815 extern void init_defrootdomain(void);
816 extern int sched_init_domains(const struct cpumask
*cpu_map
);
817 extern void rq_attach_root(struct rq
*rq
, struct root_domain
*rd
);
818 extern void sched_get_rd(struct root_domain
*rd
);
819 extern void sched_put_rd(struct root_domain
*rd
);
821 #ifdef HAVE_RT_PUSH_IPI
822 extern void rto_push_irq_work_func(struct irq_work
*work
);
824 #endif /* CONFIG_SMP */
826 #ifdef CONFIG_UCLAMP_TASK
828 * struct uclamp_bucket - Utilization clamp bucket
829 * @value: utilization clamp value for tasks on this clamp bucket
830 * @tasks: number of RUNNABLE tasks on this clamp bucket
832 * Keep track of how many tasks are RUNNABLE for a given utilization
835 struct uclamp_bucket
{
836 unsigned long value
: bits_per(SCHED_CAPACITY_SCALE
);
837 unsigned long tasks
: BITS_PER_LONG
- bits_per(SCHED_CAPACITY_SCALE
);
841 * struct uclamp_rq - rq's utilization clamp
842 * @value: currently active clamp values for a rq
843 * @bucket: utilization clamp buckets affecting a rq
845 * Keep track of RUNNABLE tasks on a rq to aggregate their clamp values.
846 * A clamp value is affecting a rq when there is at least one task RUNNABLE
847 * (or actually running) with that value.
849 * There are up to UCLAMP_CNT possible different clamp values, currently there
850 * are only two: minimum utilization and maximum utilization.
852 * All utilization clamping values are MAX aggregated, since:
853 * - for util_min: we want to run the CPU at least at the max of the minimum
854 * utilization required by its currently RUNNABLE tasks.
855 * - for util_max: we want to allow the CPU to run up to the max of the
856 * maximum utilization allowed by its currently RUNNABLE tasks.
858 * Since on each system we expect only a limited number of different
859 * utilization clamp values (UCLAMP_BUCKETS), use a simple array to track
860 * the metrics required to compute all the per-rq utilization clamp values.
864 struct uclamp_bucket bucket
[UCLAMP_BUCKETS
];
866 #endif /* CONFIG_UCLAMP_TASK */
869 * This is the main, per-CPU runqueue data structure.
871 * Locking rule: those places that want to lock multiple runqueues
872 * (such as the load balancing or the thread migration code), lock
873 * acquire operations must be ordered by ascending &runqueue.
880 * nr_running and cpu_load should be in the same cacheline because
881 * remote CPUs use both these fields when doing load calculation.
883 unsigned int nr_running
;
884 #ifdef CONFIG_NUMA_BALANCING
885 unsigned int nr_numa_running
;
886 unsigned int nr_preferred_running
;
887 unsigned int numa_migrate_on
;
889 #ifdef CONFIG_NO_HZ_COMMON
891 unsigned long last_load_update_tick
;
892 unsigned long last_blocked_load_update_tick
;
893 unsigned int has_blocked_load
;
894 #endif /* CONFIG_SMP */
895 unsigned int nohz_tick_stopped
;
897 #endif /* CONFIG_NO_HZ_COMMON */
899 unsigned long nr_load_updates
;
902 #ifdef CONFIG_UCLAMP_TASK
903 /* Utilization clamp values based on CPU's RUNNABLE tasks */
904 struct uclamp_rq uclamp
[UCLAMP_CNT
] ____cacheline_aligned
;
905 unsigned int uclamp_flags
;
906 #define UCLAMP_FLAG_IDLE 0x01
913 #ifdef CONFIG_FAIR_GROUP_SCHED
914 /* list of leaf cfs_rq on this CPU: */
915 struct list_head leaf_cfs_rq_list
;
916 struct list_head
*tmp_alone_branch
;
917 #endif /* CONFIG_FAIR_GROUP_SCHED */
920 * This is part of a global counter where only the total sum
921 * over all CPUs matters. A task can increase this counter on
922 * one CPU and if it got migrated afterwards it may decrease
923 * it on another CPU. Always updated under the runqueue lock:
925 unsigned long nr_uninterruptible
;
927 struct task_struct __rcu
*curr
;
928 struct task_struct
*idle
;
929 struct task_struct
*stop
;
930 unsigned long next_balance
;
931 struct mm_struct
*prev_mm
;
933 unsigned int clock_update_flags
;
935 /* Ensure that all clocks are in the same cache line */
936 u64 clock_task ____cacheline_aligned
;
938 unsigned long lost_idle_time
;
942 #ifdef CONFIG_MEMBARRIER
943 int membarrier_state
;
947 struct root_domain
*rd
;
948 struct sched_domain __rcu
*sd
;
950 unsigned long cpu_capacity
;
951 unsigned long cpu_capacity_orig
;
953 struct callback_head
*balance_callback
;
955 unsigned char idle_balance
;
957 unsigned long misfit_task_load
;
959 /* For active balancing */
962 struct cpu_stop_work active_balance_work
;
964 /* CPU of this runqueue: */
968 struct list_head cfs_tasks
;
970 struct sched_avg avg_rt
;
971 struct sched_avg avg_dl
;
972 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
973 struct sched_avg avg_irq
;
975 #ifdef CONFIG_SCHED_THERMAL_PRESSURE
976 struct sched_avg avg_thermal
;
981 /* This is used to determine avg_idle's max value */
982 u64 max_idle_balance_cost
;
985 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
988 #ifdef CONFIG_PARAVIRT
991 #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
992 u64 prev_steal_time_rq
;
995 /* calc_load related fields */
996 unsigned long calc_load_update
;
997 long calc_load_active
;
999 #ifdef CONFIG_SCHED_HRTICK
1001 call_single_data_t hrtick_csd
;
1003 struct hrtimer hrtick_timer
;
1006 #ifdef CONFIG_SCHEDSTATS
1008 struct sched_info rq_sched_info
;
1009 unsigned long long rq_cpu_time
;
1010 /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
1012 /* sys_sched_yield() stats */
1013 unsigned int yld_count
;
1015 /* schedule() stats */
1016 unsigned int sched_count
;
1017 unsigned int sched_goidle
;
1019 /* try_to_wake_up() stats */
1020 unsigned int ttwu_count
;
1021 unsigned int ttwu_local
;
1025 struct llist_head wake_list
;
1028 #ifdef CONFIG_CPU_IDLE
1029 /* Must be inspected within a rcu lock section */
1030 struct cpuidle_state
*idle_state
;
1034 #ifdef CONFIG_FAIR_GROUP_SCHED
1036 /* CPU runqueue to which this cfs_rq is attached */
1037 static inline struct rq
*rq_of(struct cfs_rq
*cfs_rq
)
1044 static inline struct rq
*rq_of(struct cfs_rq
*cfs_rq
)
1046 return container_of(cfs_rq
, struct rq
, cfs
);
1050 static inline int cpu_of(struct rq
*rq
)
1060 #ifdef CONFIG_SCHED_SMT
1061 extern void __update_idle_core(struct rq
*rq
);
1063 static inline void update_idle_core(struct rq
*rq
)
1065 if (static_branch_unlikely(&sched_smt_present
))
1066 __update_idle_core(rq
);
1070 static inline void update_idle_core(struct rq
*rq
) { }
1073 DECLARE_PER_CPU_SHARED_ALIGNED(struct rq
, runqueues
);
1075 #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
1076 #define this_rq() this_cpu_ptr(&runqueues)
1077 #define task_rq(p) cpu_rq(task_cpu(p))
1078 #define cpu_curr(cpu) (cpu_rq(cpu)->curr)
1079 #define raw_rq() raw_cpu_ptr(&runqueues)
1081 extern void update_rq_clock(struct rq
*rq
);
1083 static inline u64
__rq_clock_broken(struct rq
*rq
)
1085 return READ_ONCE(rq
->clock
);
1089 * rq::clock_update_flags bits
1091 * %RQCF_REQ_SKIP - will request skipping of clock update on the next
1092 * call to __schedule(). This is an optimisation to avoid
1093 * neighbouring rq clock updates.
1095 * %RQCF_ACT_SKIP - is set from inside of __schedule() when skipping is
1096 * in effect and calls to update_rq_clock() are being ignored.
1098 * %RQCF_UPDATED - is a debug flag that indicates whether a call has been
1099 * made to update_rq_clock() since the last time rq::lock was pinned.
1101 * If inside of __schedule(), clock_update_flags will have been
1102 * shifted left (a left shift is a cheap operation for the fast path
1103 * to promote %RQCF_REQ_SKIP to %RQCF_ACT_SKIP), so you must use,
1105 * if (rq-clock_update_flags >= RQCF_UPDATED)
1107 * to check if %RQCF_UPADTED is set. It'll never be shifted more than
1108 * one position though, because the next rq_unpin_lock() will shift it
1111 #define RQCF_REQ_SKIP 0x01
1112 #define RQCF_ACT_SKIP 0x02
1113 #define RQCF_UPDATED 0x04
1115 static inline void assert_clock_updated(struct rq
*rq
)
1118 * The only reason for not seeing a clock update since the
1119 * last rq_pin_lock() is if we're currently skipping updates.
1121 SCHED_WARN_ON(rq
->clock_update_flags
< RQCF_ACT_SKIP
);
1124 static inline u64
rq_clock(struct rq
*rq
)
1126 lockdep_assert_held(&rq
->lock
);
1127 assert_clock_updated(rq
);
1132 static inline u64
rq_clock_task(struct rq
*rq
)
1134 lockdep_assert_held(&rq
->lock
);
1135 assert_clock_updated(rq
);
1137 return rq
->clock_task
;
1141 * By default the decay is the default pelt decay period.
1142 * The decay shift can change the decay period in
1144 * Decay shift Decay period(ms)
1151 extern int sched_thermal_decay_shift
;
1153 static inline u64
rq_clock_thermal(struct rq
*rq
)
1155 return rq_clock_task(rq
) >> sched_thermal_decay_shift
;
1158 static inline void rq_clock_skip_update(struct rq
*rq
)
1160 lockdep_assert_held(&rq
->lock
);
1161 rq
->clock_update_flags
|= RQCF_REQ_SKIP
;
1165 * See rt task throttling, which is the only time a skip
1166 * request is cancelled.
1168 static inline void rq_clock_cancel_skipupdate(struct rq
*rq
)
1170 lockdep_assert_held(&rq
->lock
);
1171 rq
->clock_update_flags
&= ~RQCF_REQ_SKIP
;
1175 unsigned long flags
;
1176 struct pin_cookie cookie
;
1177 #ifdef CONFIG_SCHED_DEBUG
1179 * A copy of (rq::clock_update_flags & RQCF_UPDATED) for the
1180 * current pin context is stashed here in case it needs to be
1181 * restored in rq_repin_lock().
1183 unsigned int clock_update_flags
;
1187 static inline void rq_pin_lock(struct rq
*rq
, struct rq_flags
*rf
)
1189 rf
->cookie
= lockdep_pin_lock(&rq
->lock
);
1191 #ifdef CONFIG_SCHED_DEBUG
1192 rq
->clock_update_flags
&= (RQCF_REQ_SKIP
|RQCF_ACT_SKIP
);
1193 rf
->clock_update_flags
= 0;
1197 static inline void rq_unpin_lock(struct rq
*rq
, struct rq_flags
*rf
)
1199 #ifdef CONFIG_SCHED_DEBUG
1200 if (rq
->clock_update_flags
> RQCF_ACT_SKIP
)
1201 rf
->clock_update_flags
= RQCF_UPDATED
;
1204 lockdep_unpin_lock(&rq
->lock
, rf
->cookie
);
1207 static inline void rq_repin_lock(struct rq
*rq
, struct rq_flags
*rf
)
1209 lockdep_repin_lock(&rq
->lock
, rf
->cookie
);
1211 #ifdef CONFIG_SCHED_DEBUG
1213 * Restore the value we stashed in @rf for this pin context.
1215 rq
->clock_update_flags
|= rf
->clock_update_flags
;
1219 struct rq
*__task_rq_lock(struct task_struct
*p
, struct rq_flags
*rf
)
1220 __acquires(rq
->lock
);
1222 struct rq
*task_rq_lock(struct task_struct
*p
, struct rq_flags
*rf
)
1223 __acquires(p
->pi_lock
)
1224 __acquires(rq
->lock
);
1226 static inline void __task_rq_unlock(struct rq
*rq
, struct rq_flags
*rf
)
1227 __releases(rq
->lock
)
1229 rq_unpin_lock(rq
, rf
);
1230 raw_spin_unlock(&rq
->lock
);
1234 task_rq_unlock(struct rq
*rq
, struct task_struct
*p
, struct rq_flags
*rf
)
1235 __releases(rq
->lock
)
1236 __releases(p
->pi_lock
)
1238 rq_unpin_lock(rq
, rf
);
1239 raw_spin_unlock(&rq
->lock
);
1240 raw_spin_unlock_irqrestore(&p
->pi_lock
, rf
->flags
);
1244 rq_lock_irqsave(struct rq
*rq
, struct rq_flags
*rf
)
1245 __acquires(rq
->lock
)
1247 raw_spin_lock_irqsave(&rq
->lock
, rf
->flags
);
1248 rq_pin_lock(rq
, rf
);
1252 rq_lock_irq(struct rq
*rq
, struct rq_flags
*rf
)
1253 __acquires(rq
->lock
)
1255 raw_spin_lock_irq(&rq
->lock
);
1256 rq_pin_lock(rq
, rf
);
1260 rq_lock(struct rq
*rq
, struct rq_flags
*rf
)
1261 __acquires(rq
->lock
)
1263 raw_spin_lock(&rq
->lock
);
1264 rq_pin_lock(rq
, rf
);
1268 rq_relock(struct rq
*rq
, struct rq_flags
*rf
)
1269 __acquires(rq
->lock
)
1271 raw_spin_lock(&rq
->lock
);
1272 rq_repin_lock(rq
, rf
);
1276 rq_unlock_irqrestore(struct rq
*rq
, struct rq_flags
*rf
)
1277 __releases(rq
->lock
)
1279 rq_unpin_lock(rq
, rf
);
1280 raw_spin_unlock_irqrestore(&rq
->lock
, rf
->flags
);
1284 rq_unlock_irq(struct rq
*rq
, struct rq_flags
*rf
)
1285 __releases(rq
->lock
)
1287 rq_unpin_lock(rq
, rf
);
1288 raw_spin_unlock_irq(&rq
->lock
);
1292 rq_unlock(struct rq
*rq
, struct rq_flags
*rf
)
1293 __releases(rq
->lock
)
1295 rq_unpin_lock(rq
, rf
);
1296 raw_spin_unlock(&rq
->lock
);
1299 static inline struct rq
*
1300 this_rq_lock_irq(struct rq_flags
*rf
)
1301 __acquires(rq
->lock
)
1305 local_irq_disable();
1312 enum numa_topology_type
{
1317 extern enum numa_topology_type sched_numa_topology_type
;
1318 extern int sched_max_numa_distance
;
1319 extern bool find_numa_distance(int distance
);
1320 extern void sched_init_numa(void);
1321 extern void sched_domains_numa_masks_set(unsigned int cpu
);
1322 extern void sched_domains_numa_masks_clear(unsigned int cpu
);
1323 extern int sched_numa_find_closest(const struct cpumask
*cpus
, int cpu
);
1325 static inline void sched_init_numa(void) { }
1326 static inline void sched_domains_numa_masks_set(unsigned int cpu
) { }
1327 static inline void sched_domains_numa_masks_clear(unsigned int cpu
) { }
1328 static inline int sched_numa_find_closest(const struct cpumask
*cpus
, int cpu
)
1334 #ifdef CONFIG_NUMA_BALANCING
1335 /* The regions in numa_faults array from task_struct */
1336 enum numa_faults_stats
{
1342 extern void sched_setnuma(struct task_struct
*p
, int node
);
1343 extern int migrate_task_to(struct task_struct
*p
, int cpu
);
1344 extern int migrate_swap(struct task_struct
*p
, struct task_struct
*t
,
1346 extern void init_numa_balancing(unsigned long clone_flags
, struct task_struct
*p
);
1349 init_numa_balancing(unsigned long clone_flags
, struct task_struct
*p
)
1352 #endif /* CONFIG_NUMA_BALANCING */
1357 queue_balance_callback(struct rq
*rq
,
1358 struct callback_head
*head
,
1359 void (*func
)(struct rq
*rq
))
1361 lockdep_assert_held(&rq
->lock
);
1363 if (unlikely(head
->next
))
1366 head
->func
= (void (*)(struct callback_head
*))func
;
1367 head
->next
= rq
->balance_callback
;
1368 rq
->balance_callback
= head
;
1371 extern void sched_ttwu_pending(void);
1373 #define rcu_dereference_check_sched_domain(p) \
1374 rcu_dereference_check((p), \
1375 lockdep_is_held(&sched_domains_mutex))
1378 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
1379 * See destroy_sched_domains: call_rcu for details.
1381 * The domain tree of any CPU may only be accessed from within
1382 * preempt-disabled sections.
1384 #define for_each_domain(cpu, __sd) \
1385 for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); \
1386 __sd; __sd = __sd->parent)
1389 * highest_flag_domain - Return highest sched_domain containing flag.
1390 * @cpu: The CPU whose highest level of sched domain is to
1392 * @flag: The flag to check for the highest sched_domain
1393 * for the given CPU.
1395 * Returns the highest sched_domain of a CPU which contains the given flag.
1397 static inline struct sched_domain
*highest_flag_domain(int cpu
, int flag
)
1399 struct sched_domain
*sd
, *hsd
= NULL
;
1401 for_each_domain(cpu
, sd
) {
1402 if (!(sd
->flags
& flag
))
1410 static inline struct sched_domain
*lowest_flag_domain(int cpu
, int flag
)
1412 struct sched_domain
*sd
;
1414 for_each_domain(cpu
, sd
) {
1415 if (sd
->flags
& flag
)
1422 DECLARE_PER_CPU(struct sched_domain __rcu
*, sd_llc
);
1423 DECLARE_PER_CPU(int, sd_llc_size
);
1424 DECLARE_PER_CPU(int, sd_llc_id
);
1425 DECLARE_PER_CPU(struct sched_domain_shared __rcu
*, sd_llc_shared
);
1426 DECLARE_PER_CPU(struct sched_domain __rcu
*, sd_numa
);
1427 DECLARE_PER_CPU(struct sched_domain __rcu
*, sd_asym_packing
);
1428 DECLARE_PER_CPU(struct sched_domain __rcu
*, sd_asym_cpucapacity
);
1429 extern struct static_key_false sched_asym_cpucapacity
;
1431 struct sched_group_capacity
{
1434 * CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity
1437 unsigned long capacity
;
1438 unsigned long min_capacity
; /* Min per-CPU capacity in group */
1439 unsigned long max_capacity
; /* Max per-CPU capacity in group */
1440 unsigned long next_update
;
1441 int imbalance
; /* XXX unrelated to capacity but shared group state */
1443 #ifdef CONFIG_SCHED_DEBUG
1447 unsigned long cpumask
[0]; /* Balance mask */
1450 struct sched_group
{
1451 struct sched_group
*next
; /* Must be a circular list */
1454 unsigned int group_weight
;
1455 struct sched_group_capacity
*sgc
;
1456 int asym_prefer_cpu
; /* CPU of highest priority in group */
1459 * The CPUs this group covers.
1461 * NOTE: this field is variable length. (Allocated dynamically
1462 * by attaching extra space to the end of the structure,
1463 * depending on how many CPUs the kernel has booted up with)
1465 unsigned long cpumask
[0];
1468 static inline struct cpumask
*sched_group_span(struct sched_group
*sg
)
1470 return to_cpumask(sg
->cpumask
);
1474 * See build_balance_mask().
1476 static inline struct cpumask
*group_balance_mask(struct sched_group
*sg
)
1478 return to_cpumask(sg
->sgc
->cpumask
);
1482 * group_first_cpu - Returns the first CPU in the cpumask of a sched_group.
1483 * @group: The group whose first CPU is to be returned.
1485 static inline unsigned int group_first_cpu(struct sched_group
*group
)
1487 return cpumask_first(sched_group_span(group
));
1490 extern int group_balance_cpu(struct sched_group
*sg
);
1492 #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
1493 void register_sched_domain_sysctl(void);
1494 void dirty_sched_domain_sysctl(int cpu
);
1495 void unregister_sched_domain_sysctl(void);
1497 static inline void register_sched_domain_sysctl(void)
1500 static inline void dirty_sched_domain_sysctl(int cpu
)
1503 static inline void unregister_sched_domain_sysctl(void)
1508 extern int newidle_balance(struct rq
*this_rq
, struct rq_flags
*rf
);
1512 static inline void sched_ttwu_pending(void) { }
1514 static inline int newidle_balance(struct rq
*this_rq
, struct rq_flags
*rf
) { return 0; }
1516 #endif /* CONFIG_SMP */
1519 #include "autogroup.h"
1521 #ifdef CONFIG_CGROUP_SCHED
1524 * Return the group to which this tasks belongs.
1526 * We cannot use task_css() and friends because the cgroup subsystem
1527 * changes that value before the cgroup_subsys::attach() method is called,
1528 * therefore we cannot pin it and might observe the wrong value.
1530 * The same is true for autogroup's p->signal->autogroup->tg, the autogroup
1531 * core changes this before calling sched_move_task().
1533 * Instead we use a 'copy' which is updated from sched_move_task() while
1534 * holding both task_struct::pi_lock and rq::lock.
1536 static inline struct task_group
*task_group(struct task_struct
*p
)
1538 return p
->sched_task_group
;
1541 /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
1542 static inline void set_task_rq(struct task_struct
*p
, unsigned int cpu
)
1544 #if defined(CONFIG_FAIR_GROUP_SCHED) || defined(CONFIG_RT_GROUP_SCHED)
1545 struct task_group
*tg
= task_group(p
);
1548 #ifdef CONFIG_FAIR_GROUP_SCHED
1549 set_task_rq_fair(&p
->se
, p
->se
.cfs_rq
, tg
->cfs_rq
[cpu
]);
1550 p
->se
.cfs_rq
= tg
->cfs_rq
[cpu
];
1551 p
->se
.parent
= tg
->se
[cpu
];
1554 #ifdef CONFIG_RT_GROUP_SCHED
1555 p
->rt
.rt_rq
= tg
->rt_rq
[cpu
];
1556 p
->rt
.parent
= tg
->rt_se
[cpu
];
1560 #else /* CONFIG_CGROUP_SCHED */
1562 static inline void set_task_rq(struct task_struct
*p
, unsigned int cpu
) { }
1563 static inline struct task_group
*task_group(struct task_struct
*p
)
1568 #endif /* CONFIG_CGROUP_SCHED */
1570 static inline void __set_task_cpu(struct task_struct
*p
, unsigned int cpu
)
1572 set_task_rq(p
, cpu
);
1575 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
1576 * successfully executed on another CPU. We must ensure that updates of
1577 * per-task data have been completed by this moment.
1580 #ifdef CONFIG_THREAD_INFO_IN_TASK
1581 WRITE_ONCE(p
->cpu
, cpu
);
1583 WRITE_ONCE(task_thread_info(p
)->cpu
, cpu
);
1590 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
1592 #ifdef CONFIG_SCHED_DEBUG
1593 # include <linux/static_key.h>
1594 # define const_debug __read_mostly
1596 # define const_debug const
1599 #define SCHED_FEAT(name, enabled) \
1600 __SCHED_FEAT_##name ,
1603 #include "features.h"
1609 #if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_JUMP_LABEL)
1612 * To support run-time toggling of sched features, all the translation units
1613 * (but core.c) reference the sysctl_sched_features defined in core.c.
1615 extern const_debug
unsigned int sysctl_sched_features
;
1617 #define SCHED_FEAT(name, enabled) \
1618 static __always_inline bool static_branch_##name(struct static_key *key) \
1620 return static_key_##enabled(key); \
1623 #include "features.h"
1626 extern struct static_key sched_feat_keys
[__SCHED_FEAT_NR
];
1627 #define sched_feat(x) (static_branch_##x(&sched_feat_keys[__SCHED_FEAT_##x]))
1629 #else /* !(SCHED_DEBUG && CONFIG_JUMP_LABEL) */
1632 * Each translation unit has its own copy of sysctl_sched_features to allow
1633 * constants propagation at compile time and compiler optimization based on
1636 #define SCHED_FEAT(name, enabled) \
1637 (1UL << __SCHED_FEAT_##name) * enabled |
1638 static const_debug __maybe_unused
unsigned int sysctl_sched_features
=
1639 #include "features.h"
1643 #define sched_feat(x) !!(sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
1645 #endif /* SCHED_DEBUG && CONFIG_JUMP_LABEL */
1647 extern struct static_key_false sched_numa_balancing
;
1648 extern struct static_key_false sched_schedstats
;
1650 static inline u64
global_rt_period(void)
1652 return (u64
)sysctl_sched_rt_period
* NSEC_PER_USEC
;
1655 static inline u64
global_rt_runtime(void)
1657 if (sysctl_sched_rt_runtime
< 0)
1660 return (u64
)sysctl_sched_rt_runtime
* NSEC_PER_USEC
;
1663 static inline int task_current(struct rq
*rq
, struct task_struct
*p
)
1665 return rq
->curr
== p
;
1668 static inline int task_running(struct rq
*rq
, struct task_struct
*p
)
1673 return task_current(rq
, p
);
1677 static inline int task_on_rq_queued(struct task_struct
*p
)
1679 return p
->on_rq
== TASK_ON_RQ_QUEUED
;
1682 static inline int task_on_rq_migrating(struct task_struct
*p
)
1684 return READ_ONCE(p
->on_rq
) == TASK_ON_RQ_MIGRATING
;
1690 #define WF_SYNC 0x01 /* Waker goes to sleep after wakeup */
1691 #define WF_FORK 0x02 /* Child wakeup after fork */
1692 #define WF_MIGRATED 0x4 /* Internal use, task got migrated */
1695 * To aid in avoiding the subversion of "niceness" due to uneven distribution
1696 * of tasks with abnormal "nice" values across CPUs the contribution that
1697 * each task makes to its run queue's load is weighted according to its
1698 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
1699 * scaled version of the new time slice allocation that they receive on time
1703 #define WEIGHT_IDLEPRIO 3
1704 #define WMULT_IDLEPRIO 1431655765
1706 extern const int sched_prio_to_weight
[40];
1707 extern const u32 sched_prio_to_wmult
[40];
1710 * {de,en}queue flags:
1712 * DEQUEUE_SLEEP - task is no longer runnable
1713 * ENQUEUE_WAKEUP - task just became runnable
1715 * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks
1716 * are in a known state which allows modification. Such pairs
1717 * should preserve as much state as possible.
1719 * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location
1722 * ENQUEUE_HEAD - place at front of runqueue (tail if not specified)
1723 * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline)
1724 * ENQUEUE_MIGRATED - the task was migrated during wakeup
1728 #define DEQUEUE_SLEEP 0x01
1729 #define DEQUEUE_SAVE 0x02 /* Matches ENQUEUE_RESTORE */
1730 #define DEQUEUE_MOVE 0x04 /* Matches ENQUEUE_MOVE */
1731 #define DEQUEUE_NOCLOCK 0x08 /* Matches ENQUEUE_NOCLOCK */
1733 #define ENQUEUE_WAKEUP 0x01
1734 #define ENQUEUE_RESTORE 0x02
1735 #define ENQUEUE_MOVE 0x04
1736 #define ENQUEUE_NOCLOCK 0x08
1738 #define ENQUEUE_HEAD 0x10
1739 #define ENQUEUE_REPLENISH 0x20
1741 #define ENQUEUE_MIGRATED 0x40
1743 #define ENQUEUE_MIGRATED 0x00
1746 #define RETRY_TASK ((void *)-1UL)
1748 struct sched_class
{
1749 const struct sched_class
*next
;
1751 #ifdef CONFIG_UCLAMP_TASK
1755 void (*enqueue_task
) (struct rq
*rq
, struct task_struct
*p
, int flags
);
1756 void (*dequeue_task
) (struct rq
*rq
, struct task_struct
*p
, int flags
);
1757 void (*yield_task
) (struct rq
*rq
);
1758 bool (*yield_to_task
)(struct rq
*rq
, struct task_struct
*p
, bool preempt
);
1760 void (*check_preempt_curr
)(struct rq
*rq
, struct task_struct
*p
, int flags
);
1762 struct task_struct
*(*pick_next_task
)(struct rq
*rq
);
1764 void (*put_prev_task
)(struct rq
*rq
, struct task_struct
*p
);
1765 void (*set_next_task
)(struct rq
*rq
, struct task_struct
*p
, bool first
);
1768 int (*balance
)(struct rq
*rq
, struct task_struct
*prev
, struct rq_flags
*rf
);
1769 int (*select_task_rq
)(struct task_struct
*p
, int task_cpu
, int sd_flag
, int flags
);
1770 void (*migrate_task_rq
)(struct task_struct
*p
, int new_cpu
);
1772 void (*task_woken
)(struct rq
*this_rq
, struct task_struct
*task
);
1774 void (*set_cpus_allowed
)(struct task_struct
*p
,
1775 const struct cpumask
*newmask
);
1777 void (*rq_online
)(struct rq
*rq
);
1778 void (*rq_offline
)(struct rq
*rq
);
1781 void (*task_tick
)(struct rq
*rq
, struct task_struct
*p
, int queued
);
1782 void (*task_fork
)(struct task_struct
*p
);
1783 void (*task_dead
)(struct task_struct
*p
);
1786 * The switched_from() call is allowed to drop rq->lock, therefore we
1787 * cannot assume the switched_from/switched_to pair is serliazed by
1788 * rq->lock. They are however serialized by p->pi_lock.
1790 void (*switched_from
)(struct rq
*this_rq
, struct task_struct
*task
);
1791 void (*switched_to
) (struct rq
*this_rq
, struct task_struct
*task
);
1792 void (*prio_changed
) (struct rq
*this_rq
, struct task_struct
*task
,
1795 unsigned int (*get_rr_interval
)(struct rq
*rq
,
1796 struct task_struct
*task
);
1798 void (*update_curr
)(struct rq
*rq
);
1800 #define TASK_SET_GROUP 0
1801 #define TASK_MOVE_GROUP 1
1803 #ifdef CONFIG_FAIR_GROUP_SCHED
1804 void (*task_change_group
)(struct task_struct
*p
, int type
);
1808 static inline void put_prev_task(struct rq
*rq
, struct task_struct
*prev
)
1810 WARN_ON_ONCE(rq
->curr
!= prev
);
1811 prev
->sched_class
->put_prev_task(rq
, prev
);
1814 static inline void set_next_task(struct rq
*rq
, struct task_struct
*next
)
1816 WARN_ON_ONCE(rq
->curr
!= next
);
1817 next
->sched_class
->set_next_task(rq
, next
, false);
1821 #define sched_class_highest (&stop_sched_class)
1823 #define sched_class_highest (&dl_sched_class)
1826 #define for_class_range(class, _from, _to) \
1827 for (class = (_from); class != (_to); class = class->next)
1829 #define for_each_class(class) \
1830 for_class_range(class, sched_class_highest, NULL)
1832 extern const struct sched_class stop_sched_class
;
1833 extern const struct sched_class dl_sched_class
;
1834 extern const struct sched_class rt_sched_class
;
1835 extern const struct sched_class fair_sched_class
;
1836 extern const struct sched_class idle_sched_class
;
1838 static inline bool sched_stop_runnable(struct rq
*rq
)
1840 return rq
->stop
&& task_on_rq_queued(rq
->stop
);
1843 static inline bool sched_dl_runnable(struct rq
*rq
)
1845 return rq
->dl
.dl_nr_running
> 0;
1848 static inline bool sched_rt_runnable(struct rq
*rq
)
1850 return rq
->rt
.rt_queued
> 0;
1853 static inline bool sched_fair_runnable(struct rq
*rq
)
1855 return rq
->cfs
.nr_running
> 0;
1858 extern struct task_struct
*pick_next_task_fair(struct rq
*rq
, struct task_struct
*prev
, struct rq_flags
*rf
);
1859 extern struct task_struct
*pick_next_task_idle(struct rq
*rq
);
1863 extern void update_group_capacity(struct sched_domain
*sd
, int cpu
);
1865 extern void trigger_load_balance(struct rq
*rq
);
1867 extern void set_cpus_allowed_common(struct task_struct
*p
, const struct cpumask
*new_mask
);
1871 #ifdef CONFIG_CPU_IDLE
1872 static inline void idle_set_state(struct rq
*rq
,
1873 struct cpuidle_state
*idle_state
)
1875 rq
->idle_state
= idle_state
;
1878 static inline struct cpuidle_state
*idle_get_state(struct rq
*rq
)
1880 SCHED_WARN_ON(!rcu_read_lock_held());
1882 return rq
->idle_state
;
1885 static inline void idle_set_state(struct rq
*rq
,
1886 struct cpuidle_state
*idle_state
)
1890 static inline struct cpuidle_state
*idle_get_state(struct rq
*rq
)
1896 extern void schedule_idle(void);
1898 extern void sysrq_sched_debug_show(void);
1899 extern void sched_init_granularity(void);
1900 extern void update_max_interval(void);
1902 extern void init_sched_dl_class(void);
1903 extern void init_sched_rt_class(void);
1904 extern void init_sched_fair_class(void);
1906 extern void reweight_task(struct task_struct
*p
, int prio
);
1908 extern void resched_curr(struct rq
*rq
);
1909 extern void resched_cpu(int cpu
);
1911 extern struct rt_bandwidth def_rt_bandwidth
;
1912 extern void init_rt_bandwidth(struct rt_bandwidth
*rt_b
, u64 period
, u64 runtime
);
1914 extern struct dl_bandwidth def_dl_bandwidth
;
1915 extern void init_dl_bandwidth(struct dl_bandwidth
*dl_b
, u64 period
, u64 runtime
);
1916 extern void init_dl_task_timer(struct sched_dl_entity
*dl_se
);
1917 extern void init_dl_inactive_task_timer(struct sched_dl_entity
*dl_se
);
1920 #define BW_UNIT (1 << BW_SHIFT)
1921 #define RATIO_SHIFT 8
1922 unsigned long to_ratio(u64 period
, u64 runtime
);
1924 extern void init_entity_runnable_average(struct sched_entity
*se
);
1925 extern void post_init_entity_util_avg(struct task_struct
*p
);
1927 #ifdef CONFIG_NO_HZ_FULL
1928 extern bool sched_can_stop_tick(struct rq
*rq
);
1929 extern int __init
sched_tick_offload_init(void);
1932 * Tick may be needed by tasks in the runqueue depending on their policy and
1933 * requirements. If tick is needed, lets send the target an IPI to kick it out of
1934 * nohz mode if necessary.
1936 static inline void sched_update_tick_dependency(struct rq
*rq
)
1940 if (!tick_nohz_full_enabled())
1945 if (!tick_nohz_full_cpu(cpu
))
1948 if (sched_can_stop_tick(rq
))
1949 tick_nohz_dep_clear_cpu(cpu
, TICK_DEP_BIT_SCHED
);
1951 tick_nohz_dep_set_cpu(cpu
, TICK_DEP_BIT_SCHED
);
1954 static inline int sched_tick_offload_init(void) { return 0; }
1955 static inline void sched_update_tick_dependency(struct rq
*rq
) { }
1958 static inline void add_nr_running(struct rq
*rq
, unsigned count
)
1960 unsigned prev_nr
= rq
->nr_running
;
1962 rq
->nr_running
= prev_nr
+ count
;
1965 if (prev_nr
< 2 && rq
->nr_running
>= 2) {
1966 if (!READ_ONCE(rq
->rd
->overload
))
1967 WRITE_ONCE(rq
->rd
->overload
, 1);
1971 sched_update_tick_dependency(rq
);
1974 static inline void sub_nr_running(struct rq
*rq
, unsigned count
)
1976 rq
->nr_running
-= count
;
1977 /* Check if we still need preemption */
1978 sched_update_tick_dependency(rq
);
1981 extern void activate_task(struct rq
*rq
, struct task_struct
*p
, int flags
);
1982 extern void deactivate_task(struct rq
*rq
, struct task_struct
*p
, int flags
);
1984 extern void check_preempt_curr(struct rq
*rq
, struct task_struct
*p
, int flags
);
1986 extern const_debug
unsigned int sysctl_sched_nr_migrate
;
1987 extern const_debug
unsigned int sysctl_sched_migration_cost
;
1989 #ifdef CONFIG_SCHED_HRTICK
1993 * - enabled by features
1994 * - hrtimer is actually high res
1996 static inline int hrtick_enabled(struct rq
*rq
)
1998 if (!sched_feat(HRTICK
))
2000 if (!cpu_active(cpu_of(rq
)))
2002 return hrtimer_is_hres_active(&rq
->hrtick_timer
);
2005 void hrtick_start(struct rq
*rq
, u64 delay
);
2009 static inline int hrtick_enabled(struct rq
*rq
)
2014 #endif /* CONFIG_SCHED_HRTICK */
2016 #ifndef arch_scale_freq_tick
2017 static __always_inline
2018 void arch_scale_freq_tick(void)
2023 #ifndef arch_scale_freq_capacity
2024 static __always_inline
2025 unsigned long arch_scale_freq_capacity(int cpu
)
2027 return SCHED_CAPACITY_SCALE
;
2032 #ifdef CONFIG_PREEMPTION
2034 static inline void double_rq_lock(struct rq
*rq1
, struct rq
*rq2
);
2037 * fair double_lock_balance: Safely acquires both rq->locks in a fair
2038 * way at the expense of forcing extra atomic operations in all
2039 * invocations. This assures that the double_lock is acquired using the
2040 * same underlying policy as the spinlock_t on this architecture, which
2041 * reduces latency compared to the unfair variant below. However, it
2042 * also adds more overhead and therefore may reduce throughput.
2044 static inline int _double_lock_balance(struct rq
*this_rq
, struct rq
*busiest
)
2045 __releases(this_rq
->lock
)
2046 __acquires(busiest
->lock
)
2047 __acquires(this_rq
->lock
)
2049 raw_spin_unlock(&this_rq
->lock
);
2050 double_rq_lock(this_rq
, busiest
);
2057 * Unfair double_lock_balance: Optimizes throughput at the expense of
2058 * latency by eliminating extra atomic operations when the locks are
2059 * already in proper order on entry. This favors lower CPU-ids and will
2060 * grant the double lock to lower CPUs over higher ids under contention,
2061 * regardless of entry order into the function.
2063 static inline int _double_lock_balance(struct rq
*this_rq
, struct rq
*busiest
)
2064 __releases(this_rq
->lock
)
2065 __acquires(busiest
->lock
)
2066 __acquires(this_rq
->lock
)
2070 if (unlikely(!raw_spin_trylock(&busiest
->lock
))) {
2071 if (busiest
< this_rq
) {
2072 raw_spin_unlock(&this_rq
->lock
);
2073 raw_spin_lock(&busiest
->lock
);
2074 raw_spin_lock_nested(&this_rq
->lock
,
2075 SINGLE_DEPTH_NESTING
);
2078 raw_spin_lock_nested(&busiest
->lock
,
2079 SINGLE_DEPTH_NESTING
);
2084 #endif /* CONFIG_PREEMPTION */
2087 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
2089 static inline int double_lock_balance(struct rq
*this_rq
, struct rq
*busiest
)
2091 if (unlikely(!irqs_disabled())) {
2092 /* printk() doesn't work well under rq->lock */
2093 raw_spin_unlock(&this_rq
->lock
);
2097 return _double_lock_balance(this_rq
, busiest
);
2100 static inline void double_unlock_balance(struct rq
*this_rq
, struct rq
*busiest
)
2101 __releases(busiest
->lock
)
2103 raw_spin_unlock(&busiest
->lock
);
2104 lock_set_subclass(&this_rq
->lock
.dep_map
, 0, _RET_IP_
);
2107 static inline void double_lock(spinlock_t
*l1
, spinlock_t
*l2
)
2113 spin_lock_nested(l2
, SINGLE_DEPTH_NESTING
);
2116 static inline void double_lock_irq(spinlock_t
*l1
, spinlock_t
*l2
)
2122 spin_lock_nested(l2
, SINGLE_DEPTH_NESTING
);
2125 static inline void double_raw_lock(raw_spinlock_t
*l1
, raw_spinlock_t
*l2
)
2131 raw_spin_lock_nested(l2
, SINGLE_DEPTH_NESTING
);
2135 * double_rq_lock - safely lock two runqueues
2137 * Note this does not disable interrupts like task_rq_lock,
2138 * you need to do so manually before calling.
2140 static inline void double_rq_lock(struct rq
*rq1
, struct rq
*rq2
)
2141 __acquires(rq1
->lock
)
2142 __acquires(rq2
->lock
)
2144 BUG_ON(!irqs_disabled());
2146 raw_spin_lock(&rq1
->lock
);
2147 __acquire(rq2
->lock
); /* Fake it out ;) */
2150 raw_spin_lock(&rq1
->lock
);
2151 raw_spin_lock_nested(&rq2
->lock
, SINGLE_DEPTH_NESTING
);
2153 raw_spin_lock(&rq2
->lock
);
2154 raw_spin_lock_nested(&rq1
->lock
, SINGLE_DEPTH_NESTING
);
2160 * double_rq_unlock - safely unlock two runqueues
2162 * Note this does not restore interrupts like task_rq_unlock,
2163 * you need to do so manually after calling.
2165 static inline void double_rq_unlock(struct rq
*rq1
, struct rq
*rq2
)
2166 __releases(rq1
->lock
)
2167 __releases(rq2
->lock
)
2169 raw_spin_unlock(&rq1
->lock
);
2171 raw_spin_unlock(&rq2
->lock
);
2173 __release(rq2
->lock
);
2176 extern void set_rq_online (struct rq
*rq
);
2177 extern void set_rq_offline(struct rq
*rq
);
2178 extern bool sched_smp_initialized
;
2180 #else /* CONFIG_SMP */
2183 * double_rq_lock - safely lock two runqueues
2185 * Note this does not disable interrupts like task_rq_lock,
2186 * you need to do so manually before calling.
2188 static inline void double_rq_lock(struct rq
*rq1
, struct rq
*rq2
)
2189 __acquires(rq1
->lock
)
2190 __acquires(rq2
->lock
)
2192 BUG_ON(!irqs_disabled());
2194 raw_spin_lock(&rq1
->lock
);
2195 __acquire(rq2
->lock
); /* Fake it out ;) */
2199 * double_rq_unlock - safely unlock two runqueues
2201 * Note this does not restore interrupts like task_rq_unlock,
2202 * you need to do so manually after calling.
2204 static inline void double_rq_unlock(struct rq
*rq1
, struct rq
*rq2
)
2205 __releases(rq1
->lock
)
2206 __releases(rq2
->lock
)
2209 raw_spin_unlock(&rq1
->lock
);
2210 __release(rq2
->lock
);
2215 extern struct sched_entity
*__pick_first_entity(struct cfs_rq
*cfs_rq
);
2216 extern struct sched_entity
*__pick_last_entity(struct cfs_rq
*cfs_rq
);
2218 #ifdef CONFIG_SCHED_DEBUG
2219 extern bool sched_debug_enabled
;
2221 extern void print_cfs_stats(struct seq_file
*m
, int cpu
);
2222 extern void print_rt_stats(struct seq_file
*m
, int cpu
);
2223 extern void print_dl_stats(struct seq_file
*m
, int cpu
);
2224 extern void print_cfs_rq(struct seq_file
*m
, int cpu
, struct cfs_rq
*cfs_rq
);
2225 extern void print_rt_rq(struct seq_file
*m
, int cpu
, struct rt_rq
*rt_rq
);
2226 extern void print_dl_rq(struct seq_file
*m
, int cpu
, struct dl_rq
*dl_rq
);
2227 #ifdef CONFIG_NUMA_BALANCING
2229 show_numa_stats(struct task_struct
*p
, struct seq_file
*m
);
2231 print_numa_stats(struct seq_file
*m
, int node
, unsigned long tsf
,
2232 unsigned long tpf
, unsigned long gsf
, unsigned long gpf
);
2233 #endif /* CONFIG_NUMA_BALANCING */
2234 #endif /* CONFIG_SCHED_DEBUG */
2236 extern void init_cfs_rq(struct cfs_rq
*cfs_rq
);
2237 extern void init_rt_rq(struct rt_rq
*rt_rq
);
2238 extern void init_dl_rq(struct dl_rq
*dl_rq
);
2240 extern void cfs_bandwidth_usage_inc(void);
2241 extern void cfs_bandwidth_usage_dec(void);
2243 #ifdef CONFIG_NO_HZ_COMMON
2244 #define NOHZ_BALANCE_KICK_BIT 0
2245 #define NOHZ_STATS_KICK_BIT 1
2247 #define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT)
2248 #define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT)
2250 #define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
2252 #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
2254 extern void nohz_balance_exit_idle(struct rq
*rq
);
2256 static inline void nohz_balance_exit_idle(struct rq
*rq
) { }
2262 void __dl_update(struct dl_bw
*dl_b
, s64 bw
)
2264 struct root_domain
*rd
= container_of(dl_b
, struct root_domain
, dl_bw
);
2267 RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
2268 "sched RCU must be held");
2269 for_each_cpu_and(i
, rd
->span
, cpu_active_mask
) {
2270 struct rq
*rq
= cpu_rq(i
);
2272 rq
->dl
.extra_bw
+= bw
;
2277 void __dl_update(struct dl_bw
*dl_b
, s64 bw
)
2279 struct dl_rq
*dl
= container_of(dl_b
, struct dl_rq
, dl_bw
);
2286 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
2291 struct u64_stats_sync sync
;
2294 DECLARE_PER_CPU(struct irqtime
, cpu_irqtime
);
2297 * Returns the irqtime minus the softirq time computed by ksoftirqd.
2298 * Otherwise ksoftirqd's sum_exec_runtime is substracted its own runtime
2299 * and never move forward.
2301 static inline u64
irq_time_read(int cpu
)
2303 struct irqtime
*irqtime
= &per_cpu(cpu_irqtime
, cpu
);
2308 seq
= __u64_stats_fetch_begin(&irqtime
->sync
);
2309 total
= irqtime
->total
;
2310 } while (__u64_stats_fetch_retry(&irqtime
->sync
, seq
));
2314 #endif /* CONFIG_IRQ_TIME_ACCOUNTING */
2316 #ifdef CONFIG_CPU_FREQ
2317 DECLARE_PER_CPU(struct update_util_data __rcu
*, cpufreq_update_util_data
);
2320 * cpufreq_update_util - Take a note about CPU utilization changes.
2321 * @rq: Runqueue to carry out the update for.
2322 * @flags: Update reason flags.
2324 * This function is called by the scheduler on the CPU whose utilization is
2327 * It can only be called from RCU-sched read-side critical sections.
2329 * The way cpufreq is currently arranged requires it to evaluate the CPU
2330 * performance state (frequency/voltage) on a regular basis to prevent it from
2331 * being stuck in a completely inadequate performance level for too long.
2332 * That is not guaranteed to happen if the updates are only triggered from CFS
2333 * and DL, though, because they may not be coming in if only RT tasks are
2334 * active all the time (or there are RT tasks only).
2336 * As a workaround for that issue, this function is called periodically by the
2337 * RT sched class to trigger extra cpufreq updates to prevent it from stalling,
2338 * but that really is a band-aid. Going forward it should be replaced with
2339 * solutions targeted more specifically at RT tasks.
2341 static inline void cpufreq_update_util(struct rq
*rq
, unsigned int flags
)
2343 struct update_util_data
*data
;
2345 data
= rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data
,
2348 data
->func(data
, rq_clock(rq
), flags
);
2351 static inline void cpufreq_update_util(struct rq
*rq
, unsigned int flags
) {}
2352 #endif /* CONFIG_CPU_FREQ */
2354 #ifdef CONFIG_UCLAMP_TASK
2355 unsigned long uclamp_eff_value(struct task_struct
*p
, enum uclamp_id clamp_id
);
2357 static __always_inline
2358 unsigned long uclamp_rq_util_with(struct rq
*rq
, unsigned long util
,
2359 struct task_struct
*p
)
2361 unsigned long min_util
= READ_ONCE(rq
->uclamp
[UCLAMP_MIN
].value
);
2362 unsigned long max_util
= READ_ONCE(rq
->uclamp
[UCLAMP_MAX
].value
);
2365 min_util
= max(min_util
, uclamp_eff_value(p
, UCLAMP_MIN
));
2366 max_util
= max(max_util
, uclamp_eff_value(p
, UCLAMP_MAX
));
2370 * Since CPU's {min,max}_util clamps are MAX aggregated considering
2371 * RUNNABLE tasks with _different_ clamps, we can end up with an
2372 * inversion. Fix it now when the clamps are applied.
2374 if (unlikely(min_util
>= max_util
))
2377 return clamp(util
, min_util
, max_util
);
2379 #else /* CONFIG_UCLAMP_TASK */
2381 unsigned long uclamp_rq_util_with(struct rq
*rq
, unsigned long util
,
2382 struct task_struct
*p
)
2386 #endif /* CONFIG_UCLAMP_TASK */
2388 #ifdef arch_scale_freq_capacity
2389 # ifndef arch_scale_freq_invariant
2390 # define arch_scale_freq_invariant() true
2393 # define arch_scale_freq_invariant() false
2397 static inline unsigned long capacity_orig_of(int cpu
)
2399 return cpu_rq(cpu
)->cpu_capacity_orig
;
2404 * enum schedutil_type - CPU utilization type
2405 * @FREQUENCY_UTIL: Utilization used to select frequency
2406 * @ENERGY_UTIL: Utilization used during energy calculation
2408 * The utilization signals of all scheduling classes (CFS/RT/DL) and IRQ time
2409 * need to be aggregated differently depending on the usage made of them. This
2410 * enum is used within schedutil_freq_util() to differentiate the types of
2411 * utilization expected by the callers, and adjust the aggregation accordingly.
2413 enum schedutil_type
{
2418 #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
2420 unsigned long schedutil_cpu_util(int cpu
, unsigned long util_cfs
,
2421 unsigned long max
, enum schedutil_type type
,
2422 struct task_struct
*p
);
2424 static inline unsigned long cpu_bw_dl(struct rq
*rq
)
2426 return (rq
->dl
.running_bw
* SCHED_CAPACITY_SCALE
) >> BW_SHIFT
;
2429 static inline unsigned long cpu_util_dl(struct rq
*rq
)
2431 return READ_ONCE(rq
->avg_dl
.util_avg
);
2434 static inline unsigned long cpu_util_cfs(struct rq
*rq
)
2436 unsigned long util
= READ_ONCE(rq
->cfs
.avg
.util_avg
);
2438 if (sched_feat(UTIL_EST
)) {
2439 util
= max_t(unsigned long, util
,
2440 READ_ONCE(rq
->cfs
.avg
.util_est
.enqueued
));
2446 static inline unsigned long cpu_util_rt(struct rq
*rq
)
2448 return READ_ONCE(rq
->avg_rt
.util_avg
);
2450 #else /* CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
2451 static inline unsigned long schedutil_cpu_util(int cpu
, unsigned long util_cfs
,
2452 unsigned long max
, enum schedutil_type type
,
2453 struct task_struct
*p
)
2457 #endif /* CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
2459 #ifdef CONFIG_HAVE_SCHED_AVG_IRQ
2460 static inline unsigned long cpu_util_irq(struct rq
*rq
)
2462 return rq
->avg_irq
.util_avg
;
2466 unsigned long scale_irq_capacity(unsigned long util
, unsigned long irq
, unsigned long max
)
2468 util
*= (max
- irq
);
2475 static inline unsigned long cpu_util_irq(struct rq
*rq
)
2481 unsigned long scale_irq_capacity(unsigned long util
, unsigned long irq
, unsigned long max
)
2487 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
2489 #define perf_domain_span(pd) (to_cpumask(((pd)->em_pd->cpus)))
2491 DECLARE_STATIC_KEY_FALSE(sched_energy_present
);
2493 static inline bool sched_energy_enabled(void)
2495 return static_branch_unlikely(&sched_energy_present
);
2498 #else /* ! (CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */
2500 #define perf_domain_span(pd) NULL
2501 static inline bool sched_energy_enabled(void) { return false; }
2503 #endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
2505 #ifdef CONFIG_MEMBARRIER
2507 * The scheduler provides memory barriers required by membarrier between:
2508 * - prior user-space memory accesses and store to rq->membarrier_state,
2509 * - store to rq->membarrier_state and following user-space memory accesses.
2510 * In the same way it provides those guarantees around store to rq->curr.
2512 static inline void membarrier_switch_mm(struct rq
*rq
,
2513 struct mm_struct
*prev_mm
,
2514 struct mm_struct
*next_mm
)
2516 int membarrier_state
;
2518 if (prev_mm
== next_mm
)
2521 membarrier_state
= atomic_read(&next_mm
->membarrier_state
);
2522 if (READ_ONCE(rq
->membarrier_state
) == membarrier_state
)
2525 WRITE_ONCE(rq
->membarrier_state
, membarrier_state
);
2528 static inline void membarrier_switch_mm(struct rq
*rq
,
2529 struct mm_struct
*prev_mm
,
2530 struct mm_struct
*next_mm
)
2536 static inline bool is_per_cpu_kthread(struct task_struct
*p
)
2538 if (!(p
->flags
& PF_KTHREAD
))
2541 if (p
->nr_cpus_allowed
!= 1)
2548 void swake_up_all_locked(struct swait_queue_head
*q
);
2549 void __prepare_to_swait(struct swait_queue_head
*q
, struct swait_queue
*wait
);