kernel/sched/stats.h

   1
   2 #ifdef CONFIG_SCHEDSTATS
   3
   4 /*
   5  * Expects runqueue lock to be held for atomicity of update
   6  */
   7 static inline void
   8 rq_sched_info_arrive(struct rq *rq, unsigned long long delta)
   9 {
  10         if (rq) {
  11                 rq->rq_sched_info.run_delay += delta;
  12                 rq->rq_sched_info.pcount++;
  13         }
  14 }
  15
  16 /*
  17  * Expects runqueue lock to be held for atomicity of update
  18  */
  19 static inline void
  20 rq_sched_info_depart(struct rq *rq, unsigned long long delta)
  21 {
  22         if (rq)
  23                 rq->rq_cpu_time += delta;
  24 }
  25
  26 static inline void
  27 rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
  28 {
  29         if (rq)
  30                 rq->rq_sched_info.run_delay += delta;
  31 }
  32 #define schedstat_enabled()             static_branch_unlikely(&sched_schedstats)
  33 #define schedstat_inc(var)              do { if (schedstat_enabled()) { var++; } } while (0)
  34 #define schedstat_add(var, amt)         do { if (schedstat_enabled()) { var += (amt); } } while (0)
  35 #define schedstat_set(var, val)         do { if (schedstat_enabled()) { var = (val); } } while (0)
  36 #define schedstat_val(var)              (var)
  37 #define schedstat_val_or_zero(var)      ((schedstat_enabled()) ? (var) : 0)
  38
  39 #else /* !CONFIG_SCHEDSTATS */
  40 static inline void
  41 rq_sched_info_arrive(struct rq *rq, unsigned long long delta)
  42 {}
  43 static inline void
  44 rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
  45 {}
  46 static inline void
  47 rq_sched_info_depart(struct rq *rq, unsigned long long delta)
  48 {}
  49 #define schedstat_enabled()             0
  50 #define schedstat_inc(var)              do { } while (0)
  51 #define schedstat_add(var, amt)         do { } while (0)
  52 #define schedstat_set(var, val)         do { } while (0)
  53 #define schedstat_val(var)              0
  54 #define schedstat_val_or_zero(var)      0
  55 #endif /* CONFIG_SCHEDSTATS */
  56
  57 #ifdef CONFIG_SCHED_INFO
  58 static inline void sched_info_reset_dequeued(struct task_struct *t)
  59 {
  60         t->sched_info.last_queued = 0;
  61 }
  62
  63 /*
  64  * We are interested in knowing how long it was from the *first* time a
  65  * task was queued to the time that it finally hit a cpu, we call this routine
  66  * from dequeue_task() to account for possible rq->clock skew across cpus. The
  67  * delta taken on each cpu would annul the skew.
  68  */
  69 static inline void sched_info_dequeued(struct rq *rq, struct task_struct *t)
  70 {
  71         unsigned long long now = rq_clock(rq), delta = 0;
  72
  73         if (unlikely(sched_info_on()))
  74                 if (t->sched_info.last_queued)
  75                         delta = now - t->sched_info.last_queued;
  76         sched_info_reset_dequeued(t);
  77         t->sched_info.run_delay += delta;
  78
  79         rq_sched_info_dequeued(rq, delta);
  80 }
  81
  82 /*
  83  * Called when a task finally hits the cpu.  We can now calculate how
  84  * long it was waiting to run.  We also note when it began so that we
  85  * can keep stats on how long its timeslice is.
  86  */
  87 static void sched_info_arrive(struct rq *rq, struct task_struct *t)
  88 {
  89         unsigned long long now = rq_clock(rq), delta = 0;
  90
  91         if (t->sched_info.last_queued)
  92                 delta = now - t->sched_info.last_queued;
  93         sched_info_reset_dequeued(t);
  94         t->sched_info.run_delay += delta;
  95         t->sched_info.last_arrival = now;
  96         t->sched_info.pcount++;
  97
  98         rq_sched_info_arrive(rq, delta);
  99 }
 100
 101 /*
 102  * This function is only called from enqueue_task(), but also only updates
 103  * the timestamp if it is already not set.  It's assumed that
 104  * sched_info_dequeued() will clear that stamp when appropriate.
 105  */
 106 static inline void sched_info_queued(struct rq *rq, struct task_struct *t)
 107 {
 108         if (unlikely(sched_info_on()))
 109                 if (!t->sched_info.last_queued)
 110                         t->sched_info.last_queued = rq_clock(rq);
 111 }
 112
 113 /*
 114  * Called when a process ceases being the active-running process involuntarily
 115  * due, typically, to expiring its time slice (this may also be called when
 116  * switching to the idle task).  Now we can calculate how long we ran.
 117  * Also, if the process is still in the TASK_RUNNING state, call
 118  * sched_info_queued() to mark that it has now again started waiting on
 119  * the runqueue.
 120  */
 121 static inline void sched_info_depart(struct rq *rq, struct task_struct *t)
 122 {
 123         unsigned long long delta = rq_clock(rq) -
 124                                         t->sched_info.last_arrival;
 125
 126         rq_sched_info_depart(rq, delta);
 127
 128         if (t->state == TASK_RUNNING)
 129                 sched_info_queued(rq, t);
 130 }
 131
 132 /*
 133  * Called when tasks are switched involuntarily due, typically, to expiring
 134  * their time slice.  (This may also be called when switching to or from
 135  * the idle task.)  We are only called when prev != next.
 136  */
 137 static inline void
 138 __sched_info_switch(struct rq *rq,
 139                     struct task_struct *prev, struct task_struct *next)
 140 {
 141         /*
 142          * prev now departs the cpu.  It's not interesting to record
 143          * stats about how efficient we were at scheduling the idle
 144          * process, however.
 145          */
 146         if (prev != rq->idle)
 147                 sched_info_depart(rq, prev);
 148
 149         if (next != rq->idle)
 150                 sched_info_arrive(rq, next);
 151 }
 152 static inline void
 153 sched_info_switch(struct rq *rq,
 154                   struct task_struct *prev, struct task_struct *next)
 155 {
 156         if (unlikely(sched_info_on()))
 157                 __sched_info_switch(rq, prev, next);
 158 }
 159 #else
 160 #define sched_info_queued(rq, t)                do { } while (0)
 161 #define sched_info_reset_dequeued(t)    do { } while (0)
 162 #define sched_info_dequeued(rq, t)              do { } while (0)
 163 #define sched_info_depart(rq, t)                do { } while (0)
 164 #define sched_info_arrive(rq, next)             do { } while (0)
 165 #define sched_info_switch(rq, t, next)          do { } while (0)
 166 #endif /* CONFIG_SCHED_INFO */
 167
 168 /*
 169  * The following are functions that support scheduler-internal time accounting.
 170  * These functions are generally called at the timer tick.  None of this depends
 171  * on CONFIG_SCHEDSTATS.
 172  */
 173
 174 /**
 175  * get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
 176  *
 177  * @tsk:        Pointer to target task.
 178  */
 179 #ifdef CONFIG_POSIX_TIMERS
 180 static inline
 181 struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
 182 {
 183         struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
 184
 185         /* Check if cputimer isn't running. This is accessed without locking. */
 186         if (!READ_ONCE(cputimer->running))
 187                 return NULL;
 188
 189         /*
 190          * After we flush the task's sum_exec_runtime to sig->sum_sched_runtime
 191          * in __exit_signal(), we won't account to the signal struct further
 192          * cputime consumed by that task, even though the task can still be
 193          * ticking after __exit_signal().
 194          *
 195          * In order to keep a consistent behaviour between thread group cputime
 196          * and thread group cputimer accounting, lets also ignore the cputime
 197          * elapsing after __exit_signal() in any thread group timer running.
 198          *
 199          * This makes sure that POSIX CPU clocks and timers are synchronized, so
 200          * that a POSIX CPU timer won't expire while the corresponding POSIX CPU
 201          * clock delta is behind the expiring timer value.
 202          */
 203         if (unlikely(!tsk->sighand))
 204                 return NULL;
 205
 206         return cputimer;
 207 }
 208 #else
 209 static inline
 210 struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
 211 {
 212         return NULL;
 213 }
 214 #endif
 215
 216 /**
 217  * account_group_user_time - Maintain utime for a thread group.
 218  *
 219  * @tsk:        Pointer to task structure.
 220  * @cputime:    Time value by which to increment the utime field of the
 221  *              thread_group_cputime structure.
 222  *
 223  * If thread group time is being maintained, get the structure for the
 224  * running CPU and update the utime field there.
 225  */
 226 static inline void account_group_user_time(struct task_struct *tsk,
 227                                            u64 cputime)
 228 {
 229         struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
 230
 231         if (!cputimer)
 232                 return;
 233
 234         atomic64_add(cputime, &cputimer->cputime_atomic.utime);
 235 }
 236
 237 /**
 238  * account_group_system_time - Maintain stime for a thread group.
 239  *
 240  * @tsk:        Pointer to task structure.
 241  * @cputime:    Time value by which to increment the stime field of the
 242  *              thread_group_cputime structure.
 243  *
 244  * If thread group time is being maintained, get the structure for the
 245  * running CPU and update the stime field there.
 246  */
 247 static inline void account_group_system_time(struct task_struct *tsk,
 248                                              u64 cputime)
 249 {
 250         struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
 251
 252         if (!cputimer)
 253                 return;
 254
 255         atomic64_add(cputime, &cputimer->cputime_atomic.stime);
 256 }
 257
 258 /**
 259  * account_group_exec_runtime - Maintain exec runtime for a thread group.
 260  *
 261  * @tsk:        Pointer to task structure.
 262  * @ns:         Time value by which to increment the sum_exec_runtime field
 263  *              of the thread_group_cputime structure.
 264  *
 265  * If thread group time is being maintained, get the structure for the
 266  * running CPU and update the sum_exec_runtime field there.
 267  */
 268 static inline void account_group_exec_runtime(struct task_struct *tsk,
 269                                               unsigned long long ns)
 270 {
 271         struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
 272
 273         if (!cputimer)
 274                 return;
 275
 276         atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime);
 277 }