]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
2e76c24d LZ |
2 | #include <linux/cgroup.h> |
3 | #include <linux/slab.h> | |
4 | #include <linux/percpu.h> | |
5 | #include <linux/spinlock.h> | |
6 | #include <linux/cpumask.h> | |
7 | #include <linux/seq_file.h> | |
8 | #include <linux/rcupdate.h> | |
9 | #include <linux/kernel_stat.h> | |
b329fd5b | 10 | #include <linux/err.h> |
2e76c24d LZ |
11 | |
12 | #include "sched.h" | |
13 | ||
14 | /* | |
15 | * CPU accounting code for task groups. | |
16 | * | |
17 | * Based on the work by Paul Menage (menage@google.com) and Balbir Singh | |
18 | * (balbir@in.ibm.com). | |
19 | */ | |
20 | ||
d1712796 LZ |
21 | /* Time spent by the tasks of the cpu accounting group executing in ... */ |
22 | enum cpuacct_stat_index { | |
23 | CPUACCT_STAT_USER, /* ... user mode */ | |
24 | CPUACCT_STAT_SYSTEM, /* ... kernel mode */ | |
25 | ||
26 | CPUACCT_STAT_NSTATS, | |
27 | }; | |
28 | ||
9acacc2a ZL |
29 | static const char * const cpuacct_stat_desc[] = { |
30 | [CPUACCT_STAT_USER] = "user", | |
31 | [CPUACCT_STAT_SYSTEM] = "system", | |
d740037f DY |
32 | }; |
33 | ||
34 | struct cpuacct_usage { | |
9acacc2a | 35 | u64 usages[CPUACCT_STAT_NSTATS]; |
d740037f DY |
36 | }; |
37 | ||
d1712796 LZ |
38 | /* track cpu usage of a group of tasks and its child groups */ |
39 | struct cpuacct { | |
40 | struct cgroup_subsys_state css; | |
41 | /* cpuusage holds pointer to a u64-type object on every cpu */ | |
d740037f | 42 | struct cpuacct_usage __percpu *cpuusage; |
d1712796 LZ |
43 | struct kernel_cpustat __percpu *cpustat; |
44 | }; | |
45 | ||
a7c6d554 TH |
46 | static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css) |
47 | { | |
48 | return css ? container_of(css, struct cpuacct, css) : NULL; | |
49 | } | |
50 | ||
d1712796 LZ |
51 | /* return cpu accounting group to which this task belongs */ |
52 | static inline struct cpuacct *task_ca(struct task_struct *tsk) | |
53 | { | |
073219e9 | 54 | return css_ca(task_css(tsk, cpuacct_cgrp_id)); |
d1712796 LZ |
55 | } |
56 | ||
d1712796 LZ |
57 | static inline struct cpuacct *parent_ca(struct cpuacct *ca) |
58 | { | |
5c9d535b | 59 | return css_ca(ca->css.parent); |
d1712796 LZ |
60 | } |
61 | ||
d740037f | 62 | static DEFINE_PER_CPU(struct cpuacct_usage, root_cpuacct_cpuusage); |
14c6d3c8 LZ |
63 | static struct cpuacct root_cpuacct = { |
64 | .cpustat = &kernel_cpustat, | |
65 | .cpuusage = &root_cpuacct_cpuusage, | |
66 | }; | |
2e76c24d LZ |
67 | |
68 | /* create a new cpu accounting group */ | |
eb95419b TH |
69 | static struct cgroup_subsys_state * |
70 | cpuacct_css_alloc(struct cgroup_subsys_state *parent_css) | |
2e76c24d LZ |
71 | { |
72 | struct cpuacct *ca; | |
73 | ||
eb95419b | 74 | if (!parent_css) |
2e76c24d LZ |
75 | return &root_cpuacct.css; |
76 | ||
77 | ca = kzalloc(sizeof(*ca), GFP_KERNEL); | |
78 | if (!ca) | |
79 | goto out; | |
80 | ||
d740037f | 81 | ca->cpuusage = alloc_percpu(struct cpuacct_usage); |
2e76c24d LZ |
82 | if (!ca->cpuusage) |
83 | goto out_free_ca; | |
84 | ||
85 | ca->cpustat = alloc_percpu(struct kernel_cpustat); | |
86 | if (!ca->cpustat) | |
87 | goto out_free_cpuusage; | |
88 | ||
89 | return &ca->css; | |
90 | ||
91 | out_free_cpuusage: | |
92 | free_percpu(ca->cpuusage); | |
93 | out_free_ca: | |
94 | kfree(ca); | |
95 | out: | |
96 | return ERR_PTR(-ENOMEM); | |
97 | } | |
98 | ||
99 | /* destroy an existing cpu accounting group */ | |
eb95419b | 100 | static void cpuacct_css_free(struct cgroup_subsys_state *css) |
2e76c24d | 101 | { |
eb95419b | 102 | struct cpuacct *ca = css_ca(css); |
2e76c24d LZ |
103 | |
104 | free_percpu(ca->cpustat); | |
105 | free_percpu(ca->cpuusage); | |
106 | kfree(ca); | |
107 | } | |
108 | ||
d740037f | 109 | static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu, |
9acacc2a | 110 | enum cpuacct_stat_index index) |
2e76c24d | 111 | { |
d740037f | 112 | struct cpuacct_usage *cpuusage = per_cpu_ptr(ca->cpuusage, cpu); |
2e76c24d LZ |
113 | u64 data; |
114 | ||
d740037f | 115 | /* |
9acacc2a | 116 | * We allow index == CPUACCT_STAT_NSTATS here to read |
d740037f DY |
117 | * the sum of suages. |
118 | */ | |
9acacc2a | 119 | BUG_ON(index > CPUACCT_STAT_NSTATS); |
d740037f | 120 | |
2e76c24d LZ |
121 | #ifndef CONFIG_64BIT |
122 | /* | |
123 | * Take rq->lock to make 64-bit read safe on 32-bit platforms. | |
124 | */ | |
125 | raw_spin_lock_irq(&cpu_rq(cpu)->lock); | |
d740037f DY |
126 | #endif |
127 | ||
9acacc2a | 128 | if (index == CPUACCT_STAT_NSTATS) { |
d740037f DY |
129 | int i = 0; |
130 | ||
131 | data = 0; | |
9acacc2a | 132 | for (i = 0; i < CPUACCT_STAT_NSTATS; i++) |
d740037f DY |
133 | data += cpuusage->usages[i]; |
134 | } else { | |
135 | data = cpuusage->usages[index]; | |
136 | } | |
137 | ||
138 | #ifndef CONFIG_64BIT | |
2e76c24d | 139 | raw_spin_unlock_irq(&cpu_rq(cpu)->lock); |
2e76c24d LZ |
140 | #endif |
141 | ||
142 | return data; | |
143 | } | |
144 | ||
145 | static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val) | |
146 | { | |
d740037f DY |
147 | struct cpuacct_usage *cpuusage = per_cpu_ptr(ca->cpuusage, cpu); |
148 | int i; | |
2e76c24d LZ |
149 | |
150 | #ifndef CONFIG_64BIT | |
151 | /* | |
152 | * Take rq->lock to make 64-bit write safe on 32-bit platforms. | |
153 | */ | |
154 | raw_spin_lock_irq(&cpu_rq(cpu)->lock); | |
d740037f DY |
155 | #endif |
156 | ||
9acacc2a | 157 | for (i = 0; i < CPUACCT_STAT_NSTATS; i++) |
d740037f DY |
158 | cpuusage->usages[i] = val; |
159 | ||
160 | #ifndef CONFIG_64BIT | |
2e76c24d | 161 | raw_spin_unlock_irq(&cpu_rq(cpu)->lock); |
2e76c24d LZ |
162 | #endif |
163 | } | |
164 | ||
165 | /* return total cpu usage (in nanoseconds) of a group */ | |
d740037f | 166 | static u64 __cpuusage_read(struct cgroup_subsys_state *css, |
9acacc2a | 167 | enum cpuacct_stat_index index) |
2e76c24d | 168 | { |
182446d0 | 169 | struct cpuacct *ca = css_ca(css); |
2e76c24d LZ |
170 | u64 totalcpuusage = 0; |
171 | int i; | |
172 | ||
5ca3726a | 173 | for_each_possible_cpu(i) |
d740037f | 174 | totalcpuusage += cpuacct_cpuusage_read(ca, i, index); |
2e76c24d LZ |
175 | |
176 | return totalcpuusage; | |
177 | } | |
178 | ||
d740037f DY |
179 | static u64 cpuusage_user_read(struct cgroup_subsys_state *css, |
180 | struct cftype *cft) | |
181 | { | |
9acacc2a | 182 | return __cpuusage_read(css, CPUACCT_STAT_USER); |
d740037f DY |
183 | } |
184 | ||
185 | static u64 cpuusage_sys_read(struct cgroup_subsys_state *css, | |
186 | struct cftype *cft) | |
187 | { | |
9acacc2a | 188 | return __cpuusage_read(css, CPUACCT_STAT_SYSTEM); |
d740037f DY |
189 | } |
190 | ||
191 | static u64 cpuusage_read(struct cgroup_subsys_state *css, struct cftype *cft) | |
192 | { | |
9acacc2a | 193 | return __cpuusage_read(css, CPUACCT_STAT_NSTATS); |
d740037f DY |
194 | } |
195 | ||
182446d0 | 196 | static int cpuusage_write(struct cgroup_subsys_state *css, struct cftype *cft, |
1a736b77 | 197 | u64 val) |
2e76c24d | 198 | { |
182446d0 | 199 | struct cpuacct *ca = css_ca(css); |
d740037f | 200 | int cpu; |
2e76c24d | 201 | |
1a736b77 DY |
202 | /* |
203 | * Only allow '0' here to do a reset. | |
204 | */ | |
d740037f DY |
205 | if (val) |
206 | return -EINVAL; | |
2e76c24d | 207 | |
d740037f DY |
208 | for_each_possible_cpu(cpu) |
209 | cpuacct_cpuusage_write(ca, cpu, 0); | |
2e76c24d | 210 | |
d740037f | 211 | return 0; |
2e76c24d LZ |
212 | } |
213 | ||
d740037f | 214 | static int __cpuacct_percpu_seq_show(struct seq_file *m, |
9acacc2a | 215 | enum cpuacct_stat_index index) |
2e76c24d | 216 | { |
2da8ca82 | 217 | struct cpuacct *ca = css_ca(seq_css(m)); |
2e76c24d LZ |
218 | u64 percpu; |
219 | int i; | |
220 | ||
5ca3726a | 221 | for_each_possible_cpu(i) { |
d740037f | 222 | percpu = cpuacct_cpuusage_read(ca, i, index); |
2e76c24d LZ |
223 | seq_printf(m, "%llu ", (unsigned long long) percpu); |
224 | } | |
225 | seq_printf(m, "\n"); | |
226 | return 0; | |
227 | } | |
228 | ||
d740037f DY |
229 | static int cpuacct_percpu_user_seq_show(struct seq_file *m, void *V) |
230 | { | |
9acacc2a | 231 | return __cpuacct_percpu_seq_show(m, CPUACCT_STAT_USER); |
d740037f DY |
232 | } |
233 | ||
234 | static int cpuacct_percpu_sys_seq_show(struct seq_file *m, void *V) | |
235 | { | |
9acacc2a | 236 | return __cpuacct_percpu_seq_show(m, CPUACCT_STAT_SYSTEM); |
d740037f DY |
237 | } |
238 | ||
239 | static int cpuacct_percpu_seq_show(struct seq_file *m, void *V) | |
240 | { | |
9acacc2a | 241 | return __cpuacct_percpu_seq_show(m, CPUACCT_STAT_NSTATS); |
d740037f DY |
242 | } |
243 | ||
277a13e4 ZL |
244 | static int cpuacct_all_seq_show(struct seq_file *m, void *V) |
245 | { | |
246 | struct cpuacct *ca = css_ca(seq_css(m)); | |
247 | int index; | |
248 | int cpu; | |
249 | ||
250 | seq_puts(m, "cpu"); | |
251 | for (index = 0; index < CPUACCT_STAT_NSTATS; index++) | |
252 | seq_printf(m, " %s", cpuacct_stat_desc[index]); | |
253 | seq_puts(m, "\n"); | |
254 | ||
255 | for_each_possible_cpu(cpu) { | |
256 | struct cpuacct_usage *cpuusage = per_cpu_ptr(ca->cpuusage, cpu); | |
257 | ||
258 | seq_printf(m, "%d", cpu); | |
259 | ||
260 | for (index = 0; index < CPUACCT_STAT_NSTATS; index++) { | |
261 | #ifndef CONFIG_64BIT | |
262 | /* | |
263 | * Take rq->lock to make 64-bit read safe on 32-bit | |
264 | * platforms. | |
265 | */ | |
266 | raw_spin_lock_irq(&cpu_rq(cpu)->lock); | |
267 | #endif | |
268 | ||
269 | seq_printf(m, " %llu", cpuusage->usages[index]); | |
270 | ||
271 | #ifndef CONFIG_64BIT | |
272 | raw_spin_unlock_irq(&cpu_rq(cpu)->lock); | |
273 | #endif | |
274 | } | |
275 | seq_puts(m, "\n"); | |
276 | } | |
277 | return 0; | |
278 | } | |
279 | ||
2da8ca82 | 280 | static int cpuacct_stats_show(struct seq_file *sf, void *v) |
2e76c24d | 281 | { |
2da8ca82 | 282 | struct cpuacct *ca = css_ca(seq_css(sf)); |
8e546bfa | 283 | s64 val[CPUACCT_STAT_NSTATS]; |
2e76c24d | 284 | int cpu; |
8e546bfa | 285 | int stat; |
2e76c24d | 286 | |
8e546bfa | 287 | memset(val, 0, sizeof(val)); |
5ca3726a | 288 | for_each_possible_cpu(cpu) { |
8e546bfa | 289 | u64 *cpustat = per_cpu_ptr(ca->cpustat, cpu)->cpustat; |
2e76c24d | 290 | |
8e546bfa ZL |
291 | val[CPUACCT_STAT_USER] += cpustat[CPUTIME_USER]; |
292 | val[CPUACCT_STAT_USER] += cpustat[CPUTIME_NICE]; | |
293 | val[CPUACCT_STAT_SYSTEM] += cpustat[CPUTIME_SYSTEM]; | |
294 | val[CPUACCT_STAT_SYSTEM] += cpustat[CPUTIME_IRQ]; | |
295 | val[CPUACCT_STAT_SYSTEM] += cpustat[CPUTIME_SOFTIRQ]; | |
2e76c24d LZ |
296 | } |
297 | ||
8e546bfa ZL |
298 | for (stat = 0; stat < CPUACCT_STAT_NSTATS; stat++) { |
299 | seq_printf(sf, "%s %lld\n", | |
300 | cpuacct_stat_desc[stat], | |
7fb1327e | 301 | (long long)nsec_to_clock_t(val[stat])); |
8e546bfa | 302 | } |
2e76c24d LZ |
303 | |
304 | return 0; | |
305 | } | |
306 | ||
307 | static struct cftype files[] = { | |
308 | { | |
309 | .name = "usage", | |
310 | .read_u64 = cpuusage_read, | |
311 | .write_u64 = cpuusage_write, | |
312 | }, | |
d740037f DY |
313 | { |
314 | .name = "usage_user", | |
315 | .read_u64 = cpuusage_user_read, | |
316 | }, | |
317 | { | |
318 | .name = "usage_sys", | |
319 | .read_u64 = cpuusage_sys_read, | |
320 | }, | |
2e76c24d LZ |
321 | { |
322 | .name = "usage_percpu", | |
2da8ca82 | 323 | .seq_show = cpuacct_percpu_seq_show, |
2e76c24d | 324 | }, |
d740037f DY |
325 | { |
326 | .name = "usage_percpu_user", | |
327 | .seq_show = cpuacct_percpu_user_seq_show, | |
328 | }, | |
329 | { | |
330 | .name = "usage_percpu_sys", | |
331 | .seq_show = cpuacct_percpu_sys_seq_show, | |
332 | }, | |
277a13e4 ZL |
333 | { |
334 | .name = "usage_all", | |
335 | .seq_show = cpuacct_all_seq_show, | |
336 | }, | |
2e76c24d LZ |
337 | { |
338 | .name = "stat", | |
2da8ca82 | 339 | .seq_show = cpuacct_stats_show, |
2e76c24d LZ |
340 | }, |
341 | { } /* terminate */ | |
342 | }; | |
343 | ||
344 | /* | |
345 | * charge this task's execution time to its accounting group. | |
346 | * | |
347 | * called with rq->lock held. | |
348 | */ | |
349 | void cpuacct_charge(struct task_struct *tsk, u64 cputime) | |
350 | { | |
351 | struct cpuacct *ca; | |
9acacc2a | 352 | int index = CPUACCT_STAT_SYSTEM; |
bd928830 | 353 | struct pt_regs *regs = task_pt_regs(tsk); |
d740037f | 354 | |
bd928830 | 355 | if (regs && user_mode(regs)) |
9acacc2a | 356 | index = CPUACCT_STAT_USER; |
2e76c24d LZ |
357 | |
358 | rcu_read_lock(); | |
d740037f | 359 | |
73e6aafd | 360 | for (ca = task_ca(tsk); ca; ca = parent_ca(ca)) |
d740037f DY |
361 | this_cpu_ptr(ca->cpuusage)->usages[index] += cputime; |
362 | ||
2e76c24d LZ |
363 | rcu_read_unlock(); |
364 | } | |
365 | ||
1966aaf7 LZ |
366 | /* |
367 | * Add user/system time to cpuacct. | |
368 | * | |
369 | * Note: it's the caller that updates the account of the root cgroup. | |
370 | */ | |
73e6aafd | 371 | void cpuacct_account_field(struct task_struct *tsk, int index, u64 val) |
1966aaf7 | 372 | { |
1966aaf7 LZ |
373 | struct cpuacct *ca; |
374 | ||
1966aaf7 | 375 | rcu_read_lock(); |
73e6aafd ZL |
376 | for (ca = task_ca(tsk); ca != &root_cpuacct; ca = parent_ca(ca)) |
377 | this_cpu_ptr(ca->cpustat)->cpustat[index] += val; | |
1966aaf7 LZ |
378 | rcu_read_unlock(); |
379 | } | |
380 | ||
073219e9 | 381 | struct cgroup_subsys cpuacct_cgrp_subsys = { |
621e2de0 LZ |
382 | .css_alloc = cpuacct_css_alloc, |
383 | .css_free = cpuacct_css_free, | |
5577964e | 384 | .legacy_cftypes = files, |
b38e42e9 | 385 | .early_init = true, |
2e76c24d | 386 | }; |