1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * taskstats.c - Export per-task statistics to userland
5 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
6 * (C) Balbir Singh, IBM Corp. 2006
9 #include <linux/kernel.h>
10 #include <linux/taskstats_kern.h>
11 #include <linux/tsacct_kern.h>
12 #include <linux/acct.h>
13 #include <linux/delayacct.h>
14 #include <linux/cpumask.h>
15 #include <linux/percpu.h>
16 #include <linux/slab.h>
17 #include <linux/cgroupstats.h>
18 #include <linux/cgroup.h>
20 #include <linux/file.h>
21 #include <linux/pid_namespace.h>
22 #include <net/genetlink.h>
23 #include <linux/atomic.h>
24 #include <linux/sched/cputime.h>
27 * Maximum length of a cpumask that can be specified in
28 * the TASKSTATS_CMD_ATTR_REGISTER/DEREGISTER_CPUMASK attribute
30 #define TASKSTATS_CPUMASK_MAXLEN (100+6*NR_CPUS)
32 static DEFINE_PER_CPU(__u32
, taskstats_seqnum
);
33 static int family_registered
;
34 struct kmem_cache
*taskstats_cache
;
36 static struct genl_family family
;
38 static const struct nla_policy taskstats_cmd_get_policy
[] = {
39 [TASKSTATS_CMD_ATTR_PID
] = { .type
= NLA_U32
},
40 [TASKSTATS_CMD_ATTR_TGID
] = { .type
= NLA_U32
},
41 [TASKSTATS_CMD_ATTR_REGISTER_CPUMASK
] = { .type
= NLA_STRING
},
42 [TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK
] = { .type
= NLA_STRING
},};
44 static const struct nla_policy cgroupstats_cmd_get_policy
[] = {
45 [CGROUPSTATS_CMD_ATTR_FD
] = { .type
= NLA_U32
},
49 struct list_head list
;
54 struct listener_list
{
55 struct rw_semaphore sem
;
56 struct list_head list
;
58 static DEFINE_PER_CPU(struct listener_list
, listener_array
);
66 static int prepare_reply(struct genl_info
*info
, u8 cmd
, struct sk_buff
**skbp
,
73 * If new attributes are added, please revisit this allocation
75 skb
= genlmsg_new(size
, GFP_KERNEL
);
80 int seq
= this_cpu_inc_return(taskstats_seqnum
) - 1;
82 reply
= genlmsg_put(skb
, 0, seq
, &family
, 0, cmd
);
84 reply
= genlmsg_put_reply(skb
, info
, &family
, 0, cmd
);
95 * Send taskstats data in @skb to listener with nl_pid @pid
97 static int send_reply(struct sk_buff
*skb
, struct genl_info
*info
)
99 struct genlmsghdr
*genlhdr
= nlmsg_data(nlmsg_hdr(skb
));
100 void *reply
= genlmsg_data(genlhdr
);
102 genlmsg_end(skb
, reply
);
104 return genlmsg_reply(skb
, info
);
108 * Send taskstats data in @skb to listeners registered for @cpu's exit data
110 static void send_cpu_listeners(struct sk_buff
*skb
,
111 struct listener_list
*listeners
)
113 struct genlmsghdr
*genlhdr
= nlmsg_data(nlmsg_hdr(skb
));
114 struct listener
*s
, *tmp
;
115 struct sk_buff
*skb_next
, *skb_cur
= skb
;
116 void *reply
= genlmsg_data(genlhdr
);
119 genlmsg_end(skb
, reply
);
121 down_read(&listeners
->sem
);
122 list_for_each_entry(s
, &listeners
->list
, list
) {
126 if (!list_is_last(&s
->list
, &listeners
->list
)) {
127 skb_next
= skb_clone(skb_cur
, GFP_KERNEL
);
131 rc
= genlmsg_unicast(&init_net
, skb_cur
, s
->pid
);
132 if (rc
== -ECONNREFUSED
) {
138 up_read(&listeners
->sem
);
146 /* Delete invalidated entries */
147 down_write(&listeners
->sem
);
148 list_for_each_entry_safe(s
, tmp
, &listeners
->list
, list
) {
154 up_write(&listeners
->sem
);
157 static void exe_add_tsk(struct taskstats
*stats
, struct task_struct
*tsk
)
159 /* No idea if I'm allowed to access that here, now. */
160 struct file
*exe_file
= get_task_exe_file(tsk
);
163 /* Following cp_new_stat64() in stat.c . */
165 huge_encode_dev(exe_file
->f_inode
->i_sb
->s_dev
);
166 stats
->ac_exe_inode
= exe_file
->f_inode
->i_ino
;
169 stats
->ac_exe_dev
= 0;
170 stats
->ac_exe_inode
= 0;
174 static void fill_stats(struct user_namespace
*user_ns
,
175 struct pid_namespace
*pid_ns
,
176 struct task_struct
*tsk
, struct taskstats
*stats
)
178 memset(stats
, 0, sizeof(*stats
));
180 * Each accounting subsystem adds calls to its functions to
181 * fill in relevant parts of struct taskstsats as follows
183 * per-task-foo(stats, tsk);
186 delayacct_add_tsk(stats
, tsk
);
188 /* fill in basic acct fields */
189 stats
->version
= TASKSTATS_VERSION
;
190 stats
->nvcsw
= tsk
->nvcsw
;
191 stats
->nivcsw
= tsk
->nivcsw
;
192 bacct_add_tsk(user_ns
, pid_ns
, stats
, tsk
);
194 /* fill in extended acct fields */
195 xacct_add_tsk(stats
, tsk
);
197 /* add executable info */
198 exe_add_tsk(stats
, tsk
);
201 static int fill_stats_for_pid(pid_t pid
, struct taskstats
*stats
)
203 struct task_struct
*tsk
;
205 tsk
= find_get_task_by_vpid(pid
);
208 fill_stats(current_user_ns(), task_active_pid_ns(current
), tsk
, stats
);
209 put_task_struct(tsk
);
213 static int fill_stats_for_tgid(pid_t tgid
, struct taskstats
*stats
)
215 struct task_struct
*tsk
, *first
;
218 u64 delta
, utime
, stime
;
222 * Add additional stats from live tasks except zombie thread group
223 * leaders who are already counted with the dead tasks
226 first
= find_task_by_vpid(tgid
);
228 if (!first
|| !lock_task_sighand(first
, &flags
))
231 if (first
->signal
->stats
)
232 memcpy(stats
, first
->signal
->stats
, sizeof(*stats
));
234 memset(stats
, 0, sizeof(*stats
));
237 start_time
= ktime_get_ns();
242 * Accounting subsystem can call its functions here to
243 * fill in relevant parts of struct taskstsats as follows
245 * per-task-foo(stats, tsk);
247 delayacct_add_tsk(stats
, tsk
);
249 /* calculate task elapsed time in nsec */
250 delta
= start_time
- tsk
->start_time
;
251 /* Convert to micro seconds */
252 do_div(delta
, NSEC_PER_USEC
);
253 stats
->ac_etime
+= delta
;
255 task_cputime(tsk
, &utime
, &stime
);
256 stats
->ac_utime
+= div_u64(utime
, NSEC_PER_USEC
);
257 stats
->ac_stime
+= div_u64(stime
, NSEC_PER_USEC
);
259 stats
->nvcsw
+= tsk
->nvcsw
;
260 stats
->nivcsw
+= tsk
->nivcsw
;
261 } while_each_thread(first
, tsk
);
263 unlock_task_sighand(first
, &flags
);
268 stats
->version
= TASKSTATS_VERSION
;
270 * Accounting subsystems can also add calls here to modify
271 * fields of taskstats.
276 static void fill_tgid_exit(struct task_struct
*tsk
)
280 spin_lock_irqsave(&tsk
->sighand
->siglock
, flags
);
281 if (!tsk
->signal
->stats
)
285 * Each accounting subsystem calls its functions here to
286 * accumalate its per-task stats for tsk, into the per-tgid structure
288 * per-task-foo(tsk->signal->stats, tsk);
290 delayacct_add_tsk(tsk
->signal
->stats
, tsk
);
292 spin_unlock_irqrestore(&tsk
->sighand
->siglock
, flags
);
296 static int add_del_listener(pid_t pid
, const struct cpumask
*mask
, int isadd
)
298 struct listener_list
*listeners
;
299 struct listener
*s
, *tmp
, *s2
;
303 if (!cpumask_subset(mask
, cpu_possible_mask
))
306 if (current_user_ns() != &init_user_ns
)
309 if (task_active_pid_ns(current
) != &init_pid_ns
)
312 if (isadd
== REGISTER
) {
313 for_each_cpu(cpu
, mask
) {
314 s
= kmalloc_node(sizeof(struct listener
),
315 GFP_KERNEL
, cpu_to_node(cpu
));
323 listeners
= &per_cpu(listener_array
, cpu
);
324 down_write(&listeners
->sem
);
325 list_for_each_entry(s2
, &listeners
->list
, list
) {
326 if (s2
->pid
== pid
&& s2
->valid
)
329 list_add(&s
->list
, &listeners
->list
);
332 up_write(&listeners
->sem
);
333 kfree(s
); /* nop if NULL */
338 /* Deregister or cleanup */
340 for_each_cpu(cpu
, mask
) {
341 listeners
= &per_cpu(listener_array
, cpu
);
342 down_write(&listeners
->sem
);
343 list_for_each_entry_safe(s
, tmp
, &listeners
->list
, list
) {
350 up_write(&listeners
->sem
);
355 static int parse(struct nlattr
*na
, struct cpumask
*mask
)
364 if (len
> TASKSTATS_CPUMASK_MAXLEN
)
368 data
= kmalloc(len
, GFP_KERNEL
);
371 nla_strscpy(data
, na
, len
);
372 ret
= cpulist_parse(data
, mask
);
377 static struct taskstats
*mk_reply(struct sk_buff
*skb
, int type
, u32 pid
)
379 struct nlattr
*na
, *ret
;
382 aggr
= (type
== TASKSTATS_TYPE_PID
)
383 ? TASKSTATS_TYPE_AGGR_PID
384 : TASKSTATS_TYPE_AGGR_TGID
;
386 na
= nla_nest_start_noflag(skb
, aggr
);
390 if (nla_put(skb
, type
, sizeof(pid
), &pid
) < 0) {
391 nla_nest_cancel(skb
, na
);
394 ret
= nla_reserve_64bit(skb
, TASKSTATS_TYPE_STATS
,
395 sizeof(struct taskstats
), TASKSTATS_TYPE_NULL
);
397 nla_nest_cancel(skb
, na
);
400 nla_nest_end(skb
, na
);
402 return nla_data(ret
);
407 static int cgroupstats_user_cmd(struct sk_buff
*skb
, struct genl_info
*info
)
410 struct sk_buff
*rep_skb
;
411 struct cgroupstats
*stats
;
417 na
= info
->attrs
[CGROUPSTATS_CMD_ATTR_FD
];
421 fd
= nla_get_u32(info
->attrs
[CGROUPSTATS_CMD_ATTR_FD
]);
426 size
= nla_total_size(sizeof(struct cgroupstats
));
428 rc
= prepare_reply(info
, CGROUPSTATS_CMD_NEW
, &rep_skb
,
433 na
= nla_reserve(rep_skb
, CGROUPSTATS_TYPE_CGROUP_STATS
,
434 sizeof(struct cgroupstats
));
441 stats
= nla_data(na
);
442 memset(stats
, 0, sizeof(*stats
));
444 rc
= cgroupstats_build(stats
, f
.file
->f_path
.dentry
);
450 rc
= send_reply(rep_skb
, info
);
457 static int cmd_attr_register_cpumask(struct genl_info
*info
)
462 if (!alloc_cpumask_var(&mask
, GFP_KERNEL
))
464 rc
= parse(info
->attrs
[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK
], mask
);
467 rc
= add_del_listener(info
->snd_portid
, mask
, REGISTER
);
469 free_cpumask_var(mask
);
473 static int cmd_attr_deregister_cpumask(struct genl_info
*info
)
478 if (!alloc_cpumask_var(&mask
, GFP_KERNEL
))
480 rc
= parse(info
->attrs
[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK
], mask
);
483 rc
= add_del_listener(info
->snd_portid
, mask
, DEREGISTER
);
485 free_cpumask_var(mask
);
489 static size_t taskstats_packet_size(void)
493 size
= nla_total_size(sizeof(u32
)) +
494 nla_total_size_64bit(sizeof(struct taskstats
)) +
500 static int cmd_attr_pid(struct genl_info
*info
)
502 struct taskstats
*stats
;
503 struct sk_buff
*rep_skb
;
508 size
= taskstats_packet_size();
510 rc
= prepare_reply(info
, TASKSTATS_CMD_NEW
, &rep_skb
, size
);
515 pid
= nla_get_u32(info
->attrs
[TASKSTATS_CMD_ATTR_PID
]);
516 stats
= mk_reply(rep_skb
, TASKSTATS_TYPE_PID
, pid
);
520 rc
= fill_stats_for_pid(pid
, stats
);
523 return send_reply(rep_skb
, info
);
529 static int cmd_attr_tgid(struct genl_info
*info
)
531 struct taskstats
*stats
;
532 struct sk_buff
*rep_skb
;
537 size
= taskstats_packet_size();
539 rc
= prepare_reply(info
, TASKSTATS_CMD_NEW
, &rep_skb
, size
);
544 tgid
= nla_get_u32(info
->attrs
[TASKSTATS_CMD_ATTR_TGID
]);
545 stats
= mk_reply(rep_skb
, TASKSTATS_TYPE_TGID
, tgid
);
549 rc
= fill_stats_for_tgid(tgid
, stats
);
552 return send_reply(rep_skb
, info
);
558 static int taskstats_user_cmd(struct sk_buff
*skb
, struct genl_info
*info
)
560 if (info
->attrs
[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK
])
561 return cmd_attr_register_cpumask(info
);
562 else if (info
->attrs
[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK
])
563 return cmd_attr_deregister_cpumask(info
);
564 else if (info
->attrs
[TASKSTATS_CMD_ATTR_PID
])
565 return cmd_attr_pid(info
);
566 else if (info
->attrs
[TASKSTATS_CMD_ATTR_TGID
])
567 return cmd_attr_tgid(info
);
572 static struct taskstats
*taskstats_tgid_alloc(struct task_struct
*tsk
)
574 struct signal_struct
*sig
= tsk
->signal
;
575 struct taskstats
*stats_new
, *stats
;
577 /* Pairs with smp_store_release() below. */
578 stats
= smp_load_acquire(&sig
->stats
);
579 if (stats
|| thread_group_empty(tsk
))
582 /* No problem if kmem_cache_zalloc() fails */
583 stats_new
= kmem_cache_zalloc(taskstats_cache
, GFP_KERNEL
);
585 spin_lock_irq(&tsk
->sighand
->siglock
);
589 * Pairs with smp_store_release() above and order the
590 * kmem_cache_zalloc().
592 smp_store_release(&sig
->stats
, stats_new
);
596 spin_unlock_irq(&tsk
->sighand
->siglock
);
599 kmem_cache_free(taskstats_cache
, stats_new
);
604 /* Send pid data out on exit */
605 void taskstats_exit(struct task_struct
*tsk
, int group_dead
)
608 struct listener_list
*listeners
;
609 struct taskstats
*stats
;
610 struct sk_buff
*rep_skb
;
614 if (!family_registered
)
618 * Size includes space for nested attributes
620 size
= taskstats_packet_size();
622 is_thread_group
= !!taskstats_tgid_alloc(tsk
);
623 if (is_thread_group
) {
624 /* PID + STATS + TGID + STATS */
626 /* fill the tsk->signal->stats structure */
630 listeners
= raw_cpu_ptr(&listener_array
);
631 if (list_empty(&listeners
->list
))
634 rc
= prepare_reply(NULL
, TASKSTATS_CMD_NEW
, &rep_skb
, size
);
638 stats
= mk_reply(rep_skb
, TASKSTATS_TYPE_PID
,
639 task_pid_nr_ns(tsk
, &init_pid_ns
));
643 fill_stats(&init_user_ns
, &init_pid_ns
, tsk
, stats
);
645 stats
->ac_flag
|= AGROUP
;
648 * Doesn't matter if tsk is the leader or the last group member leaving
650 if (!is_thread_group
|| !group_dead
)
653 stats
= mk_reply(rep_skb
, TASKSTATS_TYPE_TGID
,
654 task_tgid_nr_ns(tsk
, &init_pid_ns
));
658 memcpy(stats
, tsk
->signal
->stats
, sizeof(*stats
));
661 send_cpu_listeners(rep_skb
, listeners
);
667 static const struct genl_ops taskstats_ops
[] = {
669 .cmd
= TASKSTATS_CMD_GET
,
670 .validate
= GENL_DONT_VALIDATE_STRICT
| GENL_DONT_VALIDATE_DUMP
,
671 .doit
= taskstats_user_cmd
,
672 .policy
= taskstats_cmd_get_policy
,
673 .maxattr
= ARRAY_SIZE(taskstats_cmd_get_policy
) - 1,
674 .flags
= GENL_ADMIN_PERM
,
677 .cmd
= CGROUPSTATS_CMD_GET
,
678 .validate
= GENL_DONT_VALIDATE_STRICT
| GENL_DONT_VALIDATE_DUMP
,
679 .doit
= cgroupstats_user_cmd
,
680 .policy
= cgroupstats_cmd_get_policy
,
681 .maxattr
= ARRAY_SIZE(cgroupstats_cmd_get_policy
) - 1,
685 static struct genl_family family __ro_after_init
= {
686 .name
= TASKSTATS_GENL_NAME
,
687 .version
= TASKSTATS_GENL_VERSION
,
688 .module
= THIS_MODULE
,
689 .ops
= taskstats_ops
,
690 .n_ops
= ARRAY_SIZE(taskstats_ops
),
691 .resv_start_op
= CGROUPSTATS_CMD_GET
+ 1,
695 /* Needed early in initialization */
696 void __init
taskstats_init_early(void)
700 taskstats_cache
= KMEM_CACHE(taskstats
, SLAB_PANIC
);
701 for_each_possible_cpu(i
) {
702 INIT_LIST_HEAD(&(per_cpu(listener_array
, i
).list
));
703 init_rwsem(&(per_cpu(listener_array
, i
).sem
));
707 static int __init
taskstats_init(void)
711 rc
= genl_register_family(&family
);
715 family_registered
= 1;
716 pr_info("registered taskstats version %d\n", TASKSTATS_GENL_VERSION
);
721 * late initcall ensures initialization of statistics collection
722 * mechanisms precedes initialization of the taskstats interface
724 late_initcall(taskstats_init
);