2 * Generic process-grouping system.
4 * Based originally on the cpuset system, extracted by Paul Menage
5 * Copyright (C) 2006 Google, Inc
7 * Notifications support
8 * Copyright (C) 2009 Nokia Corporation
9 * Author: Kirill A. Shutemov
11 * Copyright notices from the original cpuset code:
12 * --------------------------------------------------
13 * Copyright (C) 2003 BULL SA.
14 * Copyright (C) 2004-2006 Silicon Graphics, Inc.
16 * Portions derived from Patrick Mochel's sysfs code.
17 * sysfs is Copyright (c) 2001-3 Patrick Mochel
19 * 2003-10-10 Written by Simon Derr.
20 * 2003-10-22 Updates by Stephen Hemminger.
21 * 2004 May-July Rework by Paul Jackson.
22 * ---------------------------------------------------
24 * This file is subject to the terms and conditions of the GNU General Public
25 * License. See the file COPYING in the main directory of the Linux
26 * distribution for more details.
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
31 #include "cgroup-internal.h"
33 #include <linux/cred.h>
34 #include <linux/errno.h>
35 #include <linux/init_task.h>
36 #include <linux/kernel.h>
37 #include <linux/magic.h>
38 #include <linux/mutex.h>
39 #include <linux/mount.h>
40 #include <linux/pagemap.h>
41 #include <linux/proc_fs.h>
42 #include <linux/rcupdate.h>
43 #include <linux/sched.h>
44 #include <linux/sched/task.h>
45 #include <linux/slab.h>
46 #include <linux/spinlock.h>
47 #include <linux/percpu-rwsem.h>
48 #include <linux/string.h>
49 #include <linux/hashtable.h>
50 #include <linux/idr.h>
51 #include <linux/kthread.h>
52 #include <linux/atomic.h>
53 #include <linux/cpuset.h>
54 #include <linux/proc_ns.h>
55 #include <linux/nsproxy.h>
56 #include <linux/file.h>
57 #include <linux/fs_parser.h>
58 #include <linux/sched/cputime.h>
59 #include <linux/psi.h>
62 #define CREATE_TRACE_POINTS
63 #include <trace/events/cgroup.h>
65 #define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \
67 /* let's not notify more than 100 times per second */
68 #define CGROUP_FILE_NOTIFY_MIN_INTV DIV_ROUND_UP(HZ, 100)
71 * cgroup_mutex is the master lock. Any modification to cgroup or its
72 * hierarchy must be performed while holding it.
74 * css_set_lock protects task->cgroups pointer, the list of css_set
75 * objects, and the chain of tasks off each css_set.
77 * These locks are exported if CONFIG_PROVE_RCU so that accessors in
78 * cgroup.h can use them for lockdep annotations.
80 DEFINE_MUTEX(cgroup_mutex
);
81 DEFINE_SPINLOCK(css_set_lock
);
83 #ifdef CONFIG_PROVE_RCU
84 EXPORT_SYMBOL_GPL(cgroup_mutex
);
85 EXPORT_SYMBOL_GPL(css_set_lock
);
88 DEFINE_SPINLOCK(trace_cgroup_path_lock
);
89 char trace_cgroup_path
[TRACE_CGROUP_PATH_LEN
];
90 bool cgroup_debug __read_mostly
;
93 * Protects cgroup_idr and css_idr so that IDs can be released without
94 * grabbing cgroup_mutex.
96 static DEFINE_SPINLOCK(cgroup_idr_lock
);
99 * Protects cgroup_file->kn for !self csses. It synchronizes notifications
100 * against file removal/re-creation across css hiding.
102 static DEFINE_SPINLOCK(cgroup_file_kn_lock
);
104 DEFINE_PERCPU_RWSEM(cgroup_threadgroup_rwsem
);
106 #define cgroup_assert_mutex_or_rcu_locked() \
107 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
108 !lockdep_is_held(&cgroup_mutex), \
109 "cgroup_mutex or RCU read lock required");
112 * cgroup destruction makes heavy use of work items and there can be a lot
113 * of concurrent destructions. Use a separate workqueue so that cgroup
114 * destruction work items don't end up filling up max_active of system_wq
115 * which may lead to deadlock.
117 static struct workqueue_struct
*cgroup_destroy_wq
;
119 /* generate an array of cgroup subsystem pointers */
120 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys,
121 struct cgroup_subsys
*cgroup_subsys
[] = {
122 #include <linux/cgroup_subsys.h>
126 /* array of cgroup subsystem names */
127 #define SUBSYS(_x) [_x ## _cgrp_id] = #_x,
128 static const char *cgroup_subsys_name
[] = {
129 #include <linux/cgroup_subsys.h>
133 /* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */
135 DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key); \
136 DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key); \
137 EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key); \
138 EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key);
139 #include <linux/cgroup_subsys.h>
142 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key,
143 static struct static_key_true
*cgroup_subsys_enabled_key
[] = {
144 #include <linux/cgroup_subsys.h>
148 #define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key,
149 static struct static_key_true
*cgroup_subsys_on_dfl_key
[] = {
150 #include <linux/cgroup_subsys.h>
154 static DEFINE_PER_CPU(struct cgroup_rstat_cpu
, cgrp_dfl_root_rstat_cpu
);
156 /* the default hierarchy */
157 struct cgroup_root cgrp_dfl_root
= { .cgrp
.rstat_cpu
= &cgrp_dfl_root_rstat_cpu
};
158 EXPORT_SYMBOL_GPL(cgrp_dfl_root
);
161 * The default hierarchy always exists but is hidden until mounted for the
162 * first time. This is for backward compatibility.
164 static bool cgrp_dfl_visible
;
166 /* some controllers are not supported in the default hierarchy */
167 static u16 cgrp_dfl_inhibit_ss_mask
;
169 /* some controllers are implicitly enabled on the default hierarchy */
170 static u16 cgrp_dfl_implicit_ss_mask
;
172 /* some controllers can be threaded on the default hierarchy */
173 static u16 cgrp_dfl_threaded_ss_mask
;
175 /* The list of hierarchy roots */
176 LIST_HEAD(cgroup_roots
);
177 static int cgroup_root_count
;
179 /* hierarchy ID allocation and mapping, protected by cgroup_mutex */
180 static DEFINE_IDR(cgroup_hierarchy_idr
);
183 * Assign a monotonically increasing serial number to csses. It guarantees
184 * cgroups with bigger numbers are newer than those with smaller numbers.
185 * Also, as csses are always appended to the parent's ->children list, it
186 * guarantees that sibling csses are always sorted in the ascending serial
187 * number order on the list. Protected by cgroup_mutex.
189 static u64 css_serial_nr_next
= 1;
192 * These bitmasks identify subsystems with specific features to avoid
193 * having to do iterative checks repeatedly.
195 static u16 have_fork_callback __read_mostly
;
196 static u16 have_exit_callback __read_mostly
;
197 static u16 have_release_callback __read_mostly
;
198 static u16 have_canfork_callback __read_mostly
;
200 /* cgroup namespace for init task */
201 struct cgroup_namespace init_cgroup_ns
= {
202 .count
= REFCOUNT_INIT(2),
203 .user_ns
= &init_user_ns
,
204 .ns
.ops
= &cgroupns_operations
,
205 .ns
.inum
= PROC_CGROUP_INIT_INO
,
206 .root_cset
= &init_css_set
,
209 static struct file_system_type cgroup2_fs_type
;
210 static struct cftype cgroup_base_files
[];
212 static int cgroup_apply_control(struct cgroup
*cgrp
);
213 static void cgroup_finalize_control(struct cgroup
*cgrp
, int ret
);
214 static void css_task_iter_skip(struct css_task_iter
*it
,
215 struct task_struct
*task
);
216 static int cgroup_destroy_locked(struct cgroup
*cgrp
);
217 static struct cgroup_subsys_state
*css_create(struct cgroup
*cgrp
,
218 struct cgroup_subsys
*ss
);
219 static void css_release(struct percpu_ref
*ref
);
220 static void kill_css(struct cgroup_subsys_state
*css
);
221 static int cgroup_addrm_files(struct cgroup_subsys_state
*css
,
222 struct cgroup
*cgrp
, struct cftype cfts
[],
226 * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID
227 * @ssid: subsys ID of interest
229 * cgroup_subsys_enabled() can only be used with literal subsys names which
230 * is fine for individual subsystems but unsuitable for cgroup core. This
231 * is slower static_key_enabled() based test indexed by @ssid.
233 bool cgroup_ssid_enabled(int ssid
)
235 if (CGROUP_SUBSYS_COUNT
== 0)
238 return static_key_enabled(cgroup_subsys_enabled_key
[ssid
]);
242 * cgroup_on_dfl - test whether a cgroup is on the default hierarchy
243 * @cgrp: the cgroup of interest
245 * The default hierarchy is the v2 interface of cgroup and this function
246 * can be used to test whether a cgroup is on the default hierarchy for
247 * cases where a subsystem should behave differnetly depending on the
250 * List of changed behaviors:
252 * - Mount options "noprefix", "xattr", "clone_children", "release_agent"
253 * and "name" are disallowed.
255 * - When mounting an existing superblock, mount options should match.
257 * - Remount is disallowed.
259 * - rename(2) is disallowed.
261 * - "tasks" is removed. Everything should be at process granularity. Use
262 * "cgroup.procs" instead.
264 * - "cgroup.procs" is not sorted. pids will be unique unless they got
265 * recycled inbetween reads.
267 * - "release_agent" and "notify_on_release" are removed. Replacement
268 * notification mechanism will be implemented.
270 * - "cgroup.clone_children" is removed.
272 * - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup
273 * and its descendants contain no task; otherwise, 1. The file also
274 * generates kernfs notification which can be monitored through poll and
275 * [di]notify when the value of the file changes.
277 * - cpuset: tasks will be kept in empty cpusets when hotplug happens and
278 * take masks of ancestors with non-empty cpus/mems, instead of being
279 * moved to an ancestor.
281 * - cpuset: a task can be moved into an empty cpuset, and again it takes
282 * masks of ancestors.
284 * - memcg: use_hierarchy is on by default and the cgroup file for the flag
287 * - blkcg: blk-throttle becomes properly hierarchical.
289 * - debug: disallowed on the default hierarchy.
291 bool cgroup_on_dfl(const struct cgroup
*cgrp
)
293 return cgrp
->root
== &cgrp_dfl_root
;
296 /* IDR wrappers which synchronize using cgroup_idr_lock */
297 static int cgroup_idr_alloc(struct idr
*idr
, void *ptr
, int start
, int end
,
302 idr_preload(gfp_mask
);
303 spin_lock_bh(&cgroup_idr_lock
);
304 ret
= idr_alloc(idr
, ptr
, start
, end
, gfp_mask
& ~__GFP_DIRECT_RECLAIM
);
305 spin_unlock_bh(&cgroup_idr_lock
);
310 static void *cgroup_idr_replace(struct idr
*idr
, void *ptr
, int id
)
314 spin_lock_bh(&cgroup_idr_lock
);
315 ret
= idr_replace(idr
, ptr
, id
);
316 spin_unlock_bh(&cgroup_idr_lock
);
320 static void cgroup_idr_remove(struct idr
*idr
, int id
)
322 spin_lock_bh(&cgroup_idr_lock
);
324 spin_unlock_bh(&cgroup_idr_lock
);
327 static bool cgroup_has_tasks(struct cgroup
*cgrp
)
329 return cgrp
->nr_populated_csets
;
332 bool cgroup_is_threaded(struct cgroup
*cgrp
)
334 return cgrp
->dom_cgrp
!= cgrp
;
337 /* can @cgrp host both domain and threaded children? */
338 static bool cgroup_is_mixable(struct cgroup
*cgrp
)
341 * Root isn't under domain level resource control exempting it from
342 * the no-internal-process constraint, so it can serve as a thread
343 * root and a parent of resource domains at the same time.
345 return !cgroup_parent(cgrp
);
348 /* can @cgrp become a thread root? should always be true for a thread root */
349 static bool cgroup_can_be_thread_root(struct cgroup
*cgrp
)
351 /* mixables don't care */
352 if (cgroup_is_mixable(cgrp
))
355 /* domain roots can't be nested under threaded */
356 if (cgroup_is_threaded(cgrp
))
359 /* can only have either domain or threaded children */
360 if (cgrp
->nr_populated_domain_children
)
363 /* and no domain controllers can be enabled */
364 if (cgrp
->subtree_control
& ~cgrp_dfl_threaded_ss_mask
)
370 /* is @cgrp root of a threaded subtree? */
371 bool cgroup_is_thread_root(struct cgroup
*cgrp
)
373 /* thread root should be a domain */
374 if (cgroup_is_threaded(cgrp
))
377 /* a domain w/ threaded children is a thread root */
378 if (cgrp
->nr_threaded_children
)
382 * A domain which has tasks and explicit threaded controllers
383 * enabled is a thread root.
385 if (cgroup_has_tasks(cgrp
) &&
386 (cgrp
->subtree_control
& cgrp_dfl_threaded_ss_mask
))
392 /* a domain which isn't connected to the root w/o brekage can't be used */
393 static bool cgroup_is_valid_domain(struct cgroup
*cgrp
)
395 /* the cgroup itself can be a thread root */
396 if (cgroup_is_threaded(cgrp
))
399 /* but the ancestors can't be unless mixable */
400 while ((cgrp
= cgroup_parent(cgrp
))) {
401 if (!cgroup_is_mixable(cgrp
) && cgroup_is_thread_root(cgrp
))
403 if (cgroup_is_threaded(cgrp
))
410 /* subsystems visibly enabled on a cgroup */
411 static u16
cgroup_control(struct cgroup
*cgrp
)
413 struct cgroup
*parent
= cgroup_parent(cgrp
);
414 u16 root_ss_mask
= cgrp
->root
->subsys_mask
;
417 u16 ss_mask
= parent
->subtree_control
;
419 /* threaded cgroups can only have threaded controllers */
420 if (cgroup_is_threaded(cgrp
))
421 ss_mask
&= cgrp_dfl_threaded_ss_mask
;
425 if (cgroup_on_dfl(cgrp
))
426 root_ss_mask
&= ~(cgrp_dfl_inhibit_ss_mask
|
427 cgrp_dfl_implicit_ss_mask
);
431 /* subsystems enabled on a cgroup */
432 static u16
cgroup_ss_mask(struct cgroup
*cgrp
)
434 struct cgroup
*parent
= cgroup_parent(cgrp
);
437 u16 ss_mask
= parent
->subtree_ss_mask
;
439 /* threaded cgroups can only have threaded controllers */
440 if (cgroup_is_threaded(cgrp
))
441 ss_mask
&= cgrp_dfl_threaded_ss_mask
;
445 return cgrp
->root
->subsys_mask
;
449 * cgroup_css - obtain a cgroup's css for the specified subsystem
450 * @cgrp: the cgroup of interest
451 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
453 * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This
454 * function must be called either under cgroup_mutex or rcu_read_lock() and
455 * the caller is responsible for pinning the returned css if it wants to
456 * keep accessing it outside the said locks. This function may return
457 * %NULL if @cgrp doesn't have @subsys_id enabled.
459 static struct cgroup_subsys_state
*cgroup_css(struct cgroup
*cgrp
,
460 struct cgroup_subsys
*ss
)
463 return rcu_dereference_check(cgrp
->subsys
[ss
->id
],
464 lockdep_is_held(&cgroup_mutex
));
470 * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem
471 * @cgrp: the cgroup of interest
472 * @ss: the subsystem of interest
474 * Find and get @cgrp's css assocaited with @ss. If the css doesn't exist
475 * or is offline, %NULL is returned.
477 static struct cgroup_subsys_state
*cgroup_tryget_css(struct cgroup
*cgrp
,
478 struct cgroup_subsys
*ss
)
480 struct cgroup_subsys_state
*css
;
483 css
= cgroup_css(cgrp
, ss
);
484 if (css
&& !css_tryget_online(css
))
492 * cgroup_e_css_by_mask - obtain a cgroup's effective css for the specified ss
493 * @cgrp: the cgroup of interest
494 * @ss: the subsystem of interest (%NULL returns @cgrp->self)
496 * Similar to cgroup_css() but returns the effective css, which is defined
497 * as the matching css of the nearest ancestor including self which has @ss
498 * enabled. If @ss is associated with the hierarchy @cgrp is on, this
499 * function is guaranteed to return non-NULL css.
501 static struct cgroup_subsys_state
*cgroup_e_css_by_mask(struct cgroup
*cgrp
,
502 struct cgroup_subsys
*ss
)
504 lockdep_assert_held(&cgroup_mutex
);
510 * This function is used while updating css associations and thus
511 * can't test the csses directly. Test ss_mask.
513 while (!(cgroup_ss_mask(cgrp
) & (1 << ss
->id
))) {
514 cgrp
= cgroup_parent(cgrp
);
519 return cgroup_css(cgrp
, ss
);
523 * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem
524 * @cgrp: the cgroup of interest
525 * @ss: the subsystem of interest
527 * Find and get the effective css of @cgrp for @ss. The effective css is
528 * defined as the matching css of the nearest ancestor including self which
529 * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on,
530 * the root css is returned, so this function always returns a valid css.
532 * The returned css is not guaranteed to be online, and therefore it is the
533 * callers responsiblity to tryget a reference for it.
535 struct cgroup_subsys_state
*cgroup_e_css(struct cgroup
*cgrp
,
536 struct cgroup_subsys
*ss
)
538 struct cgroup_subsys_state
*css
;
541 css
= cgroup_css(cgrp
, ss
);
545 cgrp
= cgroup_parent(cgrp
);
548 return init_css_set
.subsys
[ss
->id
];
552 * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem
553 * @cgrp: the cgroup of interest
554 * @ss: the subsystem of interest
556 * Find and get the effective css of @cgrp for @ss. The effective css is
557 * defined as the matching css of the nearest ancestor including self which
558 * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on,
559 * the root css is returned, so this function always returns a valid css.
560 * The returned css must be put using css_put().
562 struct cgroup_subsys_state
*cgroup_get_e_css(struct cgroup
*cgrp
,
563 struct cgroup_subsys
*ss
)
565 struct cgroup_subsys_state
*css
;
570 css
= cgroup_css(cgrp
, ss
);
572 if (css
&& css_tryget_online(css
))
574 cgrp
= cgroup_parent(cgrp
);
577 css
= init_css_set
.subsys
[ss
->id
];
584 static void cgroup_get_live(struct cgroup
*cgrp
)
586 WARN_ON_ONCE(cgroup_is_dead(cgrp
));
587 css_get(&cgrp
->self
);
591 * __cgroup_task_count - count the number of tasks in a cgroup. The caller
592 * is responsible for taking the css_set_lock.
593 * @cgrp: the cgroup in question
595 int __cgroup_task_count(const struct cgroup
*cgrp
)
598 struct cgrp_cset_link
*link
;
600 lockdep_assert_held(&css_set_lock
);
602 list_for_each_entry(link
, &cgrp
->cset_links
, cset_link
)
603 count
+= link
->cset
->nr_tasks
;
609 * cgroup_task_count - count the number of tasks in a cgroup.
610 * @cgrp: the cgroup in question
612 int cgroup_task_count(const struct cgroup
*cgrp
)
616 spin_lock_irq(&css_set_lock
);
617 count
= __cgroup_task_count(cgrp
);
618 spin_unlock_irq(&css_set_lock
);
623 struct cgroup_subsys_state
*of_css(struct kernfs_open_file
*of
)
625 struct cgroup
*cgrp
= of
->kn
->parent
->priv
;
626 struct cftype
*cft
= of_cft(of
);
629 * This is open and unprotected implementation of cgroup_css().
630 * seq_css() is only called from a kernfs file operation which has
631 * an active reference on the file. Because all the subsystem
632 * files are drained before a css is disassociated with a cgroup,
633 * the matching css from the cgroup's subsys table is guaranteed to
634 * be and stay valid until the enclosing operation is complete.
637 return rcu_dereference_raw(cgrp
->subsys
[cft
->ss
->id
]);
641 EXPORT_SYMBOL_GPL(of_css
);
644 * for_each_css - iterate all css's of a cgroup
645 * @css: the iteration cursor
646 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
647 * @cgrp: the target cgroup to iterate css's of
649 * Should be called under cgroup_[tree_]mutex.
651 #define for_each_css(css, ssid, cgrp) \
652 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
653 if (!((css) = rcu_dereference_check( \
654 (cgrp)->subsys[(ssid)], \
655 lockdep_is_held(&cgroup_mutex)))) { } \
659 * for_each_e_css - iterate all effective css's of a cgroup
660 * @css: the iteration cursor
661 * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end
662 * @cgrp: the target cgroup to iterate css's of
664 * Should be called under cgroup_[tree_]mutex.
666 #define for_each_e_css(css, ssid, cgrp) \
667 for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \
668 if (!((css) = cgroup_e_css_by_mask(cgrp, \
669 cgroup_subsys[(ssid)]))) \
674 * do_each_subsys_mask - filter for_each_subsys with a bitmask
675 * @ss: the iteration cursor
676 * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end
677 * @ss_mask: the bitmask
679 * The block will only run for cases where the ssid-th bit (1 << ssid) of
682 #define do_each_subsys_mask(ss, ssid, ss_mask) do { \
683 unsigned long __ss_mask = (ss_mask); \
684 if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */ \
688 for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) { \
689 (ss) = cgroup_subsys[ssid]; \
692 #define while_each_subsys_mask() \
697 /* iterate over child cgrps, lock should be held throughout iteration */
698 #define cgroup_for_each_live_child(child, cgrp) \
699 list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \
700 if (({ lockdep_assert_held(&cgroup_mutex); \
701 cgroup_is_dead(child); })) \
705 /* walk live descendants in preorder */
706 #define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \
707 css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \
708 if (({ lockdep_assert_held(&cgroup_mutex); \
709 (dsct) = (d_css)->cgroup; \
710 cgroup_is_dead(dsct); })) \
714 /* walk live descendants in postorder */
715 #define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \
716 css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \
717 if (({ lockdep_assert_held(&cgroup_mutex); \
718 (dsct) = (d_css)->cgroup; \
719 cgroup_is_dead(dsct); })) \
724 * The default css_set - used by init and its children prior to any
725 * hierarchies being mounted. It contains a pointer to the root state
726 * for each subsystem. Also used to anchor the list of css_sets. Not
727 * reference-counted, to improve performance when child cgroups
728 * haven't been created.
730 struct css_set init_css_set
= {
731 .refcount
= REFCOUNT_INIT(1),
732 .dom_cset
= &init_css_set
,
733 .tasks
= LIST_HEAD_INIT(init_css_set
.tasks
),
734 .mg_tasks
= LIST_HEAD_INIT(init_css_set
.mg_tasks
),
735 .dying_tasks
= LIST_HEAD_INIT(init_css_set
.dying_tasks
),
736 .task_iters
= LIST_HEAD_INIT(init_css_set
.task_iters
),
737 .threaded_csets
= LIST_HEAD_INIT(init_css_set
.threaded_csets
),
738 .cgrp_links
= LIST_HEAD_INIT(init_css_set
.cgrp_links
),
739 .mg_preload_node
= LIST_HEAD_INIT(init_css_set
.mg_preload_node
),
740 .mg_node
= LIST_HEAD_INIT(init_css_set
.mg_node
),
743 * The following field is re-initialized when this cset gets linked
744 * in cgroup_init(). However, let's initialize the field
745 * statically too so that the default cgroup can be accessed safely
748 .dfl_cgrp
= &cgrp_dfl_root
.cgrp
,
751 static int css_set_count
= 1; /* 1 for init_css_set */
753 static bool css_set_threaded(struct css_set
*cset
)
755 return cset
->dom_cset
!= cset
;
759 * css_set_populated - does a css_set contain any tasks?
760 * @cset: target css_set
762 * css_set_populated() should be the same as !!cset->nr_tasks at steady
763 * state. However, css_set_populated() can be called while a task is being
764 * added to or removed from the linked list before the nr_tasks is
765 * properly updated. Hence, we can't just look at ->nr_tasks here.
767 static bool css_set_populated(struct css_set
*cset
)
769 lockdep_assert_held(&css_set_lock
);
771 return !list_empty(&cset
->tasks
) || !list_empty(&cset
->mg_tasks
);
775 * cgroup_update_populated - update the populated count of a cgroup
776 * @cgrp: the target cgroup
777 * @populated: inc or dec populated count
779 * One of the css_sets associated with @cgrp is either getting its first
780 * task or losing the last. Update @cgrp->nr_populated_* accordingly. The
781 * count is propagated towards root so that a given cgroup's
782 * nr_populated_children is zero iff none of its descendants contain any
785 * @cgrp's interface file "cgroup.populated" is zero if both
786 * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
787 * 1 otherwise. When the sum changes from or to zero, userland is notified
788 * that the content of the interface file has changed. This can be used to
789 * detect when @cgrp and its descendants become populated or empty.
791 static void cgroup_update_populated(struct cgroup
*cgrp
, bool populated
)
793 struct cgroup
*child
= NULL
;
794 int adj
= populated
? 1 : -1;
796 lockdep_assert_held(&css_set_lock
);
799 bool was_populated
= cgroup_is_populated(cgrp
);
802 cgrp
->nr_populated_csets
+= adj
;
804 if (cgroup_is_threaded(child
))
805 cgrp
->nr_populated_threaded_children
+= adj
;
807 cgrp
->nr_populated_domain_children
+= adj
;
810 if (was_populated
== cgroup_is_populated(cgrp
))
813 cgroup1_check_for_release(cgrp
);
814 TRACE_CGROUP_PATH(notify_populated
, cgrp
,
815 cgroup_is_populated(cgrp
));
816 cgroup_file_notify(&cgrp
->events_file
);
819 cgrp
= cgroup_parent(cgrp
);
824 * css_set_update_populated - update populated state of a css_set
825 * @cset: target css_set
826 * @populated: whether @cset is populated or depopulated
828 * @cset is either getting the first task or losing the last. Update the
829 * populated counters of all associated cgroups accordingly.
831 static void css_set_update_populated(struct css_set
*cset
, bool populated
)
833 struct cgrp_cset_link
*link
;
835 lockdep_assert_held(&css_set_lock
);
837 list_for_each_entry(link
, &cset
->cgrp_links
, cgrp_link
)
838 cgroup_update_populated(link
->cgrp
, populated
);
842 * @task is leaving, advance task iterators which are pointing to it so
843 * that they can resume at the next position. Advancing an iterator might
844 * remove it from the list, use safe walk. See css_task_iter_skip() for
847 static void css_set_skip_task_iters(struct css_set
*cset
,
848 struct task_struct
*task
)
850 struct css_task_iter
*it
, *pos
;
852 list_for_each_entry_safe(it
, pos
, &cset
->task_iters
, iters_node
)
853 css_task_iter_skip(it
, task
);
857 * css_set_move_task - move a task from one css_set to another
858 * @task: task being moved
859 * @from_cset: css_set @task currently belongs to (may be NULL)
860 * @to_cset: new css_set @task is being moved to (may be NULL)
861 * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks
863 * Move @task from @from_cset to @to_cset. If @task didn't belong to any
864 * css_set, @from_cset can be NULL. If @task is being disassociated
865 * instead of moved, @to_cset can be NULL.
867 * This function automatically handles populated counter updates and
868 * css_task_iter adjustments but the caller is responsible for managing
869 * @from_cset and @to_cset's reference counts.
871 static void css_set_move_task(struct task_struct
*task
,
872 struct css_set
*from_cset
, struct css_set
*to_cset
,
875 lockdep_assert_held(&css_set_lock
);
877 if (to_cset
&& !css_set_populated(to_cset
))
878 css_set_update_populated(to_cset
, true);
881 WARN_ON_ONCE(list_empty(&task
->cg_list
));
883 css_set_skip_task_iters(from_cset
, task
);
884 list_del_init(&task
->cg_list
);
885 if (!css_set_populated(from_cset
))
886 css_set_update_populated(from_cset
, false);
888 WARN_ON_ONCE(!list_empty(&task
->cg_list
));
893 * We are synchronized through cgroup_threadgroup_rwsem
894 * against PF_EXITING setting such that we can't race
895 * against cgroup_exit()/cgroup_free() dropping the css_set.
897 WARN_ON_ONCE(task
->flags
& PF_EXITING
);
899 cgroup_move_task(task
, to_cset
);
900 list_add_tail(&task
->cg_list
, use_mg_tasks
? &to_cset
->mg_tasks
:
906 * hash table for cgroup groups. This improves the performance to find
907 * an existing css_set. This hash doesn't (currently) take into
908 * account cgroups in empty hierarchies.
910 #define CSS_SET_HASH_BITS 7
911 static DEFINE_HASHTABLE(css_set_table
, CSS_SET_HASH_BITS
);
913 static unsigned long css_set_hash(struct cgroup_subsys_state
*css
[])
915 unsigned long key
= 0UL;
916 struct cgroup_subsys
*ss
;
919 for_each_subsys(ss
, i
)
920 key
+= (unsigned long)css
[i
];
921 key
= (key
>> 16) ^ key
;
926 void put_css_set_locked(struct css_set
*cset
)
928 struct cgrp_cset_link
*link
, *tmp_link
;
929 struct cgroup_subsys
*ss
;
932 lockdep_assert_held(&css_set_lock
);
934 if (!refcount_dec_and_test(&cset
->refcount
))
937 WARN_ON_ONCE(!list_empty(&cset
->threaded_csets
));
939 /* This css_set is dead. unlink it and release cgroup and css refs */
940 for_each_subsys(ss
, ssid
) {
941 list_del(&cset
->e_cset_node
[ssid
]);
942 css_put(cset
->subsys
[ssid
]);
944 hash_del(&cset
->hlist
);
947 list_for_each_entry_safe(link
, tmp_link
, &cset
->cgrp_links
, cgrp_link
) {
948 list_del(&link
->cset_link
);
949 list_del(&link
->cgrp_link
);
950 if (cgroup_parent(link
->cgrp
))
951 cgroup_put(link
->cgrp
);
955 if (css_set_threaded(cset
)) {
956 list_del(&cset
->threaded_csets_node
);
957 put_css_set_locked(cset
->dom_cset
);
960 kfree_rcu(cset
, rcu_head
);
964 * compare_css_sets - helper function for find_existing_css_set().
965 * @cset: candidate css_set being tested
966 * @old_cset: existing css_set for a task
967 * @new_cgrp: cgroup that's being entered by the task
968 * @template: desired set of css pointers in css_set (pre-calculated)
970 * Returns true if "cset" matches "old_cset" except for the hierarchy
971 * which "new_cgrp" belongs to, for which it should match "new_cgrp".
973 static bool compare_css_sets(struct css_set
*cset
,
974 struct css_set
*old_cset
,
975 struct cgroup
*new_cgrp
,
976 struct cgroup_subsys_state
*template[])
978 struct cgroup
*new_dfl_cgrp
;
979 struct list_head
*l1
, *l2
;
982 * On the default hierarchy, there can be csets which are
983 * associated with the same set of cgroups but different csses.
984 * Let's first ensure that csses match.
986 if (memcmp(template, cset
->subsys
, sizeof(cset
->subsys
)))
990 /* @cset's domain should match the default cgroup's */
991 if (cgroup_on_dfl(new_cgrp
))
992 new_dfl_cgrp
= new_cgrp
;
994 new_dfl_cgrp
= old_cset
->dfl_cgrp
;
996 if (new_dfl_cgrp
->dom_cgrp
!= cset
->dom_cset
->dfl_cgrp
)
1000 * Compare cgroup pointers in order to distinguish between
1001 * different cgroups in hierarchies. As different cgroups may
1002 * share the same effective css, this comparison is always
1005 l1
= &cset
->cgrp_links
;
1006 l2
= &old_cset
->cgrp_links
;
1008 struct cgrp_cset_link
*link1
, *link2
;
1009 struct cgroup
*cgrp1
, *cgrp2
;
1013 /* See if we reached the end - both lists are equal length. */
1014 if (l1
== &cset
->cgrp_links
) {
1015 BUG_ON(l2
!= &old_cset
->cgrp_links
);
1018 BUG_ON(l2
== &old_cset
->cgrp_links
);
1020 /* Locate the cgroups associated with these links. */
1021 link1
= list_entry(l1
, struct cgrp_cset_link
, cgrp_link
);
1022 link2
= list_entry(l2
, struct cgrp_cset_link
, cgrp_link
);
1023 cgrp1
= link1
->cgrp
;
1024 cgrp2
= link2
->cgrp
;
1025 /* Hierarchies should be linked in the same order. */
1026 BUG_ON(cgrp1
->root
!= cgrp2
->root
);
1029 * If this hierarchy is the hierarchy of the cgroup
1030 * that's changing, then we need to check that this
1031 * css_set points to the new cgroup; if it's any other
1032 * hierarchy, then this css_set should point to the
1033 * same cgroup as the old css_set.
1035 if (cgrp1
->root
== new_cgrp
->root
) {
1036 if (cgrp1
!= new_cgrp
)
1047 * find_existing_css_set - init css array and find the matching css_set
1048 * @old_cset: the css_set that we're using before the cgroup transition
1049 * @cgrp: the cgroup that we're moving into
1050 * @template: out param for the new set of csses, should be clear on entry
1052 static struct css_set
*find_existing_css_set(struct css_set
*old_cset
,
1053 struct cgroup
*cgrp
,
1054 struct cgroup_subsys_state
*template[])
1056 struct cgroup_root
*root
= cgrp
->root
;
1057 struct cgroup_subsys
*ss
;
1058 struct css_set
*cset
;
1063 * Build the set of subsystem state objects that we want to see in the
1064 * new css_set. while subsystems can change globally, the entries here
1065 * won't change, so no need for locking.
1067 for_each_subsys(ss
, i
) {
1068 if (root
->subsys_mask
& (1UL << i
)) {
1070 * @ss is in this hierarchy, so we want the
1071 * effective css from @cgrp.
1073 template[i
] = cgroup_e_css_by_mask(cgrp
, ss
);
1076 * @ss is not in this hierarchy, so we don't want
1077 * to change the css.
1079 template[i
] = old_cset
->subsys
[i
];
1083 key
= css_set_hash(template);
1084 hash_for_each_possible(css_set_table
, cset
, hlist
, key
) {
1085 if (!compare_css_sets(cset
, old_cset
, cgrp
, template))
1088 /* This css_set matches what we need */
1092 /* No existing cgroup group matched */
1096 static void free_cgrp_cset_links(struct list_head
*links_to_free
)
1098 struct cgrp_cset_link
*link
, *tmp_link
;
1100 list_for_each_entry_safe(link
, tmp_link
, links_to_free
, cset_link
) {
1101 list_del(&link
->cset_link
);
1107 * allocate_cgrp_cset_links - allocate cgrp_cset_links
1108 * @count: the number of links to allocate
1109 * @tmp_links: list_head the allocated links are put on
1111 * Allocate @count cgrp_cset_link structures and chain them on @tmp_links
1112 * through ->cset_link. Returns 0 on success or -errno.
1114 static int allocate_cgrp_cset_links(int count
, struct list_head
*tmp_links
)
1116 struct cgrp_cset_link
*link
;
1119 INIT_LIST_HEAD(tmp_links
);
1121 for (i
= 0; i
< count
; i
++) {
1122 link
= kzalloc(sizeof(*link
), GFP_KERNEL
);
1124 free_cgrp_cset_links(tmp_links
);
1127 list_add(&link
->cset_link
, tmp_links
);
1133 * link_css_set - a helper function to link a css_set to a cgroup
1134 * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links()
1135 * @cset: the css_set to be linked
1136 * @cgrp: the destination cgroup
1138 static void link_css_set(struct list_head
*tmp_links
, struct css_set
*cset
,
1139 struct cgroup
*cgrp
)
1141 struct cgrp_cset_link
*link
;
1143 BUG_ON(list_empty(tmp_links
));
1145 if (cgroup_on_dfl(cgrp
))
1146 cset
->dfl_cgrp
= cgrp
;
1148 link
= list_first_entry(tmp_links
, struct cgrp_cset_link
, cset_link
);
1153 * Always add links to the tail of the lists so that the lists are
1154 * in choronological order.
1156 list_move_tail(&link
->cset_link
, &cgrp
->cset_links
);
1157 list_add_tail(&link
->cgrp_link
, &cset
->cgrp_links
);
1159 if (cgroup_parent(cgrp
))
1160 cgroup_get_live(cgrp
);
1164 * find_css_set - return a new css_set with one cgroup updated
1165 * @old_cset: the baseline css_set
1166 * @cgrp: the cgroup to be updated
1168 * Return a new css_set that's equivalent to @old_cset, but with @cgrp
1169 * substituted into the appropriate hierarchy.
1171 static struct css_set
*find_css_set(struct css_set
*old_cset
,
1172 struct cgroup
*cgrp
)
1174 struct cgroup_subsys_state
*template[CGROUP_SUBSYS_COUNT
] = { };
1175 struct css_set
*cset
;
1176 struct list_head tmp_links
;
1177 struct cgrp_cset_link
*link
;
1178 struct cgroup_subsys
*ss
;
1182 lockdep_assert_held(&cgroup_mutex
);
1184 /* First see if we already have a cgroup group that matches
1185 * the desired set */
1186 spin_lock_irq(&css_set_lock
);
1187 cset
= find_existing_css_set(old_cset
, cgrp
, template);
1190 spin_unlock_irq(&css_set_lock
);
1195 cset
= kzalloc(sizeof(*cset
), GFP_KERNEL
);
1199 /* Allocate all the cgrp_cset_link objects that we'll need */
1200 if (allocate_cgrp_cset_links(cgroup_root_count
, &tmp_links
) < 0) {
1205 refcount_set(&cset
->refcount
, 1);
1206 cset
->dom_cset
= cset
;
1207 INIT_LIST_HEAD(&cset
->tasks
);
1208 INIT_LIST_HEAD(&cset
->mg_tasks
);
1209 INIT_LIST_HEAD(&cset
->dying_tasks
);
1210 INIT_LIST_HEAD(&cset
->task_iters
);
1211 INIT_LIST_HEAD(&cset
->threaded_csets
);
1212 INIT_HLIST_NODE(&cset
->hlist
);
1213 INIT_LIST_HEAD(&cset
->cgrp_links
);
1214 INIT_LIST_HEAD(&cset
->mg_preload_node
);
1215 INIT_LIST_HEAD(&cset
->mg_node
);
1217 /* Copy the set of subsystem state objects generated in
1218 * find_existing_css_set() */
1219 memcpy(cset
->subsys
, template, sizeof(cset
->subsys
));
1221 spin_lock_irq(&css_set_lock
);
1222 /* Add reference counts and links from the new css_set. */
1223 list_for_each_entry(link
, &old_cset
->cgrp_links
, cgrp_link
) {
1224 struct cgroup
*c
= link
->cgrp
;
1226 if (c
->root
== cgrp
->root
)
1228 link_css_set(&tmp_links
, cset
, c
);
1231 BUG_ON(!list_empty(&tmp_links
));
1235 /* Add @cset to the hash table */
1236 key
= css_set_hash(cset
->subsys
);
1237 hash_add(css_set_table
, &cset
->hlist
, key
);
1239 for_each_subsys(ss
, ssid
) {
1240 struct cgroup_subsys_state
*css
= cset
->subsys
[ssid
];
1242 list_add_tail(&cset
->e_cset_node
[ssid
],
1243 &css
->cgroup
->e_csets
[ssid
]);
1247 spin_unlock_irq(&css_set_lock
);
1250 * If @cset should be threaded, look up the matching dom_cset and
1251 * link them up. We first fully initialize @cset then look for the
1252 * dom_cset. It's simpler this way and safe as @cset is guaranteed
1253 * to stay empty until we return.
1255 if (cgroup_is_threaded(cset
->dfl_cgrp
)) {
1256 struct css_set
*dcset
;
1258 dcset
= find_css_set(cset
, cset
->dfl_cgrp
->dom_cgrp
);
1264 spin_lock_irq(&css_set_lock
);
1265 cset
->dom_cset
= dcset
;
1266 list_add_tail(&cset
->threaded_csets_node
,
1267 &dcset
->threaded_csets
);
1268 spin_unlock_irq(&css_set_lock
);
1274 struct cgroup_root
*cgroup_root_from_kf(struct kernfs_root
*kf_root
)
1276 struct cgroup
*root_cgrp
= kf_root
->kn
->priv
;
1278 return root_cgrp
->root
;
1281 static int cgroup_init_root_id(struct cgroup_root
*root
)
1285 lockdep_assert_held(&cgroup_mutex
);
1287 id
= idr_alloc_cyclic(&cgroup_hierarchy_idr
, root
, 0, 0, GFP_KERNEL
);
1291 root
->hierarchy_id
= id
;
1295 static void cgroup_exit_root_id(struct cgroup_root
*root
)
1297 lockdep_assert_held(&cgroup_mutex
);
1299 idr_remove(&cgroup_hierarchy_idr
, root
->hierarchy_id
);
1302 void cgroup_free_root(struct cgroup_root
*root
)
1307 static void cgroup_destroy_root(struct cgroup_root
*root
)
1309 struct cgroup
*cgrp
= &root
->cgrp
;
1310 struct cgrp_cset_link
*link
, *tmp_link
;
1312 trace_cgroup_destroy_root(root
);
1314 cgroup_lock_and_drain_offline(&cgrp_dfl_root
.cgrp
);
1316 BUG_ON(atomic_read(&root
->nr_cgrps
));
1317 BUG_ON(!list_empty(&cgrp
->self
.children
));
1319 /* Rebind all subsystems back to the default hierarchy */
1320 WARN_ON(rebind_subsystems(&cgrp_dfl_root
, root
->subsys_mask
));
1323 * Release all the links from cset_links to this hierarchy's
1326 spin_lock_irq(&css_set_lock
);
1328 list_for_each_entry_safe(link
, tmp_link
, &cgrp
->cset_links
, cset_link
) {
1329 list_del(&link
->cset_link
);
1330 list_del(&link
->cgrp_link
);
1334 spin_unlock_irq(&css_set_lock
);
1336 if (!list_empty(&root
->root_list
)) {
1337 list_del(&root
->root_list
);
1338 cgroup_root_count
--;
1341 cgroup_exit_root_id(root
);
1343 mutex_unlock(&cgroup_mutex
);
1345 kernfs_destroy_root(root
->kf_root
);
1346 cgroup_free_root(root
);
1350 * look up cgroup associated with current task's cgroup namespace on the
1351 * specified hierarchy
1353 static struct cgroup
*
1354 current_cgns_cgroup_from_root(struct cgroup_root
*root
)
1356 struct cgroup
*res
= NULL
;
1357 struct css_set
*cset
;
1359 lockdep_assert_held(&css_set_lock
);
1363 cset
= current
->nsproxy
->cgroup_ns
->root_cset
;
1364 if (cset
== &init_css_set
) {
1366 } else if (root
== &cgrp_dfl_root
) {
1367 res
= cset
->dfl_cgrp
;
1369 struct cgrp_cset_link
*link
;
1371 list_for_each_entry(link
, &cset
->cgrp_links
, cgrp_link
) {
1372 struct cgroup
*c
= link
->cgrp
;
1374 if (c
->root
== root
) {
1386 /* look up cgroup associated with given css_set on the specified hierarchy */
1387 static struct cgroup
*cset_cgroup_from_root(struct css_set
*cset
,
1388 struct cgroup_root
*root
)
1390 struct cgroup
*res
= NULL
;
1392 lockdep_assert_held(&cgroup_mutex
);
1393 lockdep_assert_held(&css_set_lock
);
1395 if (cset
== &init_css_set
) {
1397 } else if (root
== &cgrp_dfl_root
) {
1398 res
= cset
->dfl_cgrp
;
1400 struct cgrp_cset_link
*link
;
1402 list_for_each_entry(link
, &cset
->cgrp_links
, cgrp_link
) {
1403 struct cgroup
*c
= link
->cgrp
;
1405 if (c
->root
== root
) {
1417 * Return the cgroup for "task" from the given hierarchy. Must be
1418 * called with cgroup_mutex and css_set_lock held.
1420 struct cgroup
*task_cgroup_from_root(struct task_struct
*task
,
1421 struct cgroup_root
*root
)
1424 * No need to lock the task - since we hold css_set_lock the
1425 * task can't change groups.
1427 return cset_cgroup_from_root(task_css_set(task
), root
);
1431 * A task must hold cgroup_mutex to modify cgroups.
1433 * Any task can increment and decrement the count field without lock.
1434 * So in general, code holding cgroup_mutex can't rely on the count
1435 * field not changing. However, if the count goes to zero, then only
1436 * cgroup_attach_task() can increment it again. Because a count of zero
1437 * means that no tasks are currently attached, therefore there is no
1438 * way a task attached to that cgroup can fork (the other way to
1439 * increment the count). So code holding cgroup_mutex can safely
1440 * assume that if the count is zero, it will stay zero. Similarly, if
1441 * a task holds cgroup_mutex on a cgroup with zero count, it
1442 * knows that the cgroup won't be removed, as cgroup_rmdir()
1445 * A cgroup can only be deleted if both its 'count' of using tasks
1446 * is zero, and its list of 'children' cgroups is empty. Since all
1447 * tasks in the system use _some_ cgroup, and since there is always at
1448 * least one task in the system (init, pid == 1), therefore, root cgroup
1449 * always has either children cgroups and/or using tasks. So we don't
1450 * need a special hack to ensure that root cgroup cannot be deleted.
1452 * P.S. One more locking exception. RCU is used to guard the
1453 * update of a tasks cgroup pointer by cgroup_attach_task()
1456 static struct kernfs_syscall_ops cgroup_kf_syscall_ops
;
1458 static char *cgroup_file_name(struct cgroup
*cgrp
, const struct cftype
*cft
,
1461 struct cgroup_subsys
*ss
= cft
->ss
;
1463 if (cft
->ss
&& !(cft
->flags
& CFTYPE_NO_PREFIX
) &&
1464 !(cgrp
->root
->flags
& CGRP_ROOT_NOPREFIX
)) {
1465 const char *dbg
= (cft
->flags
& CFTYPE_DEBUG
) ? ".__DEBUG__." : "";
1467 snprintf(buf
, CGROUP_FILE_NAME_MAX
, "%s%s.%s",
1468 dbg
, cgroup_on_dfl(cgrp
) ? ss
->name
: ss
->legacy_name
,
1471 strscpy(buf
, cft
->name
, CGROUP_FILE_NAME_MAX
);
1477 * cgroup_file_mode - deduce file mode of a control file
1478 * @cft: the control file in question
1480 * S_IRUGO for read, S_IWUSR for write.
1482 static umode_t
cgroup_file_mode(const struct cftype
*cft
)
1486 if (cft
->read_u64
|| cft
->read_s64
|| cft
->seq_show
)
1489 if (cft
->write_u64
|| cft
->write_s64
|| cft
->write
) {
1490 if (cft
->flags
& CFTYPE_WORLD_WRITABLE
)
1500 * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask
1501 * @subtree_control: the new subtree_control mask to consider
1502 * @this_ss_mask: available subsystems
1504 * On the default hierarchy, a subsystem may request other subsystems to be
1505 * enabled together through its ->depends_on mask. In such cases, more
1506 * subsystems than specified in "cgroup.subtree_control" may be enabled.
1508 * This function calculates which subsystems need to be enabled if
1509 * @subtree_control is to be applied while restricted to @this_ss_mask.
1511 static u16
cgroup_calc_subtree_ss_mask(u16 subtree_control
, u16 this_ss_mask
)
1513 u16 cur_ss_mask
= subtree_control
;
1514 struct cgroup_subsys
*ss
;
1517 lockdep_assert_held(&cgroup_mutex
);
1519 cur_ss_mask
|= cgrp_dfl_implicit_ss_mask
;
1522 u16 new_ss_mask
= cur_ss_mask
;
1524 do_each_subsys_mask(ss
, ssid
, cur_ss_mask
) {
1525 new_ss_mask
|= ss
->depends_on
;
1526 } while_each_subsys_mask();
1529 * Mask out subsystems which aren't available. This can
1530 * happen only if some depended-upon subsystems were bound
1531 * to non-default hierarchies.
1533 new_ss_mask
&= this_ss_mask
;
1535 if (new_ss_mask
== cur_ss_mask
)
1537 cur_ss_mask
= new_ss_mask
;
1544 * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods
1545 * @kn: the kernfs_node being serviced
1547 * This helper undoes cgroup_kn_lock_live() and should be invoked before
1548 * the method finishes if locking succeeded. Note that once this function
1549 * returns the cgroup returned by cgroup_kn_lock_live() may become
1550 * inaccessible any time. If the caller intends to continue to access the
1551 * cgroup, it should pin it before invoking this function.
1553 void cgroup_kn_unlock(struct kernfs_node
*kn
)
1555 struct cgroup
*cgrp
;
1557 if (kernfs_type(kn
) == KERNFS_DIR
)
1560 cgrp
= kn
->parent
->priv
;
1562 mutex_unlock(&cgroup_mutex
);
1564 kernfs_unbreak_active_protection(kn
);
1569 * cgroup_kn_lock_live - locking helper for cgroup kernfs methods
1570 * @kn: the kernfs_node being serviced
1571 * @drain_offline: perform offline draining on the cgroup
1573 * This helper is to be used by a cgroup kernfs method currently servicing
1574 * @kn. It breaks the active protection, performs cgroup locking and
1575 * verifies that the associated cgroup is alive. Returns the cgroup if
1576 * alive; otherwise, %NULL. A successful return should be undone by a
1577 * matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the
1578 * cgroup is drained of offlining csses before return.
1580 * Any cgroup kernfs method implementation which requires locking the
1581 * associated cgroup should use this helper. It avoids nesting cgroup
1582 * locking under kernfs active protection and allows all kernfs operations
1583 * including self-removal.
1585 struct cgroup
*cgroup_kn_lock_live(struct kernfs_node
*kn
, bool drain_offline
)
1587 struct cgroup
*cgrp
;
1589 if (kernfs_type(kn
) == KERNFS_DIR
)
1592 cgrp
= kn
->parent
->priv
;
1595 * We're gonna grab cgroup_mutex which nests outside kernfs
1596 * active_ref. cgroup liveliness check alone provides enough
1597 * protection against removal. Ensure @cgrp stays accessible and
1598 * break the active_ref protection.
1600 if (!cgroup_tryget(cgrp
))
1602 kernfs_break_active_protection(kn
);
1605 cgroup_lock_and_drain_offline(cgrp
);
1607 mutex_lock(&cgroup_mutex
);
1609 if (!cgroup_is_dead(cgrp
))
1612 cgroup_kn_unlock(kn
);
1616 static void cgroup_rm_file(struct cgroup
*cgrp
, const struct cftype
*cft
)
1618 char name
[CGROUP_FILE_NAME_MAX
];
1620 lockdep_assert_held(&cgroup_mutex
);
1622 if (cft
->file_offset
) {
1623 struct cgroup_subsys_state
*css
= cgroup_css(cgrp
, cft
->ss
);
1624 struct cgroup_file
*cfile
= (void *)css
+ cft
->file_offset
;
1626 spin_lock_irq(&cgroup_file_kn_lock
);
1628 spin_unlock_irq(&cgroup_file_kn_lock
);
1630 del_timer_sync(&cfile
->notify_timer
);
1633 kernfs_remove_by_name(cgrp
->kn
, cgroup_file_name(cgrp
, cft
, name
));
1637 * css_clear_dir - remove subsys files in a cgroup directory
1640 static void css_clear_dir(struct cgroup_subsys_state
*css
)
1642 struct cgroup
*cgrp
= css
->cgroup
;
1643 struct cftype
*cfts
;
1645 if (!(css
->flags
& CSS_VISIBLE
))
1648 css
->flags
&= ~CSS_VISIBLE
;
1651 if (cgroup_on_dfl(cgrp
))
1652 cfts
= cgroup_base_files
;
1654 cfts
= cgroup1_base_files
;
1656 cgroup_addrm_files(css
, cgrp
, cfts
, false);
1658 list_for_each_entry(cfts
, &css
->ss
->cfts
, node
)
1659 cgroup_addrm_files(css
, cgrp
, cfts
, false);
1664 * css_populate_dir - create subsys files in a cgroup directory
1667 * On failure, no file is added.
1669 static int css_populate_dir(struct cgroup_subsys_state
*css
)
1671 struct cgroup
*cgrp
= css
->cgroup
;
1672 struct cftype
*cfts
, *failed_cfts
;
1675 if ((css
->flags
& CSS_VISIBLE
) || !cgrp
->kn
)
1679 if (cgroup_on_dfl(cgrp
))
1680 cfts
= cgroup_base_files
;
1682 cfts
= cgroup1_base_files
;
1684 ret
= cgroup_addrm_files(&cgrp
->self
, cgrp
, cfts
, true);
1688 list_for_each_entry(cfts
, &css
->ss
->cfts
, node
) {
1689 ret
= cgroup_addrm_files(css
, cgrp
, cfts
, true);
1697 css
->flags
|= CSS_VISIBLE
;
1701 list_for_each_entry(cfts
, &css
->ss
->cfts
, node
) {
1702 if (cfts
== failed_cfts
)
1704 cgroup_addrm_files(css
, cgrp
, cfts
, false);
1709 int rebind_subsystems(struct cgroup_root
*dst_root
, u16 ss_mask
)
1711 struct cgroup
*dcgrp
= &dst_root
->cgrp
;
1712 struct cgroup_subsys
*ss
;
1715 lockdep_assert_held(&cgroup_mutex
);
1717 do_each_subsys_mask(ss
, ssid
, ss_mask
) {
1719 * If @ss has non-root csses attached to it, can't move.
1720 * If @ss is an implicit controller, it is exempt from this
1721 * rule and can be stolen.
1723 if (css_next_child(NULL
, cgroup_css(&ss
->root
->cgrp
, ss
)) &&
1724 !ss
->implicit_on_dfl
)
1727 /* can't move between two non-dummy roots either */
1728 if (ss
->root
!= &cgrp_dfl_root
&& dst_root
!= &cgrp_dfl_root
)
1730 } while_each_subsys_mask();
1732 do_each_subsys_mask(ss
, ssid
, ss_mask
) {
1733 struct cgroup_root
*src_root
= ss
->root
;
1734 struct cgroup
*scgrp
= &src_root
->cgrp
;
1735 struct cgroup_subsys_state
*css
= cgroup_css(scgrp
, ss
);
1736 struct css_set
*cset
;
1738 WARN_ON(!css
|| cgroup_css(dcgrp
, ss
));
1740 /* disable from the source */
1741 src_root
->subsys_mask
&= ~(1 << ssid
);
1742 WARN_ON(cgroup_apply_control(scgrp
));
1743 cgroup_finalize_control(scgrp
, 0);
1746 RCU_INIT_POINTER(scgrp
->subsys
[ssid
], NULL
);
1747 rcu_assign_pointer(dcgrp
->subsys
[ssid
], css
);
1748 ss
->root
= dst_root
;
1749 css
->cgroup
= dcgrp
;
1751 spin_lock_irq(&css_set_lock
);
1752 hash_for_each(css_set_table
, i
, cset
, hlist
)
1753 list_move_tail(&cset
->e_cset_node
[ss
->id
],
1754 &dcgrp
->e_csets
[ss
->id
]);
1755 spin_unlock_irq(&css_set_lock
);
1757 /* default hierarchy doesn't enable controllers by default */
1758 dst_root
->subsys_mask
|= 1 << ssid
;
1759 if (dst_root
== &cgrp_dfl_root
) {
1760 static_branch_enable(cgroup_subsys_on_dfl_key
[ssid
]);
1762 dcgrp
->subtree_control
|= 1 << ssid
;
1763 static_branch_disable(cgroup_subsys_on_dfl_key
[ssid
]);
1766 ret
= cgroup_apply_control(dcgrp
);
1768 pr_warn("partial failure to rebind %s controller (err=%d)\n",
1773 } while_each_subsys_mask();
1775 kernfs_activate(dcgrp
->kn
);
1779 int cgroup_show_path(struct seq_file
*sf
, struct kernfs_node
*kf_node
,
1780 struct kernfs_root
*kf_root
)
1784 struct cgroup_root
*kf_cgroot
= cgroup_root_from_kf(kf_root
);
1785 struct cgroup
*ns_cgroup
;
1787 buf
= kmalloc(PATH_MAX
, GFP_KERNEL
);
1791 spin_lock_irq(&css_set_lock
);
1792 ns_cgroup
= current_cgns_cgroup_from_root(kf_cgroot
);
1793 len
= kernfs_path_from_node(kf_node
, ns_cgroup
->kn
, buf
, PATH_MAX
);
1794 spin_unlock_irq(&css_set_lock
);
1796 if (len
>= PATH_MAX
)
1799 seq_escape(sf
, buf
, " \t\n\\");
1806 enum cgroup2_param
{
1808 Opt_memory_localevents
,
1809 Opt_memory_recursiveprot
,
1813 static const struct fs_parameter_spec cgroup2_fs_parameters
[] = {
1814 fsparam_flag("nsdelegate", Opt_nsdelegate
),
1815 fsparam_flag("memory_localevents", Opt_memory_localevents
),
1816 fsparam_flag("memory_recursiveprot", Opt_memory_recursiveprot
),
1820 static int cgroup2_parse_param(struct fs_context
*fc
, struct fs_parameter
*param
)
1822 struct cgroup_fs_context
*ctx
= cgroup_fc2context(fc
);
1823 struct fs_parse_result result
;
1826 opt
= fs_parse(fc
, cgroup2_fs_parameters
, param
, &result
);
1831 case Opt_nsdelegate
:
1832 ctx
->flags
|= CGRP_ROOT_NS_DELEGATE
;
1834 case Opt_memory_localevents
:
1835 ctx
->flags
|= CGRP_ROOT_MEMORY_LOCAL_EVENTS
;
1837 case Opt_memory_recursiveprot
:
1838 ctx
->flags
|= CGRP_ROOT_MEMORY_RECURSIVE_PROT
;
1844 static void apply_cgroup_root_flags(unsigned int root_flags
)
1846 if (current
->nsproxy
->cgroup_ns
== &init_cgroup_ns
) {
1847 if (root_flags
& CGRP_ROOT_NS_DELEGATE
)
1848 cgrp_dfl_root
.flags
|= CGRP_ROOT_NS_DELEGATE
;
1850 cgrp_dfl_root
.flags
&= ~CGRP_ROOT_NS_DELEGATE
;
1852 if (root_flags
& CGRP_ROOT_MEMORY_LOCAL_EVENTS
)
1853 cgrp_dfl_root
.flags
|= CGRP_ROOT_MEMORY_LOCAL_EVENTS
;
1855 cgrp_dfl_root
.flags
&= ~CGRP_ROOT_MEMORY_LOCAL_EVENTS
;
1857 if (root_flags
& CGRP_ROOT_MEMORY_RECURSIVE_PROT
)
1858 cgrp_dfl_root
.flags
|= CGRP_ROOT_MEMORY_RECURSIVE_PROT
;
1860 cgrp_dfl_root
.flags
&= ~CGRP_ROOT_MEMORY_RECURSIVE_PROT
;
1864 static int cgroup_show_options(struct seq_file
*seq
, struct kernfs_root
*kf_root
)
1866 if (cgrp_dfl_root
.flags
& CGRP_ROOT_NS_DELEGATE
)
1867 seq_puts(seq
, ",nsdelegate");
1868 if (cgrp_dfl_root
.flags
& CGRP_ROOT_MEMORY_LOCAL_EVENTS
)
1869 seq_puts(seq
, ",memory_localevents");
1870 if (cgrp_dfl_root
.flags
& CGRP_ROOT_MEMORY_RECURSIVE_PROT
)
1871 seq_puts(seq
, ",memory_recursiveprot");
1875 static int cgroup_reconfigure(struct fs_context
*fc
)
1877 struct cgroup_fs_context
*ctx
= cgroup_fc2context(fc
);
1879 apply_cgroup_root_flags(ctx
->flags
);
1883 static void init_cgroup_housekeeping(struct cgroup
*cgrp
)
1885 struct cgroup_subsys
*ss
;
1888 INIT_LIST_HEAD(&cgrp
->self
.sibling
);
1889 INIT_LIST_HEAD(&cgrp
->self
.children
);
1890 INIT_LIST_HEAD(&cgrp
->cset_links
);
1891 INIT_LIST_HEAD(&cgrp
->pidlists
);
1892 mutex_init(&cgrp
->pidlist_mutex
);
1893 cgrp
->self
.cgroup
= cgrp
;
1894 cgrp
->self
.flags
|= CSS_ONLINE
;
1895 cgrp
->dom_cgrp
= cgrp
;
1896 cgrp
->max_descendants
= INT_MAX
;
1897 cgrp
->max_depth
= INT_MAX
;
1898 INIT_LIST_HEAD(&cgrp
->rstat_css_list
);
1899 prev_cputime_init(&cgrp
->prev_cputime
);
1901 for_each_subsys(ss
, ssid
)
1902 INIT_LIST_HEAD(&cgrp
->e_csets
[ssid
]);
1904 init_waitqueue_head(&cgrp
->offline_waitq
);
1905 INIT_WORK(&cgrp
->release_agent_work
, cgroup1_release_agent
);
1908 void init_cgroup_root(struct cgroup_fs_context
*ctx
)
1910 struct cgroup_root
*root
= ctx
->root
;
1911 struct cgroup
*cgrp
= &root
->cgrp
;
1913 INIT_LIST_HEAD(&root
->root_list
);
1914 atomic_set(&root
->nr_cgrps
, 1);
1916 init_cgroup_housekeeping(cgrp
);
1918 root
->flags
= ctx
->flags
;
1919 if (ctx
->release_agent
)
1920 strscpy(root
->release_agent_path
, ctx
->release_agent
, PATH_MAX
);
1922 strscpy(root
->name
, ctx
->name
, MAX_CGROUP_ROOT_NAMELEN
);
1923 if (ctx
->cpuset_clone_children
)
1924 set_bit(CGRP_CPUSET_CLONE_CHILDREN
, &root
->cgrp
.flags
);
1927 int cgroup_setup_root(struct cgroup_root
*root
, u16 ss_mask
)
1929 LIST_HEAD(tmp_links
);
1930 struct cgroup
*root_cgrp
= &root
->cgrp
;
1931 struct kernfs_syscall_ops
*kf_sops
;
1932 struct css_set
*cset
;
1935 lockdep_assert_held(&cgroup_mutex
);
1937 ret
= percpu_ref_init(&root_cgrp
->self
.refcnt
, css_release
,
1943 * We're accessing css_set_count without locking css_set_lock here,
1944 * but that's OK - it can only be increased by someone holding
1945 * cgroup_lock, and that's us. Later rebinding may disable
1946 * controllers on the default hierarchy and thus create new csets,
1947 * which can't be more than the existing ones. Allocate 2x.
1949 ret
= allocate_cgrp_cset_links(2 * css_set_count
, &tmp_links
);
1953 ret
= cgroup_init_root_id(root
);
1957 kf_sops
= root
== &cgrp_dfl_root
?
1958 &cgroup_kf_syscall_ops
: &cgroup1_kf_syscall_ops
;
1960 root
->kf_root
= kernfs_create_root(kf_sops
,
1961 KERNFS_ROOT_CREATE_DEACTIVATED
|
1962 KERNFS_ROOT_SUPPORT_EXPORTOP
|
1963 KERNFS_ROOT_SUPPORT_USER_XATTR
,
1965 if (IS_ERR(root
->kf_root
)) {
1966 ret
= PTR_ERR(root
->kf_root
);
1969 root_cgrp
->kn
= root
->kf_root
->kn
;
1970 WARN_ON_ONCE(cgroup_ino(root_cgrp
) != 1);
1971 root_cgrp
->ancestor_ids
[0] = cgroup_id(root_cgrp
);
1973 ret
= css_populate_dir(&root_cgrp
->self
);
1977 ret
= rebind_subsystems(root
, ss_mask
);
1981 ret
= cgroup_bpf_inherit(root_cgrp
);
1984 trace_cgroup_setup_root(root
);
1987 * There must be no failure case after here, since rebinding takes
1988 * care of subsystems' refcounts, which are explicitly dropped in
1989 * the failure exit path.
1991 list_add(&root
->root_list
, &cgroup_roots
);
1992 cgroup_root_count
++;
1995 * Link the root cgroup in this hierarchy into all the css_set
1998 spin_lock_irq(&css_set_lock
);
1999 hash_for_each(css_set_table
, i
, cset
, hlist
) {
2000 link_css_set(&tmp_links
, cset
, root_cgrp
);
2001 if (css_set_populated(cset
))
2002 cgroup_update_populated(root_cgrp
, true);
2004 spin_unlock_irq(&css_set_lock
);
2006 BUG_ON(!list_empty(&root_cgrp
->self
.children
));
2007 BUG_ON(atomic_read(&root
->nr_cgrps
) != 1);
2013 kernfs_destroy_root(root
->kf_root
);
2014 root
->kf_root
= NULL
;
2016 cgroup_exit_root_id(root
);
2018 percpu_ref_exit(&root_cgrp
->self
.refcnt
);
2020 free_cgrp_cset_links(&tmp_links
);
2024 int cgroup_do_get_tree(struct fs_context
*fc
)
2026 struct cgroup_fs_context
*ctx
= cgroup_fc2context(fc
);
2029 ctx
->kfc
.root
= ctx
->root
->kf_root
;
2030 if (fc
->fs_type
== &cgroup2_fs_type
)
2031 ctx
->kfc
.magic
= CGROUP2_SUPER_MAGIC
;
2033 ctx
->kfc
.magic
= CGROUP_SUPER_MAGIC
;
2034 ret
= kernfs_get_tree(fc
);
2037 * In non-init cgroup namespace, instead of root cgroup's dentry,
2038 * we return the dentry corresponding to the cgroupns->root_cgrp.
2040 if (!ret
&& ctx
->ns
!= &init_cgroup_ns
) {
2041 struct dentry
*nsdentry
;
2042 struct super_block
*sb
= fc
->root
->d_sb
;
2043 struct cgroup
*cgrp
;
2045 mutex_lock(&cgroup_mutex
);
2046 spin_lock_irq(&css_set_lock
);
2048 cgrp
= cset_cgroup_from_root(ctx
->ns
->root_cset
, ctx
->root
);
2050 spin_unlock_irq(&css_set_lock
);
2051 mutex_unlock(&cgroup_mutex
);
2053 nsdentry
= kernfs_node_dentry(cgrp
->kn
, sb
);
2055 if (IS_ERR(nsdentry
)) {
2056 deactivate_locked_super(sb
);
2057 ret
= PTR_ERR(nsdentry
);
2060 fc
->root
= nsdentry
;
2063 if (!ctx
->kfc
.new_sb_created
)
2064 cgroup_put(&ctx
->root
->cgrp
);
2070 * Destroy a cgroup filesystem context.
2072 static void cgroup_fs_context_free(struct fs_context
*fc
)
2074 struct cgroup_fs_context
*ctx
= cgroup_fc2context(fc
);
2077 kfree(ctx
->release_agent
);
2078 put_cgroup_ns(ctx
->ns
);
2079 kernfs_free_fs_context(fc
);
2083 static int cgroup_get_tree(struct fs_context
*fc
)
2085 struct cgroup_fs_context
*ctx
= cgroup_fc2context(fc
);
2088 cgrp_dfl_visible
= true;
2089 cgroup_get_live(&cgrp_dfl_root
.cgrp
);
2090 ctx
->root
= &cgrp_dfl_root
;
2092 ret
= cgroup_do_get_tree(fc
);
2094 apply_cgroup_root_flags(ctx
->flags
);
2098 static const struct fs_context_operations cgroup_fs_context_ops
= {
2099 .free
= cgroup_fs_context_free
,
2100 .parse_param
= cgroup2_parse_param
,
2101 .get_tree
= cgroup_get_tree
,
2102 .reconfigure
= cgroup_reconfigure
,
2105 static const struct fs_context_operations cgroup1_fs_context_ops
= {
2106 .free
= cgroup_fs_context_free
,
2107 .parse_param
= cgroup1_parse_param
,
2108 .get_tree
= cgroup1_get_tree
,
2109 .reconfigure
= cgroup1_reconfigure
,
2113 * Initialise the cgroup filesystem creation/reconfiguration context. Notably,
2114 * we select the namespace we're going to use.
2116 static int cgroup_init_fs_context(struct fs_context
*fc
)
2118 struct cgroup_fs_context
*ctx
;
2120 ctx
= kzalloc(sizeof(struct cgroup_fs_context
), GFP_KERNEL
);
2124 ctx
->ns
= current
->nsproxy
->cgroup_ns
;
2125 get_cgroup_ns(ctx
->ns
);
2126 fc
->fs_private
= &ctx
->kfc
;
2127 if (fc
->fs_type
== &cgroup2_fs_type
)
2128 fc
->ops
= &cgroup_fs_context_ops
;
2130 fc
->ops
= &cgroup1_fs_context_ops
;
2131 put_user_ns(fc
->user_ns
);
2132 fc
->user_ns
= get_user_ns(ctx
->ns
->user_ns
);
2137 static void cgroup_kill_sb(struct super_block
*sb
)
2139 struct kernfs_root
*kf_root
= kernfs_root_from_sb(sb
);
2140 struct cgroup_root
*root
= cgroup_root_from_kf(kf_root
);
2143 * If @root doesn't have any children, start killing it.
2144 * This prevents new mounts by disabling percpu_ref_tryget_live().
2145 * cgroup_mount() may wait for @root's release.
2147 * And don't kill the default root.
2149 if (list_empty(&root
->cgrp
.self
.children
) && root
!= &cgrp_dfl_root
&&
2150 !percpu_ref_is_dying(&root
->cgrp
.self
.refcnt
))
2151 percpu_ref_kill(&root
->cgrp
.self
.refcnt
);
2152 cgroup_put(&root
->cgrp
);
2156 struct file_system_type cgroup_fs_type
= {
2158 .init_fs_context
= cgroup_init_fs_context
,
2159 .parameters
= cgroup1_fs_parameters
,
2160 .kill_sb
= cgroup_kill_sb
,
2161 .fs_flags
= FS_USERNS_MOUNT
,
2164 static struct file_system_type cgroup2_fs_type
= {
2166 .init_fs_context
= cgroup_init_fs_context
,
2167 .parameters
= cgroup2_fs_parameters
,
2168 .kill_sb
= cgroup_kill_sb
,
2169 .fs_flags
= FS_USERNS_MOUNT
,
2172 #ifdef CONFIG_CPUSETS
2173 static const struct fs_context_operations cpuset_fs_context_ops
= {
2174 .get_tree
= cgroup1_get_tree
,
2175 .free
= cgroup_fs_context_free
,
2179 * This is ugly, but preserves the userspace API for existing cpuset
2180 * users. If someone tries to mount the "cpuset" filesystem, we
2181 * silently switch it to mount "cgroup" instead
2183 static int cpuset_init_fs_context(struct fs_context
*fc
)
2185 char *agent
= kstrdup("/sbin/cpuset_release_agent", GFP_USER
);
2186 struct cgroup_fs_context
*ctx
;
2189 err
= cgroup_init_fs_context(fc
);
2195 fc
->ops
= &cpuset_fs_context_ops
;
2197 ctx
= cgroup_fc2context(fc
);
2198 ctx
->subsys_mask
= 1 << cpuset_cgrp_id
;
2199 ctx
->flags
|= CGRP_ROOT_NOPREFIX
;
2200 ctx
->release_agent
= agent
;
2202 get_filesystem(&cgroup_fs_type
);
2203 put_filesystem(fc
->fs_type
);
2204 fc
->fs_type
= &cgroup_fs_type
;
2209 static struct file_system_type cpuset_fs_type
= {
2211 .init_fs_context
= cpuset_init_fs_context
,
2212 .fs_flags
= FS_USERNS_MOUNT
,
2216 int cgroup_path_ns_locked(struct cgroup
*cgrp
, char *buf
, size_t buflen
,
2217 struct cgroup_namespace
*ns
)
2219 struct cgroup
*root
= cset_cgroup_from_root(ns
->root_cset
, cgrp
->root
);
2221 return kernfs_path_from_node(cgrp
->kn
, root
->kn
, buf
, buflen
);
2224 int cgroup_path_ns(struct cgroup
*cgrp
, char *buf
, size_t buflen
,
2225 struct cgroup_namespace
*ns
)
2229 mutex_lock(&cgroup_mutex
);
2230 spin_lock_irq(&css_set_lock
);
2232 ret
= cgroup_path_ns_locked(cgrp
, buf
, buflen
, ns
);
2234 spin_unlock_irq(&css_set_lock
);
2235 mutex_unlock(&cgroup_mutex
);
2239 EXPORT_SYMBOL_GPL(cgroup_path_ns
);
2242 * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy
2243 * @task: target task
2244 * @buf: the buffer to write the path into
2245 * @buflen: the length of the buffer
2247 * Determine @task's cgroup on the first (the one with the lowest non-zero
2248 * hierarchy_id) cgroup hierarchy and copy its path into @buf. This
2249 * function grabs cgroup_mutex and shouldn't be used inside locks used by
2250 * cgroup controller callbacks.
2252 * Return value is the same as kernfs_path().
2254 int task_cgroup_path(struct task_struct
*task
, char *buf
, size_t buflen
)
2256 struct cgroup_root
*root
;
2257 struct cgroup
*cgrp
;
2258 int hierarchy_id
= 1;
2261 mutex_lock(&cgroup_mutex
);
2262 spin_lock_irq(&css_set_lock
);
2264 root
= idr_get_next(&cgroup_hierarchy_idr
, &hierarchy_id
);
2267 cgrp
= task_cgroup_from_root(task
, root
);
2268 ret
= cgroup_path_ns_locked(cgrp
, buf
, buflen
, &init_cgroup_ns
);
2270 /* if no hierarchy exists, everyone is in "/" */
2271 ret
= strlcpy(buf
, "/", buflen
);
2274 spin_unlock_irq(&css_set_lock
);
2275 mutex_unlock(&cgroup_mutex
);
2278 EXPORT_SYMBOL_GPL(task_cgroup_path
);
2281 * cgroup_migrate_add_task - add a migration target task to a migration context
2282 * @task: target task
2283 * @mgctx: target migration context
2285 * Add @task, which is a migration target, to @mgctx->tset. This function
2286 * becomes noop if @task doesn't need to be migrated. @task's css_set
2287 * should have been added as a migration source and @task->cg_list will be
2288 * moved from the css_set's tasks list to mg_tasks one.
2290 static void cgroup_migrate_add_task(struct task_struct
*task
,
2291 struct cgroup_mgctx
*mgctx
)
2293 struct css_set
*cset
;
2295 lockdep_assert_held(&css_set_lock
);
2297 /* @task either already exited or can't exit until the end */
2298 if (task
->flags
& PF_EXITING
)
2301 /* cgroup_threadgroup_rwsem protects racing against forks */
2302 WARN_ON_ONCE(list_empty(&task
->cg_list
));
2304 cset
= task_css_set(task
);
2305 if (!cset
->mg_src_cgrp
)
2308 mgctx
->tset
.nr_tasks
++;
2310 list_move_tail(&task
->cg_list
, &cset
->mg_tasks
);
2311 if (list_empty(&cset
->mg_node
))
2312 list_add_tail(&cset
->mg_node
,
2313 &mgctx
->tset
.src_csets
);
2314 if (list_empty(&cset
->mg_dst_cset
->mg_node
))
2315 list_add_tail(&cset
->mg_dst_cset
->mg_node
,
2316 &mgctx
->tset
.dst_csets
);
2320 * cgroup_taskset_first - reset taskset and return the first task
2321 * @tset: taskset of interest
2322 * @dst_cssp: output variable for the destination css
2324 * @tset iteration is initialized and the first task is returned.
2326 struct task_struct
*cgroup_taskset_first(struct cgroup_taskset
*tset
,
2327 struct cgroup_subsys_state
**dst_cssp
)
2329 tset
->cur_cset
= list_first_entry(tset
->csets
, struct css_set
, mg_node
);
2330 tset
->cur_task
= NULL
;
2332 return cgroup_taskset_next(tset
, dst_cssp
);
2336 * cgroup_taskset_next - iterate to the next task in taskset
2337 * @tset: taskset of interest
2338 * @dst_cssp: output variable for the destination css
2340 * Return the next task in @tset. Iteration must have been initialized
2341 * with cgroup_taskset_first().
2343 struct task_struct
*cgroup_taskset_next(struct cgroup_taskset
*tset
,
2344 struct cgroup_subsys_state
**dst_cssp
)
2346 struct css_set
*cset
= tset
->cur_cset
;
2347 struct task_struct
*task
= tset
->cur_task
;
2349 while (&cset
->mg_node
!= tset
->csets
) {
2351 task
= list_first_entry(&cset
->mg_tasks
,
2352 struct task_struct
, cg_list
);
2354 task
= list_next_entry(task
, cg_list
);
2356 if (&task
->cg_list
!= &cset
->mg_tasks
) {
2357 tset
->cur_cset
= cset
;
2358 tset
->cur_task
= task
;
2361 * This function may be called both before and
2362 * after cgroup_taskset_migrate(). The two cases
2363 * can be distinguished by looking at whether @cset
2364 * has its ->mg_dst_cset set.
2366 if (cset
->mg_dst_cset
)
2367 *dst_cssp
= cset
->mg_dst_cset
->subsys
[tset
->ssid
];
2369 *dst_cssp
= cset
->subsys
[tset
->ssid
];
2374 cset
= list_next_entry(cset
, mg_node
);
2382 * cgroup_taskset_migrate - migrate a taskset
2383 * @mgctx: migration context
2385 * Migrate tasks in @mgctx as setup by migration preparation functions.
2386 * This function fails iff one of the ->can_attach callbacks fails and
2387 * guarantees that either all or none of the tasks in @mgctx are migrated.
2388 * @mgctx is consumed regardless of success.
2390 static int cgroup_migrate_execute(struct cgroup_mgctx
*mgctx
)
2392 struct cgroup_taskset
*tset
= &mgctx
->tset
;
2393 struct cgroup_subsys
*ss
;
2394 struct task_struct
*task
, *tmp_task
;
2395 struct css_set
*cset
, *tmp_cset
;
2396 int ssid
, failed_ssid
, ret
;
2398 /* check that we can legitimately attach to the cgroup */
2399 if (tset
->nr_tasks
) {
2400 do_each_subsys_mask(ss
, ssid
, mgctx
->ss_mask
) {
2401 if (ss
->can_attach
) {
2403 ret
= ss
->can_attach(tset
);
2406 goto out_cancel_attach
;
2409 } while_each_subsys_mask();
2413 * Now that we're guaranteed success, proceed to move all tasks to
2414 * the new cgroup. There are no failure cases after here, so this
2415 * is the commit point.
2417 spin_lock_irq(&css_set_lock
);
2418 list_for_each_entry(cset
, &tset
->src_csets
, mg_node
) {
2419 list_for_each_entry_safe(task
, tmp_task
, &cset
->mg_tasks
, cg_list
) {
2420 struct css_set
*from_cset
= task_css_set(task
);
2421 struct css_set
*to_cset
= cset
->mg_dst_cset
;
2423 get_css_set(to_cset
);
2424 to_cset
->nr_tasks
++;
2425 css_set_move_task(task
, from_cset
, to_cset
, true);
2426 from_cset
->nr_tasks
--;
2428 * If the source or destination cgroup is frozen,
2429 * the task might require to change its state.
2431 cgroup_freezer_migrate_task(task
, from_cset
->dfl_cgrp
,
2433 put_css_set_locked(from_cset
);
2437 spin_unlock_irq(&css_set_lock
);
2440 * Migration is committed, all target tasks are now on dst_csets.
2441 * Nothing is sensitive to fork() after this point. Notify
2442 * controllers that migration is complete.
2444 tset
->csets
= &tset
->dst_csets
;
2446 if (tset
->nr_tasks
) {
2447 do_each_subsys_mask(ss
, ssid
, mgctx
->ss_mask
) {
2452 } while_each_subsys_mask();
2456 goto out_release_tset
;
2459 if (tset
->nr_tasks
) {
2460 do_each_subsys_mask(ss
, ssid
, mgctx
->ss_mask
) {
2461 if (ssid
== failed_ssid
)
2463 if (ss
->cancel_attach
) {
2465 ss
->cancel_attach(tset
);
2467 } while_each_subsys_mask();
2470 spin_lock_irq(&css_set_lock
);
2471 list_splice_init(&tset
->dst_csets
, &tset
->src_csets
);
2472 list_for_each_entry_safe(cset
, tmp_cset
, &tset
->src_csets
, mg_node
) {
2473 list_splice_tail_init(&cset
->mg_tasks
, &cset
->tasks
);
2474 list_del_init(&cset
->mg_node
);
2476 spin_unlock_irq(&css_set_lock
);
2479 * Re-initialize the cgroup_taskset structure in case it is reused
2480 * again in another cgroup_migrate_add_task()/cgroup_migrate_execute()
2484 tset
->csets
= &tset
->src_csets
;
2489 * cgroup_migrate_vet_dst - verify whether a cgroup can be migration destination
2490 * @dst_cgrp: destination cgroup to test
2492 * On the default hierarchy, except for the mixable, (possible) thread root
2493 * and threaded cgroups, subtree_control must be zero for migration
2494 * destination cgroups with tasks so that child cgroups don't compete
2497 int cgroup_migrate_vet_dst(struct cgroup
*dst_cgrp
)
2499 /* v1 doesn't have any restriction */
2500 if (!cgroup_on_dfl(dst_cgrp
))
2503 /* verify @dst_cgrp can host resources */
2504 if (!cgroup_is_valid_domain(dst_cgrp
->dom_cgrp
))
2507 /* mixables don't care */
2508 if (cgroup_is_mixable(dst_cgrp
))
2512 * If @dst_cgrp is already or can become a thread root or is
2513 * threaded, it doesn't matter.
2515 if (cgroup_can_be_thread_root(dst_cgrp
) || cgroup_is_threaded(dst_cgrp
))
2518 /* apply no-internal-process constraint */
2519 if (dst_cgrp
->subtree_control
)
2526 * cgroup_migrate_finish - cleanup after attach
2527 * @mgctx: migration context
2529 * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See
2530 * those functions for details.
2532 void cgroup_migrate_finish(struct cgroup_mgctx
*mgctx
)
2534 LIST_HEAD(preloaded
);
2535 struct css_set
*cset
, *tmp_cset
;
2537 lockdep_assert_held(&cgroup_mutex
);
2539 spin_lock_irq(&css_set_lock
);
2541 list_splice_tail_init(&mgctx
->preloaded_src_csets
, &preloaded
);
2542 list_splice_tail_init(&mgctx
->preloaded_dst_csets
, &preloaded
);
2544 list_for_each_entry_safe(cset
, tmp_cset
, &preloaded
, mg_preload_node
) {
2545 cset
->mg_src_cgrp
= NULL
;
2546 cset
->mg_dst_cgrp
= NULL
;
2547 cset
->mg_dst_cset
= NULL
;
2548 list_del_init(&cset
->mg_preload_node
);
2549 put_css_set_locked(cset
);
2552 spin_unlock_irq(&css_set_lock
);
2556 * cgroup_migrate_add_src - add a migration source css_set
2557 * @src_cset: the source css_set to add
2558 * @dst_cgrp: the destination cgroup
2559 * @mgctx: migration context
2561 * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin
2562 * @src_cset and add it to @mgctx->src_csets, which should later be cleaned
2563 * up by cgroup_migrate_finish().
2565 * This function may be called without holding cgroup_threadgroup_rwsem
2566 * even if the target is a process. Threads may be created and destroyed
2567 * but as long as cgroup_mutex is not dropped, no new css_set can be put
2568 * into play and the preloaded css_sets are guaranteed to cover all
2571 void cgroup_migrate_add_src(struct css_set
*src_cset
,
2572 struct cgroup
*dst_cgrp
,
2573 struct cgroup_mgctx
*mgctx
)
2575 struct cgroup
*src_cgrp
;
2577 lockdep_assert_held(&cgroup_mutex
);
2578 lockdep_assert_held(&css_set_lock
);
2581 * If ->dead, @src_set is associated with one or more dead cgroups
2582 * and doesn't contain any migratable tasks. Ignore it early so
2583 * that the rest of migration path doesn't get confused by it.
2588 src_cgrp
= cset_cgroup_from_root(src_cset
, dst_cgrp
->root
);
2590 if (!list_empty(&src_cset
->mg_preload_node
))
2593 WARN_ON(src_cset
->mg_src_cgrp
);
2594 WARN_ON(src_cset
->mg_dst_cgrp
);
2595 WARN_ON(!list_empty(&src_cset
->mg_tasks
));
2596 WARN_ON(!list_empty(&src_cset
->mg_node
));
2598 src_cset
->mg_src_cgrp
= src_cgrp
;
2599 src_cset
->mg_dst_cgrp
= dst_cgrp
;
2600 get_css_set(src_cset
);
2601 list_add_tail(&src_cset
->mg_preload_node
, &mgctx
->preloaded_src_csets
);
2605 * cgroup_migrate_prepare_dst - prepare destination css_sets for migration
2606 * @mgctx: migration context
2608 * Tasks are about to be moved and all the source css_sets have been
2609 * preloaded to @mgctx->preloaded_src_csets. This function looks up and
2610 * pins all destination css_sets, links each to its source, and append them
2611 * to @mgctx->preloaded_dst_csets.
2613 * This function must be called after cgroup_migrate_add_src() has been
2614 * called on each migration source css_set. After migration is performed
2615 * using cgroup_migrate(), cgroup_migrate_finish() must be called on
2618 int cgroup_migrate_prepare_dst(struct cgroup_mgctx
*mgctx
)
2620 struct css_set
*src_cset
, *tmp_cset
;
2622 lockdep_assert_held(&cgroup_mutex
);
2624 /* look up the dst cset for each src cset and link it to src */
2625 list_for_each_entry_safe(src_cset
, tmp_cset
, &mgctx
->preloaded_src_csets
,
2627 struct css_set
*dst_cset
;
2628 struct cgroup_subsys
*ss
;
2631 dst_cset
= find_css_set(src_cset
, src_cset
->mg_dst_cgrp
);
2635 WARN_ON_ONCE(src_cset
->mg_dst_cset
|| dst_cset
->mg_dst_cset
);
2638 * If src cset equals dst, it's noop. Drop the src.
2639 * cgroup_migrate() will skip the cset too. Note that we
2640 * can't handle src == dst as some nodes are used by both.
2642 if (src_cset
== dst_cset
) {
2643 src_cset
->mg_src_cgrp
= NULL
;
2644 src_cset
->mg_dst_cgrp
= NULL
;
2645 list_del_init(&src_cset
->mg_preload_node
);
2646 put_css_set(src_cset
);
2647 put_css_set(dst_cset
);
2651 src_cset
->mg_dst_cset
= dst_cset
;
2653 if (list_empty(&dst_cset
->mg_preload_node
))
2654 list_add_tail(&dst_cset
->mg_preload_node
,
2655 &mgctx
->preloaded_dst_csets
);
2657 put_css_set(dst_cset
);
2659 for_each_subsys(ss
, ssid
)
2660 if (src_cset
->subsys
[ssid
] != dst_cset
->subsys
[ssid
])
2661 mgctx
->ss_mask
|= 1 << ssid
;
2668 * cgroup_migrate - migrate a process or task to a cgroup
2669 * @leader: the leader of the process or the task to migrate
2670 * @threadgroup: whether @leader points to the whole process or a single task
2671 * @mgctx: migration context
2673 * Migrate a process or task denoted by @leader. If migrating a process,
2674 * the caller must be holding cgroup_threadgroup_rwsem. The caller is also
2675 * responsible for invoking cgroup_migrate_add_src() and
2676 * cgroup_migrate_prepare_dst() on the targets before invoking this
2677 * function and following up with cgroup_migrate_finish().
2679 * As long as a controller's ->can_attach() doesn't fail, this function is
2680 * guaranteed to succeed. This means that, excluding ->can_attach()
2681 * failure, when migrating multiple targets, the success or failure can be
2682 * decided for all targets by invoking group_migrate_prepare_dst() before
2683 * actually starting migrating.
2685 int cgroup_migrate(struct task_struct
*leader
, bool threadgroup
,
2686 struct cgroup_mgctx
*mgctx
)
2688 struct task_struct
*task
;
2691 * Prevent freeing of tasks while we take a snapshot. Tasks that are
2692 * already PF_EXITING could be freed from underneath us unless we
2693 * take an rcu_read_lock.
2695 spin_lock_irq(&css_set_lock
);
2699 cgroup_migrate_add_task(task
, mgctx
);
2702 } while_each_thread(leader
, task
);
2704 spin_unlock_irq(&css_set_lock
);
2706 return cgroup_migrate_execute(mgctx
);
2710 * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup
2711 * @dst_cgrp: the cgroup to attach to
2712 * @leader: the task or the leader of the threadgroup to be attached
2713 * @threadgroup: attach the whole threadgroup?
2715 * Call holding cgroup_mutex and cgroup_threadgroup_rwsem.
2717 int cgroup_attach_task(struct cgroup
*dst_cgrp
, struct task_struct
*leader
,
2720 DEFINE_CGROUP_MGCTX(mgctx
);
2721 struct task_struct
*task
;
2724 /* look up all src csets */
2725 spin_lock_irq(&css_set_lock
);
2729 cgroup_migrate_add_src(task_css_set(task
), dst_cgrp
, &mgctx
);
2732 } while_each_thread(leader
, task
);
2734 spin_unlock_irq(&css_set_lock
);
2736 /* prepare dst csets and commit */
2737 ret
= cgroup_migrate_prepare_dst(&mgctx
);
2739 ret
= cgroup_migrate(leader
, threadgroup
, &mgctx
);
2741 cgroup_migrate_finish(&mgctx
);
2744 TRACE_CGROUP_PATH(attach_task
, dst_cgrp
, leader
, threadgroup
);
2749 struct task_struct
*cgroup_procs_write_start(char *buf
, bool threadgroup
,
2751 __acquires(&cgroup_threadgroup_rwsem
)
2753 struct task_struct
*tsk
;
2756 if (kstrtoint(strstrip(buf
), 0, &pid
) || pid
< 0)
2757 return ERR_PTR(-EINVAL
);
2760 * If we migrate a single thread, we don't care about threadgroup
2761 * stability. If the thread is `current`, it won't exit(2) under our
2762 * hands or change PID through exec(2). We exclude
2763 * cgroup_update_dfl_csses and other cgroup_{proc,thread}s_write
2764 * callers by cgroup_mutex.
2765 * Therefore, we can skip the global lock.
2767 lockdep_assert_held(&cgroup_mutex
);
2768 if (pid
|| threadgroup
) {
2769 percpu_down_write(&cgroup_threadgroup_rwsem
);
2777 tsk
= find_task_by_vpid(pid
);
2779 tsk
= ERR_PTR(-ESRCH
);
2780 goto out_unlock_threadgroup
;
2787 tsk
= tsk
->group_leader
;
2790 * kthreads may acquire PF_NO_SETAFFINITY during initialization.
2791 * If userland migrates such a kthread to a non-root cgroup, it can
2792 * become trapped in a cpuset, or RT kthread may be born in a
2793 * cgroup with no rt_runtime allocated. Just say no.
2795 if (tsk
->no_cgroup_migration
|| (tsk
->flags
& PF_NO_SETAFFINITY
)) {
2796 tsk
= ERR_PTR(-EINVAL
);
2797 goto out_unlock_threadgroup
;
2800 get_task_struct(tsk
);
2801 goto out_unlock_rcu
;
2803 out_unlock_threadgroup
:
2805 percpu_up_write(&cgroup_threadgroup_rwsem
);
2813 void cgroup_procs_write_finish(struct task_struct
*task
, bool locked
)
2814 __releases(&cgroup_threadgroup_rwsem
)
2816 struct cgroup_subsys
*ss
;
2819 /* release reference from cgroup_procs_write_start() */
2820 put_task_struct(task
);
2823 percpu_up_write(&cgroup_threadgroup_rwsem
);
2824 for_each_subsys(ss
, ssid
)
2825 if (ss
->post_attach
)
2829 static void cgroup_print_ss_mask(struct seq_file
*seq
, u16 ss_mask
)
2831 struct cgroup_subsys
*ss
;
2832 bool printed
= false;
2835 do_each_subsys_mask(ss
, ssid
, ss_mask
) {
2838 seq_puts(seq
, ss
->name
);
2840 } while_each_subsys_mask();
2842 seq_putc(seq
, '\n');
2845 /* show controllers which are enabled from the parent */
2846 static int cgroup_controllers_show(struct seq_file
*seq
, void *v
)
2848 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
2850 cgroup_print_ss_mask(seq
, cgroup_control(cgrp
));
2854 /* show controllers which are enabled for a given cgroup's children */
2855 static int cgroup_subtree_control_show(struct seq_file
*seq
, void *v
)
2857 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
2859 cgroup_print_ss_mask(seq
, cgrp
->subtree_control
);
2864 * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy
2865 * @cgrp: root of the subtree to update csses for
2867 * @cgrp's control masks have changed and its subtree's css associations
2868 * need to be updated accordingly. This function looks up all css_sets
2869 * which are attached to the subtree, creates the matching updated css_sets
2870 * and migrates the tasks to the new ones.
2872 static int cgroup_update_dfl_csses(struct cgroup
*cgrp
)
2874 DEFINE_CGROUP_MGCTX(mgctx
);
2875 struct cgroup_subsys_state
*d_css
;
2876 struct cgroup
*dsct
;
2877 struct css_set
*src_cset
;
2880 lockdep_assert_held(&cgroup_mutex
);
2882 percpu_down_write(&cgroup_threadgroup_rwsem
);
2884 /* look up all csses currently attached to @cgrp's subtree */
2885 spin_lock_irq(&css_set_lock
);
2886 cgroup_for_each_live_descendant_pre(dsct
, d_css
, cgrp
) {
2887 struct cgrp_cset_link
*link
;
2889 list_for_each_entry(link
, &dsct
->cset_links
, cset_link
)
2890 cgroup_migrate_add_src(link
->cset
, dsct
, &mgctx
);
2892 spin_unlock_irq(&css_set_lock
);
2894 /* NULL dst indicates self on default hierarchy */
2895 ret
= cgroup_migrate_prepare_dst(&mgctx
);
2899 spin_lock_irq(&css_set_lock
);
2900 list_for_each_entry(src_cset
, &mgctx
.preloaded_src_csets
, mg_preload_node
) {
2901 struct task_struct
*task
, *ntask
;
2903 /* all tasks in src_csets need to be migrated */
2904 list_for_each_entry_safe(task
, ntask
, &src_cset
->tasks
, cg_list
)
2905 cgroup_migrate_add_task(task
, &mgctx
);
2907 spin_unlock_irq(&css_set_lock
);
2909 ret
= cgroup_migrate_execute(&mgctx
);
2911 cgroup_migrate_finish(&mgctx
);
2912 percpu_up_write(&cgroup_threadgroup_rwsem
);
2917 * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses
2918 * @cgrp: root of the target subtree
2920 * Because css offlining is asynchronous, userland may try to re-enable a
2921 * controller while the previous css is still around. This function grabs
2922 * cgroup_mutex and drains the previous css instances of @cgrp's subtree.
2924 void cgroup_lock_and_drain_offline(struct cgroup
*cgrp
)
2925 __acquires(&cgroup_mutex
)
2927 struct cgroup
*dsct
;
2928 struct cgroup_subsys_state
*d_css
;
2929 struct cgroup_subsys
*ss
;
2933 mutex_lock(&cgroup_mutex
);
2935 cgroup_for_each_live_descendant_post(dsct
, d_css
, cgrp
) {
2936 for_each_subsys(ss
, ssid
) {
2937 struct cgroup_subsys_state
*css
= cgroup_css(dsct
, ss
);
2940 if (!css
|| !percpu_ref_is_dying(&css
->refcnt
))
2943 cgroup_get_live(dsct
);
2944 prepare_to_wait(&dsct
->offline_waitq
, &wait
,
2945 TASK_UNINTERRUPTIBLE
);
2947 mutex_unlock(&cgroup_mutex
);
2949 finish_wait(&dsct
->offline_waitq
, &wait
);
2958 * cgroup_save_control - save control masks and dom_cgrp of a subtree
2959 * @cgrp: root of the target subtree
2961 * Save ->subtree_control, ->subtree_ss_mask and ->dom_cgrp to the
2962 * respective old_ prefixed fields for @cgrp's subtree including @cgrp
2965 static void cgroup_save_control(struct cgroup
*cgrp
)
2967 struct cgroup
*dsct
;
2968 struct cgroup_subsys_state
*d_css
;
2970 cgroup_for_each_live_descendant_pre(dsct
, d_css
, cgrp
) {
2971 dsct
->old_subtree_control
= dsct
->subtree_control
;
2972 dsct
->old_subtree_ss_mask
= dsct
->subtree_ss_mask
;
2973 dsct
->old_dom_cgrp
= dsct
->dom_cgrp
;
2978 * cgroup_propagate_control - refresh control masks of a subtree
2979 * @cgrp: root of the target subtree
2981 * For @cgrp and its subtree, ensure ->subtree_ss_mask matches
2982 * ->subtree_control and propagate controller availability through the
2983 * subtree so that descendants don't have unavailable controllers enabled.
2985 static void cgroup_propagate_control(struct cgroup
*cgrp
)
2987 struct cgroup
*dsct
;
2988 struct cgroup_subsys_state
*d_css
;
2990 cgroup_for_each_live_descendant_pre(dsct
, d_css
, cgrp
) {
2991 dsct
->subtree_control
&= cgroup_control(dsct
);
2992 dsct
->subtree_ss_mask
=
2993 cgroup_calc_subtree_ss_mask(dsct
->subtree_control
,
2994 cgroup_ss_mask(dsct
));
2999 * cgroup_restore_control - restore control masks and dom_cgrp of a subtree
3000 * @cgrp: root of the target subtree
3002 * Restore ->subtree_control, ->subtree_ss_mask and ->dom_cgrp from the
3003 * respective old_ prefixed fields for @cgrp's subtree including @cgrp
3006 static void cgroup_restore_control(struct cgroup
*cgrp
)
3008 struct cgroup
*dsct
;
3009 struct cgroup_subsys_state
*d_css
;
3011 cgroup_for_each_live_descendant_post(dsct
, d_css
, cgrp
) {
3012 dsct
->subtree_control
= dsct
->old_subtree_control
;
3013 dsct
->subtree_ss_mask
= dsct
->old_subtree_ss_mask
;
3014 dsct
->dom_cgrp
= dsct
->old_dom_cgrp
;
3018 static bool css_visible(struct cgroup_subsys_state
*css
)
3020 struct cgroup_subsys
*ss
= css
->ss
;
3021 struct cgroup
*cgrp
= css
->cgroup
;
3023 if (cgroup_control(cgrp
) & (1 << ss
->id
))
3025 if (!(cgroup_ss_mask(cgrp
) & (1 << ss
->id
)))
3027 return cgroup_on_dfl(cgrp
) && ss
->implicit_on_dfl
;
3031 * cgroup_apply_control_enable - enable or show csses according to control
3032 * @cgrp: root of the target subtree
3034 * Walk @cgrp's subtree and create new csses or make the existing ones
3035 * visible. A css is created invisible if it's being implicitly enabled
3036 * through dependency. An invisible css is made visible when the userland
3037 * explicitly enables it.
3039 * Returns 0 on success, -errno on failure. On failure, csses which have
3040 * been processed already aren't cleaned up. The caller is responsible for
3041 * cleaning up with cgroup_apply_control_disable().
3043 static int cgroup_apply_control_enable(struct cgroup
*cgrp
)
3045 struct cgroup
*dsct
;
3046 struct cgroup_subsys_state
*d_css
;
3047 struct cgroup_subsys
*ss
;
3050 cgroup_for_each_live_descendant_pre(dsct
, d_css
, cgrp
) {
3051 for_each_subsys(ss
, ssid
) {
3052 struct cgroup_subsys_state
*css
= cgroup_css(dsct
, ss
);
3054 if (!(cgroup_ss_mask(dsct
) & (1 << ss
->id
)))
3058 css
= css_create(dsct
, ss
);
3060 return PTR_ERR(css
);
3063 WARN_ON_ONCE(percpu_ref_is_dying(&css
->refcnt
));
3065 if (css_visible(css
)) {
3066 ret
= css_populate_dir(css
);
3077 * cgroup_apply_control_disable - kill or hide csses according to control
3078 * @cgrp: root of the target subtree
3080 * Walk @cgrp's subtree and kill and hide csses so that they match
3081 * cgroup_ss_mask() and cgroup_visible_mask().
3083 * A css is hidden when the userland requests it to be disabled while other
3084 * subsystems are still depending on it. The css must not actively control
3085 * resources and be in the vanilla state if it's made visible again later.
3086 * Controllers which may be depended upon should provide ->css_reset() for
3089 static void cgroup_apply_control_disable(struct cgroup
*cgrp
)
3091 struct cgroup
*dsct
;
3092 struct cgroup_subsys_state
*d_css
;
3093 struct cgroup_subsys
*ss
;
3096 cgroup_for_each_live_descendant_post(dsct
, d_css
, cgrp
) {
3097 for_each_subsys(ss
, ssid
) {
3098 struct cgroup_subsys_state
*css
= cgroup_css(dsct
, ss
);
3103 WARN_ON_ONCE(percpu_ref_is_dying(&css
->refcnt
));
3106 !(cgroup_ss_mask(dsct
) & (1 << ss
->id
))) {
3108 } else if (!css_visible(css
)) {
3118 * cgroup_apply_control - apply control mask updates to the subtree
3119 * @cgrp: root of the target subtree
3121 * subsystems can be enabled and disabled in a subtree using the following
3124 * 1. Call cgroup_save_control() to stash the current state.
3125 * 2. Update ->subtree_control masks in the subtree as desired.
3126 * 3. Call cgroup_apply_control() to apply the changes.
3127 * 4. Optionally perform other related operations.
3128 * 5. Call cgroup_finalize_control() to finish up.
3130 * This function implements step 3 and propagates the mask changes
3131 * throughout @cgrp's subtree, updates csses accordingly and perform
3132 * process migrations.
3134 static int cgroup_apply_control(struct cgroup
*cgrp
)
3138 cgroup_propagate_control(cgrp
);
3140 ret
= cgroup_apply_control_enable(cgrp
);
3145 * At this point, cgroup_e_css_by_mask() results reflect the new csses
3146 * making the following cgroup_update_dfl_csses() properly update
3147 * css associations of all tasks in the subtree.
3149 ret
= cgroup_update_dfl_csses(cgrp
);
3157 * cgroup_finalize_control - finalize control mask update
3158 * @cgrp: root of the target subtree
3159 * @ret: the result of the update
3161 * Finalize control mask update. See cgroup_apply_control() for more info.
3163 static void cgroup_finalize_control(struct cgroup
*cgrp
, int ret
)
3166 cgroup_restore_control(cgrp
);
3167 cgroup_propagate_control(cgrp
);
3170 cgroup_apply_control_disable(cgrp
);
3173 static int cgroup_vet_subtree_control_enable(struct cgroup
*cgrp
, u16 enable
)
3175 u16 domain_enable
= enable
& ~cgrp_dfl_threaded_ss_mask
;
3177 /* if nothing is getting enabled, nothing to worry about */
3181 /* can @cgrp host any resources? */
3182 if (!cgroup_is_valid_domain(cgrp
->dom_cgrp
))
3185 /* mixables don't care */
3186 if (cgroup_is_mixable(cgrp
))
3189 if (domain_enable
) {
3190 /* can't enable domain controllers inside a thread subtree */
3191 if (cgroup_is_thread_root(cgrp
) || cgroup_is_threaded(cgrp
))
3195 * Threaded controllers can handle internal competitions
3196 * and are always allowed inside a (prospective) thread
3199 if (cgroup_can_be_thread_root(cgrp
) || cgroup_is_threaded(cgrp
))
3204 * Controllers can't be enabled for a cgroup with tasks to avoid
3205 * child cgroups competing against tasks.
3207 if (cgroup_has_tasks(cgrp
))
3213 /* change the enabled child controllers for a cgroup in the default hierarchy */
3214 static ssize_t
cgroup_subtree_control_write(struct kernfs_open_file
*of
,
3215 char *buf
, size_t nbytes
,
3218 u16 enable
= 0, disable
= 0;
3219 struct cgroup
*cgrp
, *child
;
3220 struct cgroup_subsys
*ss
;
3225 * Parse input - space separated list of subsystem names prefixed
3226 * with either + or -.
3228 buf
= strstrip(buf
);
3229 while ((tok
= strsep(&buf
, " "))) {
3232 do_each_subsys_mask(ss
, ssid
, ~cgrp_dfl_inhibit_ss_mask
) {
3233 if (!cgroup_ssid_enabled(ssid
) ||
3234 strcmp(tok
+ 1, ss
->name
))
3238 enable
|= 1 << ssid
;
3239 disable
&= ~(1 << ssid
);
3240 } else if (*tok
== '-') {
3241 disable
|= 1 << ssid
;
3242 enable
&= ~(1 << ssid
);
3247 } while_each_subsys_mask();
3248 if (ssid
== CGROUP_SUBSYS_COUNT
)
3252 cgrp
= cgroup_kn_lock_live(of
->kn
, true);
3256 for_each_subsys(ss
, ssid
) {
3257 if (enable
& (1 << ssid
)) {
3258 if (cgrp
->subtree_control
& (1 << ssid
)) {
3259 enable
&= ~(1 << ssid
);
3263 if (!(cgroup_control(cgrp
) & (1 << ssid
))) {
3267 } else if (disable
& (1 << ssid
)) {
3268 if (!(cgrp
->subtree_control
& (1 << ssid
))) {
3269 disable
&= ~(1 << ssid
);
3273 /* a child has it enabled? */
3274 cgroup_for_each_live_child(child
, cgrp
) {
3275 if (child
->subtree_control
& (1 << ssid
)) {
3283 if (!enable
&& !disable
) {
3288 ret
= cgroup_vet_subtree_control_enable(cgrp
, enable
);
3292 /* save and update control masks and prepare csses */
3293 cgroup_save_control(cgrp
);
3295 cgrp
->subtree_control
|= enable
;
3296 cgrp
->subtree_control
&= ~disable
;
3298 ret
= cgroup_apply_control(cgrp
);
3299 cgroup_finalize_control(cgrp
, ret
);
3303 kernfs_activate(cgrp
->kn
);
3305 cgroup_kn_unlock(of
->kn
);
3306 return ret
?: nbytes
;
3310 * cgroup_enable_threaded - make @cgrp threaded
3311 * @cgrp: the target cgroup
3313 * Called when "threaded" is written to the cgroup.type interface file and
3314 * tries to make @cgrp threaded and join the parent's resource domain.
3315 * This function is never called on the root cgroup as cgroup.type doesn't
3318 static int cgroup_enable_threaded(struct cgroup
*cgrp
)
3320 struct cgroup
*parent
= cgroup_parent(cgrp
);
3321 struct cgroup
*dom_cgrp
= parent
->dom_cgrp
;
3322 struct cgroup
*dsct
;
3323 struct cgroup_subsys_state
*d_css
;
3326 lockdep_assert_held(&cgroup_mutex
);
3328 /* noop if already threaded */
3329 if (cgroup_is_threaded(cgrp
))
3333 * If @cgroup is populated or has domain controllers enabled, it
3334 * can't be switched. While the below cgroup_can_be_thread_root()
3335 * test can catch the same conditions, that's only when @parent is
3336 * not mixable, so let's check it explicitly.
3338 if (cgroup_is_populated(cgrp
) ||
3339 cgrp
->subtree_control
& ~cgrp_dfl_threaded_ss_mask
)
3342 /* we're joining the parent's domain, ensure its validity */
3343 if (!cgroup_is_valid_domain(dom_cgrp
) ||
3344 !cgroup_can_be_thread_root(dom_cgrp
))
3348 * The following shouldn't cause actual migrations and should
3351 cgroup_save_control(cgrp
);
3353 cgroup_for_each_live_descendant_pre(dsct
, d_css
, cgrp
)
3354 if (dsct
== cgrp
|| cgroup_is_threaded(dsct
))
3355 dsct
->dom_cgrp
= dom_cgrp
;
3357 ret
= cgroup_apply_control(cgrp
);
3359 parent
->nr_threaded_children
++;
3361 cgroup_finalize_control(cgrp
, ret
);
3365 static int cgroup_type_show(struct seq_file
*seq
, void *v
)
3367 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3369 if (cgroup_is_threaded(cgrp
))
3370 seq_puts(seq
, "threaded\n");
3371 else if (!cgroup_is_valid_domain(cgrp
))
3372 seq_puts(seq
, "domain invalid\n");
3373 else if (cgroup_is_thread_root(cgrp
))
3374 seq_puts(seq
, "domain threaded\n");
3376 seq_puts(seq
, "domain\n");
3381 static ssize_t
cgroup_type_write(struct kernfs_open_file
*of
, char *buf
,
3382 size_t nbytes
, loff_t off
)
3384 struct cgroup
*cgrp
;
3387 /* only switching to threaded mode is supported */
3388 if (strcmp(strstrip(buf
), "threaded"))
3391 /* drain dying csses before we re-apply (threaded) subtree control */
3392 cgrp
= cgroup_kn_lock_live(of
->kn
, true);
3396 /* threaded can only be enabled */
3397 ret
= cgroup_enable_threaded(cgrp
);
3399 cgroup_kn_unlock(of
->kn
);
3400 return ret
?: nbytes
;
3403 static int cgroup_max_descendants_show(struct seq_file
*seq
, void *v
)
3405 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3406 int descendants
= READ_ONCE(cgrp
->max_descendants
);
3408 if (descendants
== INT_MAX
)
3409 seq_puts(seq
, "max\n");
3411 seq_printf(seq
, "%d\n", descendants
);
3416 static ssize_t
cgroup_max_descendants_write(struct kernfs_open_file
*of
,
3417 char *buf
, size_t nbytes
, loff_t off
)
3419 struct cgroup
*cgrp
;
3423 buf
= strstrip(buf
);
3424 if (!strcmp(buf
, "max")) {
3425 descendants
= INT_MAX
;
3427 ret
= kstrtoint(buf
, 0, &descendants
);
3432 if (descendants
< 0)
3435 cgrp
= cgroup_kn_lock_live(of
->kn
, false);
3439 cgrp
->max_descendants
= descendants
;
3441 cgroup_kn_unlock(of
->kn
);
3446 static int cgroup_max_depth_show(struct seq_file
*seq
, void *v
)
3448 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3449 int depth
= READ_ONCE(cgrp
->max_depth
);
3451 if (depth
== INT_MAX
)
3452 seq_puts(seq
, "max\n");
3454 seq_printf(seq
, "%d\n", depth
);
3459 static ssize_t
cgroup_max_depth_write(struct kernfs_open_file
*of
,
3460 char *buf
, size_t nbytes
, loff_t off
)
3462 struct cgroup
*cgrp
;
3466 buf
= strstrip(buf
);
3467 if (!strcmp(buf
, "max")) {
3470 ret
= kstrtoint(buf
, 0, &depth
);
3478 cgrp
= cgroup_kn_lock_live(of
->kn
, false);
3482 cgrp
->max_depth
= depth
;
3484 cgroup_kn_unlock(of
->kn
);
3489 static int cgroup_events_show(struct seq_file
*seq
, void *v
)
3491 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3493 seq_printf(seq
, "populated %d\n", cgroup_is_populated(cgrp
));
3494 seq_printf(seq
, "frozen %d\n", test_bit(CGRP_FROZEN
, &cgrp
->flags
));
3499 static int cgroup_stat_show(struct seq_file
*seq
, void *v
)
3501 struct cgroup
*cgroup
= seq_css(seq
)->cgroup
;
3503 seq_printf(seq
, "nr_descendants %d\n",
3504 cgroup
->nr_descendants
);
3505 seq_printf(seq
, "nr_dying_descendants %d\n",
3506 cgroup
->nr_dying_descendants
);
3511 static int __maybe_unused
cgroup_extra_stat_show(struct seq_file
*seq
,
3512 struct cgroup
*cgrp
, int ssid
)
3514 struct cgroup_subsys
*ss
= cgroup_subsys
[ssid
];
3515 struct cgroup_subsys_state
*css
;
3518 if (!ss
->css_extra_stat_show
)
3521 css
= cgroup_tryget_css(cgrp
, ss
);
3525 ret
= ss
->css_extra_stat_show(seq
, css
);
3530 static int cpu_stat_show(struct seq_file
*seq
, void *v
)
3532 struct cgroup __maybe_unused
*cgrp
= seq_css(seq
)->cgroup
;
3535 cgroup_base_stat_cputime_show(seq
);
3536 #ifdef CONFIG_CGROUP_SCHED
3537 ret
= cgroup_extra_stat_show(seq
, cgrp
, cpu_cgrp_id
);
3543 static int cgroup_io_pressure_show(struct seq_file
*seq
, void *v
)
3545 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3546 struct psi_group
*psi
= cgroup_ino(cgrp
) == 1 ? &psi_system
: &cgrp
->psi
;
3548 return psi_show(seq
, psi
, PSI_IO
);
3550 static int cgroup_memory_pressure_show(struct seq_file
*seq
, void *v
)
3552 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3553 struct psi_group
*psi
= cgroup_ino(cgrp
) == 1 ? &psi_system
: &cgrp
->psi
;
3555 return psi_show(seq
, psi
, PSI_MEM
);
3557 static int cgroup_cpu_pressure_show(struct seq_file
*seq
, void *v
)
3559 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3560 struct psi_group
*psi
= cgroup_ino(cgrp
) == 1 ? &psi_system
: &cgrp
->psi
;
3562 return psi_show(seq
, psi
, PSI_CPU
);
3565 static ssize_t
cgroup_pressure_write(struct kernfs_open_file
*of
, char *buf
,
3566 size_t nbytes
, enum psi_res res
)
3568 struct psi_trigger
*new;
3569 struct cgroup
*cgrp
;
3571 cgrp
= cgroup_kn_lock_live(of
->kn
, false);
3576 cgroup_kn_unlock(of
->kn
);
3578 new = psi_trigger_create(&cgrp
->psi
, buf
, nbytes
, res
);
3581 return PTR_ERR(new);
3584 psi_trigger_replace(&of
->priv
, new);
3591 static ssize_t
cgroup_io_pressure_write(struct kernfs_open_file
*of
,
3592 char *buf
, size_t nbytes
,
3595 return cgroup_pressure_write(of
, buf
, nbytes
, PSI_IO
);
3598 static ssize_t
cgroup_memory_pressure_write(struct kernfs_open_file
*of
,
3599 char *buf
, size_t nbytes
,
3602 return cgroup_pressure_write(of
, buf
, nbytes
, PSI_MEM
);
3605 static ssize_t
cgroup_cpu_pressure_write(struct kernfs_open_file
*of
,
3606 char *buf
, size_t nbytes
,
3609 return cgroup_pressure_write(of
, buf
, nbytes
, PSI_CPU
);
3612 static __poll_t
cgroup_pressure_poll(struct kernfs_open_file
*of
,
3615 return psi_trigger_poll(&of
->priv
, of
->file
, pt
);
3618 static void cgroup_pressure_release(struct kernfs_open_file
*of
)
3620 psi_trigger_replace(&of
->priv
, NULL
);
3622 #endif /* CONFIG_PSI */
3624 static int cgroup_freeze_show(struct seq_file
*seq
, void *v
)
3626 struct cgroup
*cgrp
= seq_css(seq
)->cgroup
;
3628 seq_printf(seq
, "%d\n", cgrp
->freezer
.freeze
);
3633 static ssize_t
cgroup_freeze_write(struct kernfs_open_file
*of
,
3634 char *buf
, size_t nbytes
, loff_t off
)
3636 struct cgroup
*cgrp
;
3640 ret
= kstrtoint(strstrip(buf
), 0, &freeze
);
3644 if (freeze
< 0 || freeze
> 1)
3647 cgrp
= cgroup_kn_lock_live(of
->kn
, false);
3651 cgroup_freeze(cgrp
, freeze
);
3653 cgroup_kn_unlock(of
->kn
);
3658 static int cgroup_file_open(struct kernfs_open_file
*of
)
3660 struct cftype
*cft
= of
->kn
->priv
;
3663 return cft
->open(of
);
3667 static void cgroup_file_release(struct kernfs_open_file
*of
)
3669 struct cftype
*cft
= of
->kn
->priv
;
3675 static ssize_t
cgroup_file_write(struct kernfs_open_file
*of
, char *buf
,
3676 size_t nbytes
, loff_t off
)
3678 struct cgroup_namespace
*ns
= current
->nsproxy
->cgroup_ns
;
3679 struct cgroup
*cgrp
= of
->kn
->parent
->priv
;
3680 struct cftype
*cft
= of
->kn
->priv
;
3681 struct cgroup_subsys_state
*css
;
3688 * If namespaces are delegation boundaries, disallow writes to
3689 * files in an non-init namespace root from inside the namespace
3690 * except for the files explicitly marked delegatable -
3691 * cgroup.procs and cgroup.subtree_control.
3693 if ((cgrp
->root
->flags
& CGRP_ROOT_NS_DELEGATE
) &&
3694 !(cft
->flags
& CFTYPE_NS_DELEGATABLE
) &&
3695 ns
!= &init_cgroup_ns
&& ns
->root_cset
->dfl_cgrp
== cgrp
)
3699 return cft
->write(of
, buf
, nbytes
, off
);
3702 * kernfs guarantees that a file isn't deleted with operations in
3703 * flight, which means that the matching css is and stays alive and
3704 * doesn't need to be pinned. The RCU locking is not necessary
3705 * either. It's just for the convenience of using cgroup_css().
3708 css
= cgroup_css(cgrp
, cft
->ss
);
3711 if (cft
->write_u64
) {
3712 unsigned long long v
;
3713 ret
= kstrtoull(buf
, 0, &v
);
3715 ret
= cft
->write_u64(css
, cft
, v
);
3716 } else if (cft
->write_s64
) {
3718 ret
= kstrtoll(buf
, 0, &v
);
3720 ret
= cft
->write_s64(css
, cft
, v
);
3725 return ret
?: nbytes
;
3728 static __poll_t
cgroup_file_poll(struct kernfs_open_file
*of
, poll_table
*pt
)
3730 struct cftype
*cft
= of
->kn
->priv
;
3733 return cft
->poll(of
, pt
);
3735 return kernfs_generic_poll(of
, pt
);
3738 static void *cgroup_seqfile_start(struct seq_file
*seq
, loff_t
*ppos
)
3740 return seq_cft(seq
)->seq_start(seq
, ppos
);
3743 static void *cgroup_seqfile_next(struct seq_file
*seq
, void *v
, loff_t
*ppos
)
3745 return seq_cft(seq
)->seq_next(seq
, v
, ppos
);
3748 static void cgroup_seqfile_stop(struct seq_file
*seq
, void *v
)
3750 if (seq_cft(seq
)->seq_stop
)
3751 seq_cft(seq
)->seq_stop(seq
, v
);
3754 static int cgroup_seqfile_show(struct seq_file
*m
, void *arg
)
3756 struct cftype
*cft
= seq_cft(m
);
3757 struct cgroup_subsys_state
*css
= seq_css(m
);
3760 return cft
->seq_show(m
, arg
);
3763 seq_printf(m
, "%llu\n", cft
->read_u64(css
, cft
));
3764 else if (cft
->read_s64
)
3765 seq_printf(m
, "%lld\n", cft
->read_s64(css
, cft
));
3771 static struct kernfs_ops cgroup_kf_single_ops
= {
3772 .atomic_write_len
= PAGE_SIZE
,
3773 .open
= cgroup_file_open
,
3774 .release
= cgroup_file_release
,
3775 .write
= cgroup_file_write
,
3776 .poll
= cgroup_file_poll
,
3777 .seq_show
= cgroup_seqfile_show
,
3780 static struct kernfs_ops cgroup_kf_ops
= {
3781 .atomic_write_len
= PAGE_SIZE
,
3782 .open
= cgroup_file_open
,
3783 .release
= cgroup_file_release
,
3784 .write
= cgroup_file_write
,
3785 .poll
= cgroup_file_poll
,
3786 .seq_start
= cgroup_seqfile_start
,
3787 .seq_next
= cgroup_seqfile_next
,
3788 .seq_stop
= cgroup_seqfile_stop
,
3789 .seq_show
= cgroup_seqfile_show
,
3792 /* set uid and gid of cgroup dirs and files to that of the creator */
3793 static int cgroup_kn_set_ugid(struct kernfs_node
*kn
)
3795 struct iattr iattr
= { .ia_valid
= ATTR_UID
| ATTR_GID
,
3796 .ia_uid
= current_fsuid(),
3797 .ia_gid
= current_fsgid(), };
3799 if (uid_eq(iattr
.ia_uid
, GLOBAL_ROOT_UID
) &&
3800 gid_eq(iattr
.ia_gid
, GLOBAL_ROOT_GID
))
3803 return kernfs_setattr(kn
, &iattr
);
3806 static void cgroup_file_notify_timer(struct timer_list
*timer
)
3808 cgroup_file_notify(container_of(timer
, struct cgroup_file
,
3812 static int cgroup_add_file(struct cgroup_subsys_state
*css
, struct cgroup
*cgrp
,
3815 char name
[CGROUP_FILE_NAME_MAX
];
3816 struct kernfs_node
*kn
;
3817 struct lock_class_key
*key
= NULL
;
3820 #ifdef CONFIG_DEBUG_LOCK_ALLOC
3821 key
= &cft
->lockdep_key
;
3823 kn
= __kernfs_create_file(cgrp
->kn
, cgroup_file_name(cgrp
, cft
, name
),
3824 cgroup_file_mode(cft
),
3825 GLOBAL_ROOT_UID
, GLOBAL_ROOT_GID
,
3826 0, cft
->kf_ops
, cft
,
3831 ret
= cgroup_kn_set_ugid(kn
);
3837 if (cft
->file_offset
) {
3838 struct cgroup_file
*cfile
= (void *)css
+ cft
->file_offset
;
3840 timer_setup(&cfile
->notify_timer
, cgroup_file_notify_timer
, 0);
3842 spin_lock_irq(&cgroup_file_kn_lock
);
3844 spin_unlock_irq(&cgroup_file_kn_lock
);
3851 * cgroup_addrm_files - add or remove files to a cgroup directory
3852 * @css: the target css
3853 * @cgrp: the target cgroup (usually css->cgroup)
3854 * @cfts: array of cftypes to be added
3855 * @is_add: whether to add or remove
3857 * Depending on @is_add, add or remove files defined by @cfts on @cgrp.
3858 * For removals, this function never fails.
3860 static int cgroup_addrm_files(struct cgroup_subsys_state
*css
,
3861 struct cgroup
*cgrp
, struct cftype cfts
[],
3864 struct cftype
*cft
, *cft_end
= NULL
;
3867 lockdep_assert_held(&cgroup_mutex
);
3870 for (cft
= cfts
; cft
!= cft_end
&& cft
->name
[0] != '\0'; cft
++) {
3871 /* does cft->flags tell us to skip this file on @cgrp? */
3872 if ((cft
->flags
& __CFTYPE_ONLY_ON_DFL
) && !cgroup_on_dfl(cgrp
))
3874 if ((cft
->flags
& __CFTYPE_NOT_ON_DFL
) && cgroup_on_dfl(cgrp
))
3876 if ((cft
->flags
& CFTYPE_NOT_ON_ROOT
) && !cgroup_parent(cgrp
))
3878 if ((cft
->flags
& CFTYPE_ONLY_ON_ROOT
) && cgroup_parent(cgrp
))
3880 if ((cft
->flags
& CFTYPE_DEBUG
) && !cgroup_debug
)
3883 ret
= cgroup_add_file(css
, cgrp
, cft
);
3885 pr_warn("%s: failed to add %s, err=%d\n",
3886 __func__
, cft
->name
, ret
);
3892 cgroup_rm_file(cgrp
, cft
);
3898 static int cgroup_apply_cftypes(struct cftype
*cfts
, bool is_add
)
3900 struct cgroup_subsys
*ss
= cfts
[0].ss
;
3901 struct cgroup
*root
= &ss
->root
->cgrp
;
3902 struct cgroup_subsys_state
*css
;
3905 lockdep_assert_held(&cgroup_mutex
);
3907 /* add/rm files for all cgroups created before */
3908 css_for_each_descendant_pre(css
, cgroup_css(root
, ss
)) {
3909 struct cgroup
*cgrp
= css
->cgroup
;
3911 if (!(css
->flags
& CSS_VISIBLE
))
3914 ret
= cgroup_addrm_files(css
, cgrp
, cfts
, is_add
);
3920 kernfs_activate(root
->kn
);
3924 static void cgroup_exit_cftypes(struct cftype
*cfts
)
3928 for (cft
= cfts
; cft
->name
[0] != '\0'; cft
++) {
3929 /* free copy for custom atomic_write_len, see init_cftypes() */
3930 if (cft
->max_write_len
&& cft
->max_write_len
!= PAGE_SIZE
)
3935 /* revert flags set by cgroup core while adding @cfts */
3936 cft
->flags
&= ~(__CFTYPE_ONLY_ON_DFL
| __CFTYPE_NOT_ON_DFL
);
3940 static int cgroup_init_cftypes(struct cgroup_subsys
*ss
, struct cftype
*cfts
)
3944 for (cft
= cfts
; cft
->name
[0] != '\0'; cft
++) {
3945 struct kernfs_ops
*kf_ops
;
3947 WARN_ON(cft
->ss
|| cft
->kf_ops
);
3950 kf_ops
= &cgroup_kf_ops
;
3952 kf_ops
= &cgroup_kf_single_ops
;
3955 * Ugh... if @cft wants a custom max_write_len, we need to
3956 * make a copy of kf_ops to set its atomic_write_len.
3958 if (cft
->max_write_len
&& cft
->max_write_len
!= PAGE_SIZE
) {
3959 kf_ops
= kmemdup(kf_ops
, sizeof(*kf_ops
), GFP_KERNEL
);
3961 cgroup_exit_cftypes(cfts
);
3964 kf_ops
->atomic_write_len
= cft
->max_write_len
;
3967 cft
->kf_ops
= kf_ops
;
3974 static int cgroup_rm_cftypes_locked(struct cftype
*cfts
)
3976 lockdep_assert_held(&cgroup_mutex
);
3978 if (!cfts
|| !cfts
[0].ss
)
3981 list_del(&cfts
->node
);
3982 cgroup_apply_cftypes(cfts
, false);
3983 cgroup_exit_cftypes(cfts
);
3988 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
3989 * @cfts: zero-length name terminated array of cftypes
3991 * Unregister @cfts. Files described by @cfts are removed from all
3992 * existing cgroups and all future cgroups won't have them either. This
3993 * function can be called anytime whether @cfts' subsys is attached or not.
3995 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
3998 int cgroup_rm_cftypes(struct cftype
*cfts
)
4002 mutex_lock(&cgroup_mutex
);
4003 ret
= cgroup_rm_cftypes_locked(cfts
);
4004 mutex_unlock(&cgroup_mutex
);
4009 * cgroup_add_cftypes - add an array of cftypes to a subsystem
4010 * @ss: target cgroup subsystem
4011 * @cfts: zero-length name terminated array of cftypes
4013 * Register @cfts to @ss. Files described by @cfts are created for all
4014 * existing cgroups to which @ss is attached and all future cgroups will
4015 * have them too. This function can be called anytime whether @ss is
4018 * Returns 0 on successful registration, -errno on failure. Note that this
4019 * function currently returns 0 as long as @cfts registration is successful
4020 * even if some file creation attempts on existing cgroups fail.
4022 static int cgroup_add_cftypes(struct cgroup_subsys
*ss
, struct cftype
*cfts
)
4026 if (!cgroup_ssid_enabled(ss
->id
))
4029 if (!cfts
|| cfts
[0].name
[0] == '\0')
4032 ret
= cgroup_init_cftypes(ss
, cfts
);
4036 mutex_lock(&cgroup_mutex
);
4038 list_add_tail(&cfts
->node
, &ss
->cfts
);
4039 ret
= cgroup_apply_cftypes(cfts
, true);
4041 cgroup_rm_cftypes_locked(cfts
);
4043 mutex_unlock(&cgroup_mutex
);
4048 * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy
4049 * @ss: target cgroup subsystem
4050 * @cfts: zero-length name terminated array of cftypes
4052 * Similar to cgroup_add_cftypes() but the added files are only used for
4053 * the default hierarchy.
4055 int cgroup_add_dfl_cftypes(struct cgroup_subsys
*ss
, struct cftype
*cfts
)
4059 for (cft
= cfts
; cft
&& cft
->name
[0] != '\0'; cft
++)
4060 cft
->flags
|= __CFTYPE_ONLY_ON_DFL
;
4061 return cgroup_add_cftypes(ss
, cfts
);
4065 * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies
4066 * @ss: target cgroup subsystem
4067 * @cfts: zero-length name terminated array of cftypes
4069 * Similar to cgroup_add_cftypes() but the added files are only used for
4070 * the legacy hierarchies.
4072 int cgroup_add_legacy_cftypes(struct cgroup_subsys
*ss
, struct cftype
*cfts
)
4076 for (cft
= cfts
; cft
&& cft
->name
[0] != '\0'; cft
++)
4077 cft
->flags
|= __CFTYPE_NOT_ON_DFL
;
4078 return cgroup_add_cftypes(ss
, cfts
);
4082 * cgroup_file_notify - generate a file modified event for a cgroup_file
4083 * @cfile: target cgroup_file
4085 * @cfile must have been obtained by setting cftype->file_offset.
4087 void cgroup_file_notify(struct cgroup_file
*cfile
)
4089 unsigned long flags
;
4091 spin_lock_irqsave(&cgroup_file_kn_lock
, flags
);
4093 unsigned long last
= cfile
->notified_at
;
4094 unsigned long next
= last
+ CGROUP_FILE_NOTIFY_MIN_INTV
;
4096 if (time_in_range(jiffies
, last
, next
)) {
4097 timer_reduce(&cfile
->notify_timer
, next
);
4099 kernfs_notify(cfile
->kn
);
4100 cfile
->notified_at
= jiffies
;
4103 spin_unlock_irqrestore(&cgroup_file_kn_lock
, flags
);
4107 * css_next_child - find the next child of a given css
4108 * @pos: the current position (%NULL to initiate traversal)
4109 * @parent: css whose children to walk
4111 * This function returns the next child of @parent and should be called
4112 * under either cgroup_mutex or RCU read lock. The only requirement is
4113 * that @parent and @pos are accessible. The next sibling is guaranteed to
4114 * be returned regardless of their states.
4116 * If a subsystem synchronizes ->css_online() and the start of iteration, a
4117 * css which finished ->css_online() is guaranteed to be visible in the
4118 * future iterations and will stay visible until the last reference is put.
4119 * A css which hasn't finished ->css_online() or already finished
4120 * ->css_offline() may show up during traversal. It's each subsystem's
4121 * responsibility to synchronize against on/offlining.
4123 struct cgroup_subsys_state
*css_next_child(struct cgroup_subsys_state
*pos
,
4124 struct cgroup_subsys_state
*parent
)
4126 struct cgroup_subsys_state
*next
;
4128 cgroup_assert_mutex_or_rcu_locked();
4131 * @pos could already have been unlinked from the sibling list.
4132 * Once a cgroup is removed, its ->sibling.next is no longer
4133 * updated when its next sibling changes. CSS_RELEASED is set when
4134 * @pos is taken off list, at which time its next pointer is valid,
4135 * and, as releases are serialized, the one pointed to by the next
4136 * pointer is guaranteed to not have started release yet. This
4137 * implies that if we observe !CSS_RELEASED on @pos in this RCU
4138 * critical section, the one pointed to by its next pointer is
4139 * guaranteed to not have finished its RCU grace period even if we
4140 * have dropped rcu_read_lock() inbetween iterations.
4142 * If @pos has CSS_RELEASED set, its next pointer can't be
4143 * dereferenced; however, as each css is given a monotonically
4144 * increasing unique serial number and always appended to the
4145 * sibling list, the next one can be found by walking the parent's
4146 * children until the first css with higher serial number than
4147 * @pos's. While this path can be slower, it happens iff iteration
4148 * races against release and the race window is very small.
4151 next
= list_entry_rcu(parent
->children
.next
, struct cgroup_subsys_state
, sibling
);
4152 } else if (likely(!(pos
->flags
& CSS_RELEASED
))) {
4153 next
= list_entry_rcu(pos
->sibling
.next
, struct cgroup_subsys_state
, sibling
);
4155 list_for_each_entry_rcu(next
, &parent
->children
, sibling
,
4156 lockdep_is_held(&cgroup_mutex
))
4157 if (next
->serial_nr
> pos
->serial_nr
)
4162 * @next, if not pointing to the head, can be dereferenced and is
4165 if (&next
->sibling
!= &parent
->children
)
4171 * css_next_descendant_pre - find the next descendant for pre-order walk
4172 * @pos: the current position (%NULL to initiate traversal)
4173 * @root: css whose descendants to walk
4175 * To be used by css_for_each_descendant_pre(). Find the next descendant
4176 * to visit for pre-order traversal of @root's descendants. @root is
4177 * included in the iteration and the first node to be visited.
4179 * While this function requires cgroup_mutex or RCU read locking, it
4180 * doesn't require the whole traversal to be contained in a single critical
4181 * section. This function will return the correct next descendant as long
4182 * as both @pos and @root are accessible and @pos is a descendant of @root.
4184 * If a subsystem synchronizes ->css_online() and the start of iteration, a
4185 * css which finished ->css_online() is guaranteed to be visible in the
4186 * future iterations and will stay visible until the last reference is put.
4187 * A css which hasn't finished ->css_online() or already finished
4188 * ->css_offline() may show up during traversal. It's each subsystem's
4189 * responsibility to synchronize against on/offlining.
4191 struct cgroup_subsys_state
*
4192 css_next_descendant_pre(struct cgroup_subsys_state
*pos
,
4193 struct cgroup_subsys_state
*root
)
4195 struct cgroup_subsys_state
*next
;
4197 cgroup_assert_mutex_or_rcu_locked();
4199 /* if first iteration, visit @root */
4203 /* visit the first child if exists */
4204 next
= css_next_child(NULL
, pos
);
4208 /* no child, visit my or the closest ancestor's next sibling */
4209 while (pos
!= root
) {
4210 next
= css_next_child(pos
, pos
->parent
);
4218 EXPORT_SYMBOL_GPL(css_next_descendant_pre
);
4221 * css_rightmost_descendant - return the rightmost descendant of a css
4222 * @pos: css of interest
4224 * Return the rightmost descendant of @pos. If there's no descendant, @pos
4225 * is returned. This can be used during pre-order traversal to skip
4228 * While this function requires cgroup_mutex or RCU read locking, it
4229 * doesn't require the whole traversal to be contained in a single critical
4230 * section. This function will return the correct rightmost descendant as
4231 * long as @pos is accessible.
4233 struct cgroup_subsys_state
*
4234 css_rightmost_descendant(struct cgroup_subsys_state
*pos
)
4236 struct cgroup_subsys_state
*last
, *tmp
;
4238 cgroup_assert_mutex_or_rcu_locked();
4242 /* ->prev isn't RCU safe, walk ->next till the end */
4244 css_for_each_child(tmp
, last
)
4251 static struct cgroup_subsys_state
*
4252 css_leftmost_descendant(struct cgroup_subsys_state
*pos
)
4254 struct cgroup_subsys_state
*last
;
4258 pos
= css_next_child(NULL
, pos
);
4265 * css_next_descendant_post - find the next descendant for post-order walk
4266 * @pos: the current position (%NULL to initiate traversal)
4267 * @root: css whose descendants to walk
4269 * To be used by css_for_each_descendant_post(). Find the next descendant
4270 * to visit for post-order traversal of @root's descendants. @root is
4271 * included in the iteration and the last node to be visited.
4273 * While this function requires cgroup_mutex or RCU read locking, it
4274 * doesn't require the whole traversal to be contained in a single critical
4275 * section. This function will return the correct next descendant as long
4276 * as both @pos and @cgroup are accessible and @pos is a descendant of
4279 * If a subsystem synchronizes ->css_online() and the start of iteration, a
4280 * css which finished ->css_online() is guaranteed to be visible in the
4281 * future iterations and will stay visible until the last reference is put.
4282 * A css which hasn't finished ->css_online() or already finished
4283 * ->css_offline() may show up during traversal. It's each subsystem's
4284 * responsibility to synchronize against on/offlining.
4286 struct cgroup_subsys_state
*
4287 css_next_descendant_post(struct cgroup_subsys_state
*pos
,
4288 struct cgroup_subsys_state
*root
)
4290 struct cgroup_subsys_state
*next
;
4292 cgroup_assert_mutex_or_rcu_locked();
4294 /* if first iteration, visit leftmost descendant which may be @root */
4296 return css_leftmost_descendant(root
);
4298 /* if we visited @root, we're done */
4302 /* if there's an unvisited sibling, visit its leftmost descendant */
4303 next
= css_next_child(pos
, pos
->parent
);
4305 return css_leftmost_descendant(next
);
4307 /* no sibling left, visit parent */
4312 * css_has_online_children - does a css have online children
4313 * @css: the target css
4315 * Returns %true if @css has any online children; otherwise, %false. This
4316 * function can be called from any context but the caller is responsible
4317 * for synchronizing against on/offlining as necessary.
4319 bool css_has_online_children(struct cgroup_subsys_state
*css
)
4321 struct cgroup_subsys_state
*child
;
4325 css_for_each_child(child
, css
) {
4326 if (child
->flags
& CSS_ONLINE
) {
4335 static struct css_set
*css_task_iter_next_css_set(struct css_task_iter
*it
)
4337 struct list_head
*l
;
4338 struct cgrp_cset_link
*link
;
4339 struct css_set
*cset
;
4341 lockdep_assert_held(&css_set_lock
);
4343 /* find the next threaded cset */
4344 if (it
->tcset_pos
) {
4345 l
= it
->tcset_pos
->next
;
4347 if (l
!= it
->tcset_head
) {
4349 return container_of(l
, struct css_set
,
4350 threaded_csets_node
);
4353 it
->tcset_pos
= NULL
;
4356 /* find the next cset */
4359 if (l
== it
->cset_head
) {
4360 it
->cset_pos
= NULL
;
4365 cset
= container_of(l
, struct css_set
, e_cset_node
[it
->ss
->id
]);
4367 link
= list_entry(l
, struct cgrp_cset_link
, cset_link
);
4373 /* initialize threaded css_set walking */
4374 if (it
->flags
& CSS_TASK_ITER_THREADED
) {
4376 put_css_set_locked(it
->cur_dcset
);
4377 it
->cur_dcset
= cset
;
4380 it
->tcset_head
= &cset
->threaded_csets
;
4381 it
->tcset_pos
= &cset
->threaded_csets
;
4388 * css_task_iter_advance_css_set - advance a task itererator to the next css_set
4389 * @it: the iterator to advance
4391 * Advance @it to the next css_set to walk.
4393 static void css_task_iter_advance_css_set(struct css_task_iter
*it
)
4395 struct css_set
*cset
;
4397 lockdep_assert_held(&css_set_lock
);
4399 /* Advance to the next non-empty css_set and find first non-empty tasks list*/
4400 while ((cset
= css_task_iter_next_css_set(it
))) {
4401 if (!list_empty(&cset
->tasks
)) {
4402 it
->cur_tasks_head
= &cset
->tasks
;
4404 } else if (!list_empty(&cset
->mg_tasks
)) {
4405 it
->cur_tasks_head
= &cset
->mg_tasks
;
4407 } else if (!list_empty(&cset
->dying_tasks
)) {
4408 it
->cur_tasks_head
= &cset
->dying_tasks
;
4413 it
->task_pos
= NULL
;
4416 it
->task_pos
= it
->cur_tasks_head
->next
;
4419 * We don't keep css_sets locked across iteration steps and thus
4420 * need to take steps to ensure that iteration can be resumed after
4421 * the lock is re-acquired. Iteration is performed at two levels -
4422 * css_sets and tasks in them.
4424 * Once created, a css_set never leaves its cgroup lists, so a
4425 * pinned css_set is guaranteed to stay put and we can resume
4426 * iteration afterwards.
4428 * Tasks may leave @cset across iteration steps. This is resolved
4429 * by registering each iterator with the css_set currently being
4430 * walked and making css_set_move_task() advance iterators whose
4431 * next task is leaving.
4434 list_del(&it
->iters_node
);
4435 put_css_set_locked(it
->cur_cset
);
4438 it
->cur_cset
= cset
;
4439 list_add(&it
->iters_node
, &cset
->task_iters
);
4442 static void css_task_iter_skip(struct css_task_iter
*it
,
4443 struct task_struct
*task
)
4445 lockdep_assert_held(&css_set_lock
);
4447 if (it
->task_pos
== &task
->cg_list
) {
4448 it
->task_pos
= it
->task_pos
->next
;
4449 it
->flags
|= CSS_TASK_ITER_SKIPPED
;
4453 static void css_task_iter_advance(struct css_task_iter
*it
)
4455 struct task_struct
*task
;
4457 lockdep_assert_held(&css_set_lock
);
4461 * Advance iterator to find next entry. We go through cset
4462 * tasks, mg_tasks and dying_tasks, when consumed we move onto
4465 if (it
->flags
& CSS_TASK_ITER_SKIPPED
)
4466 it
->flags
&= ~CSS_TASK_ITER_SKIPPED
;
4468 it
->task_pos
= it
->task_pos
->next
;
4470 if (it
->task_pos
== &it
->cur_cset
->tasks
) {
4471 it
->cur_tasks_head
= &it
->cur_cset
->mg_tasks
;
4472 it
->task_pos
= it
->cur_tasks_head
->next
;
4474 if (it
->task_pos
== &it
->cur_cset
->mg_tasks
) {
4475 it
->cur_tasks_head
= &it
->cur_cset
->dying_tasks
;
4476 it
->task_pos
= it
->cur_tasks_head
->next
;
4478 if (it
->task_pos
== &it
->cur_cset
->dying_tasks
)
4479 css_task_iter_advance_css_set(it
);
4481 /* called from start, proceed to the first cset */
4482 css_task_iter_advance_css_set(it
);
4488 task
= list_entry(it
->task_pos
, struct task_struct
, cg_list
);
4490 if (it
->flags
& CSS_TASK_ITER_PROCS
) {
4491 /* if PROCS, skip over tasks which aren't group leaders */
4492 if (!thread_group_leader(task
))
4495 /* and dying leaders w/o live member threads */
4496 if (it
->cur_tasks_head
== &it
->cur_cset
->dying_tasks
&&
4497 !atomic_read(&task
->signal
->live
))
4500 /* skip all dying ones */
4501 if (it
->cur_tasks_head
== &it
->cur_cset
->dying_tasks
)
4507 * css_task_iter_start - initiate task iteration
4508 * @css: the css to walk tasks of
4509 * @flags: CSS_TASK_ITER_* flags
4510 * @it: the task iterator to use
4512 * Initiate iteration through the tasks of @css. The caller can call
4513 * css_task_iter_next() to walk through the tasks until the function
4514 * returns NULL. On completion of iteration, css_task_iter_end() must be
4517 void css_task_iter_start(struct cgroup_subsys_state
*css
, unsigned int flags
,
4518 struct css_task_iter
*it
)
4520 memset(it
, 0, sizeof(*it
));
4522 spin_lock_irq(&css_set_lock
);
4528 it
->cset_pos
= &css
->cgroup
->e_csets
[css
->ss
->id
];
4530 it
->cset_pos
= &css
->cgroup
->cset_links
;
4532 it
->cset_head
= it
->cset_pos
;
4534 css_task_iter_advance(it
);
4536 spin_unlock_irq(&css_set_lock
);
4540 * css_task_iter_next - return the next task for the iterator
4541 * @it: the task iterator being iterated
4543 * The "next" function for task iteration. @it should have been
4544 * initialized via css_task_iter_start(). Returns NULL when the iteration
4547 struct task_struct
*css_task_iter_next(struct css_task_iter
*it
)
4550 put_task_struct(it
->cur_task
);
4551 it
->cur_task
= NULL
;
4554 spin_lock_irq(&css_set_lock
);
4556 /* @it may be half-advanced by skips, finish advancing */
4557 if (it
->flags
& CSS_TASK_ITER_SKIPPED
)
4558 css_task_iter_advance(it
);
4561 it
->cur_task
= list_entry(it
->task_pos
, struct task_struct
,
4563 get_task_struct(it
->cur_task
);
4564 css_task_iter_advance(it
);
4567 spin_unlock_irq(&css_set_lock
);
4569 return it
->cur_task
;
4573 * css_task_iter_end - finish task iteration
4574 * @it: the task iterator to finish
4576 * Finish task iteration started by css_task_iter_start().
4578 void css_task_iter_end(struct css_task_iter
*it
)
4581 spin_lock_irq(&css_set_lock
);
4582 list_del(&it
->iters_node
);
4583 put_css_set_locked(it
->cur_cset
);
4584 spin_unlock_irq(&css_set_lock
);
4588 put_css_set(it
->cur_dcset
);
4591 put_task_struct(it
->cur_task
);
4594 static void cgroup_procs_release(struct kernfs_open_file
*of
)
4597 css_task_iter_end(of
->priv
);
4602 static void *cgroup_procs_next(struct seq_file
*s
, void *v
, loff_t
*pos
)
4604 struct kernfs_open_file
*of
= s
->private;
4605 struct css_task_iter
*it
= of
->priv
;
4610 return css_task_iter_next(it
);
4613 static void *__cgroup_procs_start(struct seq_file
*s
, loff_t
*pos
,
4614 unsigned int iter_flags
)
4616 struct kernfs_open_file
*of
= s
->private;
4617 struct cgroup
*cgrp
= seq_css(s
)->cgroup
;
4618 struct css_task_iter
*it
= of
->priv
;
4621 * When a seq_file is seeked, it's always traversed sequentially
4622 * from position 0, so we can simply keep iterating on !0 *pos.
4625 if (WARN_ON_ONCE((*pos
)))
4626 return ERR_PTR(-EINVAL
);
4628 it
= kzalloc(sizeof(*it
), GFP_KERNEL
);
4630 return ERR_PTR(-ENOMEM
);
4632 css_task_iter_start(&cgrp
->self
, iter_flags
, it
);
4633 } else if (!(*pos
)) {
4634 css_task_iter_end(it
);
4635 css_task_iter_start(&cgrp
->self
, iter_flags
, it
);
4637 return it
->cur_task
;
4639 return cgroup_procs_next(s
, NULL
, NULL
);
4642 static void *cgroup_procs_start(struct seq_file
*s
, loff_t
*pos
)
4644 struct cgroup
*cgrp
= seq_css(s
)->cgroup
;
4647 * All processes of a threaded subtree belong to the domain cgroup
4648 * of the subtree. Only threads can be distributed across the
4649 * subtree. Reject reads on cgroup.procs in the subtree proper.
4650 * They're always empty anyway.
4652 if (cgroup_is_threaded(cgrp
))
4653 return ERR_PTR(-EOPNOTSUPP
);
4655 return __cgroup_procs_start(s
, pos
, CSS_TASK_ITER_PROCS
|
4656 CSS_TASK_ITER_THREADED
);
4659 static int cgroup_procs_show(struct seq_file
*s
, void *v
)
4661 seq_printf(s
, "%d\n", task_pid_vnr(v
));
4665 static int cgroup_may_write(const struct cgroup
*cgrp
, struct super_block
*sb
)
4668 struct inode
*inode
;
4670 lockdep_assert_held(&cgroup_mutex
);
4672 inode
= kernfs_get_inode(sb
, cgrp
->procs_file
.kn
);
4676 ret
= inode_permission(inode
, MAY_WRITE
);
4681 static int cgroup_procs_write_permission(struct cgroup
*src_cgrp
,
4682 struct cgroup
*dst_cgrp
,
4683 struct super_block
*sb
)
4685 struct cgroup_namespace
*ns
= current
->nsproxy
->cgroup_ns
;
4686 struct cgroup
*com_cgrp
= src_cgrp
;
4689 lockdep_assert_held(&cgroup_mutex
);
4691 /* find the common ancestor */
4692 while (!cgroup_is_descendant(dst_cgrp
, com_cgrp
))
4693 com_cgrp
= cgroup_parent(com_cgrp
);
4695 /* %current should be authorized to migrate to the common ancestor */
4696 ret
= cgroup_may_write(com_cgrp
, sb
);
4701 * If namespaces are delegation boundaries, %current must be able
4702 * to see both source and destination cgroups from its namespace.
4704 if ((cgrp_dfl_root
.flags
& CGRP_ROOT_NS_DELEGATE
) &&
4705 (!cgroup_is_descendant(src_cgrp
, ns
->root_cset
->dfl_cgrp
) ||
4706 !cgroup_is_descendant(dst_cgrp
, ns
->root_cset
->dfl_cgrp
)))
4712 static int cgroup_attach_permissions(struct cgroup
*src_cgrp
,
4713 struct cgroup
*dst_cgrp
,
4714 struct super_block
*sb
, bool threadgroup
)
4718 ret
= cgroup_procs_write_permission(src_cgrp
, dst_cgrp
, sb
);
4722 ret
= cgroup_migrate_vet_dst(dst_cgrp
);
4726 if (!threadgroup
&& (src_cgrp
->dom_cgrp
!= dst_cgrp
->dom_cgrp
))
4732 static ssize_t
cgroup_procs_write(struct kernfs_open_file
*of
,
4733 char *buf
, size_t nbytes
, loff_t off
)
4735 struct cgroup
*src_cgrp
, *dst_cgrp
;
4736 struct task_struct
*task
;
4740 dst_cgrp
= cgroup_kn_lock_live(of
->kn
, false);
4744 task
= cgroup_procs_write_start(buf
, true, &locked
);
4745 ret
= PTR_ERR_OR_ZERO(task
);
4749 /* find the source cgroup */
4750 spin_lock_irq(&css_set_lock
);
4751 src_cgrp
= task_cgroup_from_root(task
, &cgrp_dfl_root
);
4752 spin_unlock_irq(&css_set_lock
);
4754 ret
= cgroup_attach_permissions(src_cgrp
, dst_cgrp
,
4755 of
->file
->f_path
.dentry
->d_sb
, true);
4759 ret
= cgroup_attach_task(dst_cgrp
, task
, true);
4762 cgroup_procs_write_finish(task
, locked
);
4764 cgroup_kn_unlock(of
->kn
);
4766 return ret
?: nbytes
;
4769 static void *cgroup_threads_start(struct seq_file
*s
, loff_t
*pos
)
4771 return __cgroup_procs_start(s
, pos
, 0);
4774 static ssize_t
cgroup_threads_write(struct kernfs_open_file
*of
,
4775 char *buf
, size_t nbytes
, loff_t off
)
4777 struct cgroup
*src_cgrp
, *dst_cgrp
;
4778 struct task_struct
*task
;
4782 buf
= strstrip(buf
);
4784 dst_cgrp
= cgroup_kn_lock_live(of
->kn
, false);
4788 task
= cgroup_procs_write_start(buf
, false, &locked
);
4789 ret
= PTR_ERR_OR_ZERO(task
);
4793 /* find the source cgroup */
4794 spin_lock_irq(&css_set_lock
);
4795 src_cgrp
= task_cgroup_from_root(task
, &cgrp_dfl_root
);
4796 spin_unlock_irq(&css_set_lock
);
4798 /* thread migrations follow the cgroup.procs delegation rule */
4799 ret
= cgroup_attach_permissions(src_cgrp
, dst_cgrp
,
4800 of
->file
->f_path
.dentry
->d_sb
, false);
4804 ret
= cgroup_attach_task(dst_cgrp
, task
, false);
4807 cgroup_procs_write_finish(task
, locked
);
4809 cgroup_kn_unlock(of
->kn
);
4811 return ret
?: nbytes
;
4814 /* cgroup core interface files for the default hierarchy */
4815 static struct cftype cgroup_base_files
[] = {
4817 .name
= "cgroup.type",
4818 .flags
= CFTYPE_NOT_ON_ROOT
,
4819 .seq_show
= cgroup_type_show
,
4820 .write
= cgroup_type_write
,
4823 .name
= "cgroup.procs",
4824 .flags
= CFTYPE_NS_DELEGATABLE
,
4825 .file_offset
= offsetof(struct cgroup
, procs_file
),
4826 .release
= cgroup_procs_release
,
4827 .seq_start
= cgroup_procs_start
,
4828 .seq_next
= cgroup_procs_next
,
4829 .seq_show
= cgroup_procs_show
,
4830 .write
= cgroup_procs_write
,
4833 .name
= "cgroup.threads",
4834 .flags
= CFTYPE_NS_DELEGATABLE
,
4835 .release
= cgroup_procs_release
,
4836 .seq_start
= cgroup_threads_start
,
4837 .seq_next
= cgroup_procs_next
,
4838 .seq_show
= cgroup_procs_show
,
4839 .write
= cgroup_threads_write
,
4842 .name
= "cgroup.controllers",
4843 .seq_show
= cgroup_controllers_show
,
4846 .name
= "cgroup.subtree_control",
4847 .flags
= CFTYPE_NS_DELEGATABLE
,
4848 .seq_show
= cgroup_subtree_control_show
,
4849 .write
= cgroup_subtree_control_write
,
4852 .name
= "cgroup.events",
4853 .flags
= CFTYPE_NOT_ON_ROOT
,
4854 .file_offset
= offsetof(struct cgroup
, events_file
),
4855 .seq_show
= cgroup_events_show
,
4858 .name
= "cgroup.max.descendants",
4859 .seq_show
= cgroup_max_descendants_show
,
4860 .write
= cgroup_max_descendants_write
,
4863 .name
= "cgroup.max.depth",
4864 .seq_show
= cgroup_max_depth_show
,
4865 .write
= cgroup_max_depth_write
,
4868 .name
= "cgroup.stat",
4869 .seq_show
= cgroup_stat_show
,
4872 .name
= "cgroup.freeze",
4873 .flags
= CFTYPE_NOT_ON_ROOT
,
4874 .seq_show
= cgroup_freeze_show
,
4875 .write
= cgroup_freeze_write
,
4879 .seq_show
= cpu_stat_show
,
4883 .name
= "io.pressure",
4884 .seq_show
= cgroup_io_pressure_show
,
4885 .write
= cgroup_io_pressure_write
,
4886 .poll
= cgroup_pressure_poll
,
4887 .release
= cgroup_pressure_release
,
4890 .name
= "memory.pressure",
4891 .seq_show
= cgroup_memory_pressure_show
,
4892 .write
= cgroup_memory_pressure_write
,
4893 .poll
= cgroup_pressure_poll
,
4894 .release
= cgroup_pressure_release
,
4897 .name
= "cpu.pressure",
4898 .seq_show
= cgroup_cpu_pressure_show
,
4899 .write
= cgroup_cpu_pressure_write
,
4900 .poll
= cgroup_pressure_poll
,
4901 .release
= cgroup_pressure_release
,
4903 #endif /* CONFIG_PSI */
4908 * css destruction is four-stage process.
4910 * 1. Destruction starts. Killing of the percpu_ref is initiated.
4911 * Implemented in kill_css().
4913 * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs
4914 * and thus css_tryget_online() is guaranteed to fail, the css can be
4915 * offlined by invoking offline_css(). After offlining, the base ref is
4916 * put. Implemented in css_killed_work_fn().
4918 * 3. When the percpu_ref reaches zero, the only possible remaining
4919 * accessors are inside RCU read sections. css_release() schedules the
4922 * 4. After the grace period, the css can be freed. Implemented in
4923 * css_free_work_fn().
4925 * It is actually hairier because both step 2 and 4 require process context
4926 * and thus involve punting to css->destroy_work adding two additional
4927 * steps to the already complex sequence.
4929 static void css_free_rwork_fn(struct work_struct
*work
)
4931 struct cgroup_subsys_state
*css
= container_of(to_rcu_work(work
),
4932 struct cgroup_subsys_state
, destroy_rwork
);
4933 struct cgroup_subsys
*ss
= css
->ss
;
4934 struct cgroup
*cgrp
= css
->cgroup
;
4936 percpu_ref_exit(&css
->refcnt
);
4940 struct cgroup_subsys_state
*parent
= css
->parent
;
4944 cgroup_idr_remove(&ss
->css_idr
, id
);
4950 /* cgroup free path */
4951 atomic_dec(&cgrp
->root
->nr_cgrps
);
4952 cgroup1_pidlist_destroy_all(cgrp
);
4953 cancel_work_sync(&cgrp
->release_agent_work
);
4955 if (cgroup_parent(cgrp
)) {
4957 * We get a ref to the parent, and put the ref when
4958 * this cgroup is being freed, so it's guaranteed
4959 * that the parent won't be destroyed before its
4962 cgroup_put(cgroup_parent(cgrp
));
4963 kernfs_put(cgrp
->kn
);
4964 psi_cgroup_free(cgrp
);
4965 if (cgroup_on_dfl(cgrp
))
4966 cgroup_rstat_exit(cgrp
);
4970 * This is root cgroup's refcnt reaching zero,
4971 * which indicates that the root should be
4974 cgroup_destroy_root(cgrp
->root
);
4979 static void css_release_work_fn(struct work_struct
*work
)
4981 struct cgroup_subsys_state
*css
=
4982 container_of(work
, struct cgroup_subsys_state
, destroy_work
);
4983 struct cgroup_subsys
*ss
= css
->ss
;
4984 struct cgroup
*cgrp
= css
->cgroup
;
4986 mutex_lock(&cgroup_mutex
);
4988 css
->flags
|= CSS_RELEASED
;
4989 list_del_rcu(&css
->sibling
);
4992 /* css release path */
4993 if (!list_empty(&css
->rstat_css_node
)) {
4994 cgroup_rstat_flush(cgrp
);
4995 list_del_rcu(&css
->rstat_css_node
);
4998 cgroup_idr_replace(&ss
->css_idr
, NULL
, css
->id
);
4999 if (ss
->css_released
)
5000 ss
->css_released(css
);
5002 struct cgroup
*tcgrp
;
5004 /* cgroup release path */
5005 TRACE_CGROUP_PATH(release
, cgrp
);
5007 if (cgroup_on_dfl(cgrp
))
5008 cgroup_rstat_flush(cgrp
);
5010 spin_lock_irq(&css_set_lock
);
5011 for (tcgrp
= cgroup_parent(cgrp
); tcgrp
;
5012 tcgrp
= cgroup_parent(tcgrp
))
5013 tcgrp
->nr_dying_descendants
--;
5014 spin_unlock_irq(&css_set_lock
);
5017 * There are two control paths which try to determine
5018 * cgroup from dentry without going through kernfs -
5019 * cgroupstats_build() and css_tryget_online_from_dir().
5020 * Those are supported by RCU protecting clearing of
5021 * cgrp->kn->priv backpointer.
5024 RCU_INIT_POINTER(*(void __rcu __force
**)&cgrp
->kn
->priv
,
5028 mutex_unlock(&cgroup_mutex
);
5030 INIT_RCU_WORK(&css
->destroy_rwork
, css_free_rwork_fn
);
5031 queue_rcu_work(cgroup_destroy_wq
, &css
->destroy_rwork
);
5034 static void css_release(struct percpu_ref
*ref
)
5036 struct cgroup_subsys_state
*css
=
5037 container_of(ref
, struct cgroup_subsys_state
, refcnt
);
5039 INIT_WORK(&css
->destroy_work
, css_release_work_fn
);
5040 queue_work(cgroup_destroy_wq
, &css
->destroy_work
);
5043 static void init_and_link_css(struct cgroup_subsys_state
*css
,
5044 struct cgroup_subsys
*ss
, struct cgroup
*cgrp
)
5046 lockdep_assert_held(&cgroup_mutex
);
5048 cgroup_get_live(cgrp
);
5050 memset(css
, 0, sizeof(*css
));
5054 INIT_LIST_HEAD(&css
->sibling
);
5055 INIT_LIST_HEAD(&css
->children
);
5056 INIT_LIST_HEAD(&css
->rstat_css_node
);
5057 css
->serial_nr
= css_serial_nr_next
++;
5058 atomic_set(&css
->online_cnt
, 0);
5060 if (cgroup_parent(cgrp
)) {
5061 css
->parent
= cgroup_css(cgroup_parent(cgrp
), ss
);
5062 css_get(css
->parent
);
5065 if (cgroup_on_dfl(cgrp
) && ss
->css_rstat_flush
)
5066 list_add_rcu(&css
->rstat_css_node
, &cgrp
->rstat_css_list
);
5068 BUG_ON(cgroup_css(cgrp
, ss
));
5071 /* invoke ->css_online() on a new CSS and mark it online if successful */
5072 static int online_css(struct cgroup_subsys_state
*css
)
5074 struct cgroup_subsys
*ss
= css
->ss
;
5077 lockdep_assert_held(&cgroup_mutex
);
5080 ret
= ss
->css_online(css
);
5082 css
->flags
|= CSS_ONLINE
;
5083 rcu_assign_pointer(css
->cgroup
->subsys
[ss
->id
], css
);
5085 atomic_inc(&css
->online_cnt
);
5087 atomic_inc(&css
->parent
->online_cnt
);
5092 /* if the CSS is online, invoke ->css_offline() on it and mark it offline */
5093 static void offline_css(struct cgroup_subsys_state
*css
)
5095 struct cgroup_subsys
*ss
= css
->ss
;
5097 lockdep_assert_held(&cgroup_mutex
);
5099 if (!(css
->flags
& CSS_ONLINE
))
5102 if (ss
->css_offline
)
5103 ss
->css_offline(css
);
5105 css
->flags
&= ~CSS_ONLINE
;
5106 RCU_INIT_POINTER(css
->cgroup
->subsys
[ss
->id
], NULL
);
5108 wake_up_all(&css
->cgroup
->offline_waitq
);
5112 * css_create - create a cgroup_subsys_state
5113 * @cgrp: the cgroup new css will be associated with
5114 * @ss: the subsys of new css
5116 * Create a new css associated with @cgrp - @ss pair. On success, the new
5117 * css is online and installed in @cgrp. This function doesn't create the
5118 * interface files. Returns 0 on success, -errno on failure.
5120 static struct cgroup_subsys_state
*css_create(struct cgroup
*cgrp
,
5121 struct cgroup_subsys
*ss
)
5123 struct cgroup
*parent
= cgroup_parent(cgrp
);
5124 struct cgroup_subsys_state
*parent_css
= cgroup_css(parent
, ss
);
5125 struct cgroup_subsys_state
*css
;
5128 lockdep_assert_held(&cgroup_mutex
);
5130 css
= ss
->css_alloc(parent_css
);
5132 css
= ERR_PTR(-ENOMEM
);
5136 init_and_link_css(css
, ss
, cgrp
);
5138 err
= percpu_ref_init(&css
->refcnt
, css_release
, 0, GFP_KERNEL
);
5142 err
= cgroup_idr_alloc(&ss
->css_idr
, NULL
, 2, 0, GFP_KERNEL
);
5147 /* @css is ready to be brought online now, make it visible */
5148 list_add_tail_rcu(&css
->sibling
, &parent_css
->children
);
5149 cgroup_idr_replace(&ss
->css_idr
, css
, css
->id
);
5151 err
= online_css(css
);
5155 if (ss
->broken_hierarchy
&& !ss
->warned_broken_hierarchy
&&
5156 cgroup_parent(parent
)) {
5157 pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n",
5158 current
->comm
, current
->pid
, ss
->name
);
5159 if (!strcmp(ss
->name
, "memory"))
5160 pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n");
5161 ss
->warned_broken_hierarchy
= true;
5167 list_del_rcu(&css
->sibling
);
5169 list_del_rcu(&css
->rstat_css_node
);
5170 INIT_RCU_WORK(&css
->destroy_rwork
, css_free_rwork_fn
);
5171 queue_rcu_work(cgroup_destroy_wq
, &css
->destroy_rwork
);
5172 return ERR_PTR(err
);
5176 * The returned cgroup is fully initialized including its control mask, but
5177 * it isn't associated with its kernfs_node and doesn't have the control
5180 static struct cgroup
*cgroup_create(struct cgroup
*parent
, const char *name
,
5183 struct cgroup_root
*root
= parent
->root
;
5184 struct cgroup
*cgrp
, *tcgrp
;
5185 struct kernfs_node
*kn
;
5186 int level
= parent
->level
+ 1;
5189 /* allocate the cgroup and its ID, 0 is reserved for the root */
5190 cgrp
= kzalloc(struct_size(cgrp
, ancestor_ids
, (level
+ 1)),
5193 return ERR_PTR(-ENOMEM
);
5195 ret
= percpu_ref_init(&cgrp
->self
.refcnt
, css_release
, 0, GFP_KERNEL
);
5199 if (cgroup_on_dfl(parent
)) {
5200 ret
= cgroup_rstat_init(cgrp
);
5202 goto out_cancel_ref
;
5205 /* create the directory */
5206 kn
= kernfs_create_dir(parent
->kn
, name
, mode
, cgrp
);
5213 init_cgroup_housekeeping(cgrp
);
5215 cgrp
->self
.parent
= &parent
->self
;
5217 cgrp
->level
= level
;
5219 ret
= psi_cgroup_alloc(cgrp
);
5221 goto out_kernfs_remove
;
5223 ret
= cgroup_bpf_inherit(cgrp
);
5228 * New cgroup inherits effective freeze counter, and
5229 * if the parent has to be frozen, the child has too.
5231 cgrp
->freezer
.e_freeze
= parent
->freezer
.e_freeze
;
5232 if (cgrp
->freezer
.e_freeze
) {
5234 * Set the CGRP_FREEZE flag, so when a process will be
5235 * attached to the child cgroup, it will become frozen.
5236 * At this point the new cgroup is unpopulated, so we can
5237 * consider it frozen immediately.
5239 set_bit(CGRP_FREEZE
, &cgrp
->flags
);
5240 set_bit(CGRP_FROZEN
, &cgrp
->flags
);
5243 spin_lock_irq(&css_set_lock
);
5244 for (tcgrp
= cgrp
; tcgrp
; tcgrp
= cgroup_parent(tcgrp
)) {
5245 cgrp
->ancestor_ids
[tcgrp
->level
] = cgroup_id(tcgrp
);
5247 if (tcgrp
!= cgrp
) {
5248 tcgrp
->nr_descendants
++;
5251 * If the new cgroup is frozen, all ancestor cgroups
5252 * get a new frozen descendant, but their state can't
5253 * change because of this.
5255 if (cgrp
->freezer
.e_freeze
)
5256 tcgrp
->freezer
.nr_frozen_descendants
++;
5259 spin_unlock_irq(&css_set_lock
);
5261 if (notify_on_release(parent
))
5262 set_bit(CGRP_NOTIFY_ON_RELEASE
, &cgrp
->flags
);
5264 if (test_bit(CGRP_CPUSET_CLONE_CHILDREN
, &parent
->flags
))
5265 set_bit(CGRP_CPUSET_CLONE_CHILDREN
, &cgrp
->flags
);
5267 cgrp
->self
.serial_nr
= css_serial_nr_next
++;
5269 /* allocation complete, commit to creation */
5270 list_add_tail_rcu(&cgrp
->self
.sibling
, &cgroup_parent(cgrp
)->self
.children
);
5271 atomic_inc(&root
->nr_cgrps
);
5272 cgroup_get_live(parent
);
5275 * On the default hierarchy, a child doesn't automatically inherit
5276 * subtree_control from the parent. Each is configured manually.
5278 if (!cgroup_on_dfl(cgrp
))
5279 cgrp
->subtree_control
= cgroup_control(cgrp
);
5281 cgroup_propagate_control(cgrp
);
5286 psi_cgroup_free(cgrp
);
5288 kernfs_remove(cgrp
->kn
);
5290 if (cgroup_on_dfl(parent
))
5291 cgroup_rstat_exit(cgrp
);
5293 percpu_ref_exit(&cgrp
->self
.refcnt
);
5296 return ERR_PTR(ret
);
5299 static bool cgroup_check_hierarchy_limits(struct cgroup
*parent
)
5301 struct cgroup
*cgroup
;
5305 lockdep_assert_held(&cgroup_mutex
);
5307 for (cgroup
= parent
; cgroup
; cgroup
= cgroup_parent(cgroup
)) {
5308 if (cgroup
->nr_descendants
>= cgroup
->max_descendants
)
5311 if (level
> cgroup
->max_depth
)
5322 int cgroup_mkdir(struct kernfs_node
*parent_kn
, const char *name
, umode_t mode
)
5324 struct cgroup
*parent
, *cgrp
;
5327 /* do not accept '\n' to prevent making /proc/<pid>/cgroup unparsable */
5328 if (strchr(name
, '\n'))
5331 parent
= cgroup_kn_lock_live(parent_kn
, false);
5335 if (!cgroup_check_hierarchy_limits(parent
)) {
5340 cgrp
= cgroup_create(parent
, name
, mode
);
5342 ret
= PTR_ERR(cgrp
);
5347 * This extra ref will be put in cgroup_free_fn() and guarantees
5348 * that @cgrp->kn is always accessible.
5350 kernfs_get(cgrp
->kn
);
5352 ret
= cgroup_kn_set_ugid(cgrp
->kn
);
5356 ret
= css_populate_dir(&cgrp
->self
);
5360 ret
= cgroup_apply_control_enable(cgrp
);
5364 TRACE_CGROUP_PATH(mkdir
, cgrp
);
5366 /* let's create and online css's */
5367 kernfs_activate(cgrp
->kn
);
5373 cgroup_destroy_locked(cgrp
);
5375 cgroup_kn_unlock(parent_kn
);
5380 * This is called when the refcnt of a css is confirmed to be killed.
5381 * css_tryget_online() is now guaranteed to fail. Tell the subsystem to
5382 * initate destruction and put the css ref from kill_css().
5384 static void css_killed_work_fn(struct work_struct
*work
)
5386 struct cgroup_subsys_state
*css
=
5387 container_of(work
, struct cgroup_subsys_state
, destroy_work
);
5389 mutex_lock(&cgroup_mutex
);
5394 /* @css can't go away while we're holding cgroup_mutex */
5396 } while (css
&& atomic_dec_and_test(&css
->online_cnt
));
5398 mutex_unlock(&cgroup_mutex
);
5401 /* css kill confirmation processing requires process context, bounce */
5402 static void css_killed_ref_fn(struct percpu_ref
*ref
)
5404 struct cgroup_subsys_state
*css
=
5405 container_of(ref
, struct cgroup_subsys_state
, refcnt
);
5407 if (atomic_dec_and_test(&css
->online_cnt
)) {
5408 INIT_WORK(&css
->destroy_work
, css_killed_work_fn
);
5409 queue_work(cgroup_destroy_wq
, &css
->destroy_work
);
5414 * kill_css - destroy a css
5415 * @css: css to destroy
5417 * This function initiates destruction of @css by removing cgroup interface
5418 * files and putting its base reference. ->css_offline() will be invoked
5419 * asynchronously once css_tryget_online() is guaranteed to fail and when
5420 * the reference count reaches zero, @css will be released.
5422 static void kill_css(struct cgroup_subsys_state
*css
)
5424 lockdep_assert_held(&cgroup_mutex
);
5426 if (css
->flags
& CSS_DYING
)
5429 css
->flags
|= CSS_DYING
;
5432 * This must happen before css is disassociated with its cgroup.
5433 * See seq_css() for details.
5438 * Killing would put the base ref, but we need to keep it alive
5439 * until after ->css_offline().
5444 * cgroup core guarantees that, by the time ->css_offline() is
5445 * invoked, no new css reference will be given out via
5446 * css_tryget_online(). We can't simply call percpu_ref_kill() and
5447 * proceed to offlining css's because percpu_ref_kill() doesn't
5448 * guarantee that the ref is seen as killed on all CPUs on return.
5450 * Use percpu_ref_kill_and_confirm() to get notifications as each
5451 * css is confirmed to be seen as killed on all CPUs.
5453 percpu_ref_kill_and_confirm(&css
->refcnt
, css_killed_ref_fn
);
5457 * cgroup_destroy_locked - the first stage of cgroup destruction
5458 * @cgrp: cgroup to be destroyed
5460 * css's make use of percpu refcnts whose killing latency shouldn't be
5461 * exposed to userland and are RCU protected. Also, cgroup core needs to
5462 * guarantee that css_tryget_online() won't succeed by the time
5463 * ->css_offline() is invoked. To satisfy all the requirements,
5464 * destruction is implemented in the following two steps.
5466 * s1. Verify @cgrp can be destroyed and mark it dying. Remove all
5467 * userland visible parts and start killing the percpu refcnts of
5468 * css's. Set up so that the next stage will be kicked off once all
5469 * the percpu refcnts are confirmed to be killed.
5471 * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the
5472 * rest of destruction. Once all cgroup references are gone, the
5473 * cgroup is RCU-freed.
5475 * This function implements s1. After this step, @cgrp is gone as far as
5476 * the userland is concerned and a new cgroup with the same name may be
5477 * created. As cgroup doesn't care about the names internally, this
5478 * doesn't cause any problem.
5480 static int cgroup_destroy_locked(struct cgroup
*cgrp
)
5481 __releases(&cgroup_mutex
) __acquires(&cgroup_mutex
)
5483 struct cgroup
*tcgrp
, *parent
= cgroup_parent(cgrp
);
5484 struct cgroup_subsys_state
*css
;
5485 struct cgrp_cset_link
*link
;
5488 lockdep_assert_held(&cgroup_mutex
);
5491 * Only migration can raise populated from zero and we're already
5492 * holding cgroup_mutex.
5494 if (cgroup_is_populated(cgrp
))
5498 * Make sure there's no live children. We can't test emptiness of
5499 * ->self.children as dead children linger on it while being
5500 * drained; otherwise, "rmdir parent/child parent" may fail.
5502 if (css_has_online_children(&cgrp
->self
))
5506 * Mark @cgrp and the associated csets dead. The former prevents
5507 * further task migration and child creation by disabling
5508 * cgroup_lock_live_group(). The latter makes the csets ignored by
5509 * the migration path.
5511 cgrp
->self
.flags
&= ~CSS_ONLINE
;
5513 spin_lock_irq(&css_set_lock
);
5514 list_for_each_entry(link
, &cgrp
->cset_links
, cset_link
)
5515 link
->cset
->dead
= true;
5516 spin_unlock_irq(&css_set_lock
);
5518 /* initiate massacre of all css's */
5519 for_each_css(css
, ssid
, cgrp
)
5522 /* clear and remove @cgrp dir, @cgrp has an extra ref on its kn */
5523 css_clear_dir(&cgrp
->self
);
5524 kernfs_remove(cgrp
->kn
);
5526 if (parent
&& cgroup_is_threaded(cgrp
))
5527 parent
->nr_threaded_children
--;
5529 spin_lock_irq(&css_set_lock
);
5530 for (tcgrp
= cgroup_parent(cgrp
); tcgrp
; tcgrp
= cgroup_parent(tcgrp
)) {
5531 tcgrp
->nr_descendants
--;
5532 tcgrp
->nr_dying_descendants
++;
5534 * If the dying cgroup is frozen, decrease frozen descendants
5535 * counters of ancestor cgroups.
5537 if (test_bit(CGRP_FROZEN
, &cgrp
->flags
))
5538 tcgrp
->freezer
.nr_frozen_descendants
--;
5540 spin_unlock_irq(&css_set_lock
);
5542 cgroup1_check_for_release(parent
);
5544 cgroup_bpf_offline(cgrp
);
5546 /* put the base reference */
5547 percpu_ref_kill(&cgrp
->self
.refcnt
);
5552 int cgroup_rmdir(struct kernfs_node
*kn
)
5554 struct cgroup
*cgrp
;
5557 cgrp
= cgroup_kn_lock_live(kn
, false);
5561 ret
= cgroup_destroy_locked(cgrp
);
5563 TRACE_CGROUP_PATH(rmdir
, cgrp
);
5565 cgroup_kn_unlock(kn
);
5569 static struct kernfs_syscall_ops cgroup_kf_syscall_ops
= {
5570 .show_options
= cgroup_show_options
,
5571 .mkdir
= cgroup_mkdir
,
5572 .rmdir
= cgroup_rmdir
,
5573 .show_path
= cgroup_show_path
,
5576 static void __init
cgroup_init_subsys(struct cgroup_subsys
*ss
, bool early
)
5578 struct cgroup_subsys_state
*css
;
5580 pr_debug("Initializing cgroup subsys %s\n", ss
->name
);
5582 mutex_lock(&cgroup_mutex
);
5584 idr_init(&ss
->css_idr
);
5585 INIT_LIST_HEAD(&ss
->cfts
);
5587 /* Create the root cgroup state for this subsystem */
5588 ss
->root
= &cgrp_dfl_root
;
5589 css
= ss
->css_alloc(cgroup_css(&cgrp_dfl_root
.cgrp
, ss
));
5590 /* We don't handle early failures gracefully */
5591 BUG_ON(IS_ERR(css
));
5592 init_and_link_css(css
, ss
, &cgrp_dfl_root
.cgrp
);
5595 * Root csses are never destroyed and we can't initialize
5596 * percpu_ref during early init. Disable refcnting.
5598 css
->flags
|= CSS_NO_REF
;
5601 /* allocation can't be done safely during early init */
5604 css
->id
= cgroup_idr_alloc(&ss
->css_idr
, css
, 1, 2, GFP_KERNEL
);
5605 BUG_ON(css
->id
< 0);
5608 /* Update the init_css_set to contain a subsys
5609 * pointer to this state - since the subsystem is
5610 * newly registered, all tasks and hence the
5611 * init_css_set is in the subsystem's root cgroup. */
5612 init_css_set
.subsys
[ss
->id
] = css
;
5614 have_fork_callback
|= (bool)ss
->fork
<< ss
->id
;
5615 have_exit_callback
|= (bool)ss
->exit
<< ss
->id
;
5616 have_release_callback
|= (bool)ss
->release
<< ss
->id
;
5617 have_canfork_callback
|= (bool)ss
->can_fork
<< ss
->id
;
5619 /* At system boot, before all subsystems have been
5620 * registered, no tasks have been forked, so we don't
5621 * need to invoke fork callbacks here. */
5622 BUG_ON(!list_empty(&init_task
.tasks
));
5624 BUG_ON(online_css(css
));
5626 mutex_unlock(&cgroup_mutex
);
5630 * cgroup_init_early - cgroup initialization at system boot
5632 * Initialize cgroups at system boot, and initialize any
5633 * subsystems that request early init.
5635 int __init
cgroup_init_early(void)
5637 static struct cgroup_fs_context __initdata ctx
;
5638 struct cgroup_subsys
*ss
;
5641 ctx
.root
= &cgrp_dfl_root
;
5642 init_cgroup_root(&ctx
);
5643 cgrp_dfl_root
.cgrp
.self
.flags
|= CSS_NO_REF
;
5645 RCU_INIT_POINTER(init_task
.cgroups
, &init_css_set
);
5647 for_each_subsys(ss
, i
) {
5648 WARN(!ss
->css_alloc
|| !ss
->css_free
|| ss
->name
|| ss
->id
,
5649 "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n",
5650 i
, cgroup_subsys_name
[i
], ss
->css_alloc
, ss
->css_free
,
5652 WARN(strlen(cgroup_subsys_name
[i
]) > MAX_CGROUP_TYPE_NAMELEN
,
5653 "cgroup_subsys_name %s too long\n", cgroup_subsys_name
[i
]);
5656 ss
->name
= cgroup_subsys_name
[i
];
5657 if (!ss
->legacy_name
)
5658 ss
->legacy_name
= cgroup_subsys_name
[i
];
5661 cgroup_init_subsys(ss
, true);
5666 static u16 cgroup_disable_mask __initdata
;
5669 * cgroup_init - cgroup initialization
5671 * Register cgroup filesystem and /proc file, and initialize
5672 * any subsystems that didn't request early init.
5674 int __init
cgroup_init(void)
5676 struct cgroup_subsys
*ss
;
5679 BUILD_BUG_ON(CGROUP_SUBSYS_COUNT
> 16);
5680 BUG_ON(cgroup_init_cftypes(NULL
, cgroup_base_files
));
5681 BUG_ON(cgroup_init_cftypes(NULL
, cgroup1_base_files
));
5683 cgroup_rstat_boot();
5686 * The latency of the synchronize_rcu() is too high for cgroups,
5687 * avoid it at the cost of forcing all readers into the slow path.
5689 rcu_sync_enter_start(&cgroup_threadgroup_rwsem
.rss
);
5691 get_user_ns(init_cgroup_ns
.user_ns
);
5693 mutex_lock(&cgroup_mutex
);
5696 * Add init_css_set to the hash table so that dfl_root can link to
5699 hash_add(css_set_table
, &init_css_set
.hlist
,
5700 css_set_hash(init_css_set
.subsys
));
5702 BUG_ON(cgroup_setup_root(&cgrp_dfl_root
, 0));
5704 mutex_unlock(&cgroup_mutex
);
5706 for_each_subsys(ss
, ssid
) {
5707 if (ss
->early_init
) {
5708 struct cgroup_subsys_state
*css
=
5709 init_css_set
.subsys
[ss
->id
];
5711 css
->id
= cgroup_idr_alloc(&ss
->css_idr
, css
, 1, 2,
5713 BUG_ON(css
->id
< 0);
5715 cgroup_init_subsys(ss
, false);
5718 list_add_tail(&init_css_set
.e_cset_node
[ssid
],
5719 &cgrp_dfl_root
.cgrp
.e_csets
[ssid
]);
5722 * Setting dfl_root subsys_mask needs to consider the
5723 * disabled flag and cftype registration needs kmalloc,
5724 * both of which aren't available during early_init.
5726 if (cgroup_disable_mask
& (1 << ssid
)) {
5727 static_branch_disable(cgroup_subsys_enabled_key
[ssid
]);
5728 printk(KERN_INFO
"Disabling %s control group subsystem\n",
5733 if (cgroup1_ssid_disabled(ssid
))
5734 printk(KERN_INFO
"Disabling %s control group subsystem in v1 mounts\n",
5737 cgrp_dfl_root
.subsys_mask
|= 1 << ss
->id
;
5739 /* implicit controllers must be threaded too */
5740 WARN_ON(ss
->implicit_on_dfl
&& !ss
->threaded
);
5742 if (ss
->implicit_on_dfl
)
5743 cgrp_dfl_implicit_ss_mask
|= 1 << ss
->id
;
5744 else if (!ss
->dfl_cftypes
)
5745 cgrp_dfl_inhibit_ss_mask
|= 1 << ss
->id
;
5748 cgrp_dfl_threaded_ss_mask
|= 1 << ss
->id
;
5750 if (ss
->dfl_cftypes
== ss
->legacy_cftypes
) {
5751 WARN_ON(cgroup_add_cftypes(ss
, ss
->dfl_cftypes
));
5753 WARN_ON(cgroup_add_dfl_cftypes(ss
, ss
->dfl_cftypes
));
5754 WARN_ON(cgroup_add_legacy_cftypes(ss
, ss
->legacy_cftypes
));
5758 ss
->bind(init_css_set
.subsys
[ssid
]);
5760 mutex_lock(&cgroup_mutex
);
5761 css_populate_dir(init_css_set
.subsys
[ssid
]);
5762 mutex_unlock(&cgroup_mutex
);
5765 /* init_css_set.subsys[] has been updated, re-hash */
5766 hash_del(&init_css_set
.hlist
);
5767 hash_add(css_set_table
, &init_css_set
.hlist
,
5768 css_set_hash(init_css_set
.subsys
));
5770 WARN_ON(sysfs_create_mount_point(fs_kobj
, "cgroup"));
5771 WARN_ON(register_filesystem(&cgroup_fs_type
));
5772 WARN_ON(register_filesystem(&cgroup2_fs_type
));
5773 WARN_ON(!proc_create_single("cgroups", 0, NULL
, proc_cgroupstats_show
));
5774 #ifdef CONFIG_CPUSETS
5775 WARN_ON(register_filesystem(&cpuset_fs_type
));
5781 static int __init
cgroup_wq_init(void)
5784 * There isn't much point in executing destruction path in
5785 * parallel. Good chunk is serialized with cgroup_mutex anyway.
5786 * Use 1 for @max_active.
5788 * We would prefer to do this in cgroup_init() above, but that
5789 * is called before init_workqueues(): so leave this until after.
5791 cgroup_destroy_wq
= alloc_workqueue("cgroup_destroy", 0, 1);
5792 BUG_ON(!cgroup_destroy_wq
);
5795 core_initcall(cgroup_wq_init
);
5797 void cgroup_path_from_kernfs_id(u64 id
, char *buf
, size_t buflen
)
5799 struct kernfs_node
*kn
;
5801 kn
= kernfs_find_and_get_node_by_id(cgrp_dfl_root
.kf_root
, id
);
5804 kernfs_path(kn
, buf
, buflen
);
5809 * proc_cgroup_show()
5810 * - Print task's cgroup paths into seq_file, one line for each hierarchy
5811 * - Used for /proc/<pid>/cgroup.
5813 int proc_cgroup_show(struct seq_file
*m
, struct pid_namespace
*ns
,
5814 struct pid
*pid
, struct task_struct
*tsk
)
5818 struct cgroup_root
*root
;
5821 buf
= kmalloc(PATH_MAX
, GFP_KERNEL
);
5825 mutex_lock(&cgroup_mutex
);
5826 spin_lock_irq(&css_set_lock
);
5828 for_each_root(root
) {
5829 struct cgroup_subsys
*ss
;
5830 struct cgroup
*cgrp
;
5831 int ssid
, count
= 0;
5833 if (root
== &cgrp_dfl_root
&& !cgrp_dfl_visible
)
5836 seq_printf(m
, "%d:", root
->hierarchy_id
);
5837 if (root
!= &cgrp_dfl_root
)
5838 for_each_subsys(ss
, ssid
)
5839 if (root
->subsys_mask
& (1 << ssid
))
5840 seq_printf(m
, "%s%s", count
++ ? "," : "",
5842 if (strlen(root
->name
))
5843 seq_printf(m
, "%sname=%s", count
? "," : "",
5847 cgrp
= task_cgroup_from_root(tsk
, root
);
5850 * On traditional hierarchies, all zombie tasks show up as
5851 * belonging to the root cgroup. On the default hierarchy,
5852 * while a zombie doesn't show up in "cgroup.procs" and
5853 * thus can't be migrated, its /proc/PID/cgroup keeps
5854 * reporting the cgroup it belonged to before exiting. If
5855 * the cgroup is removed before the zombie is reaped,
5856 * " (deleted)" is appended to the cgroup path.
5858 if (cgroup_on_dfl(cgrp
) || !(tsk
->flags
& PF_EXITING
)) {
5859 retval
= cgroup_path_ns_locked(cgrp
, buf
, PATH_MAX
,
5860 current
->nsproxy
->cgroup_ns
);
5861 if (retval
>= PATH_MAX
)
5862 retval
= -ENAMETOOLONG
;
5871 if (cgroup_on_dfl(cgrp
) && cgroup_is_dead(cgrp
))
5872 seq_puts(m
, " (deleted)\n");
5879 spin_unlock_irq(&css_set_lock
);
5880 mutex_unlock(&cgroup_mutex
);
5887 * cgroup_fork - initialize cgroup related fields during copy_process()
5888 * @child: pointer to task_struct of forking parent process.
5890 * A task is associated with the init_css_set until cgroup_post_fork()
5891 * attaches it to the target css_set.
5893 void cgroup_fork(struct task_struct
*child
)
5895 RCU_INIT_POINTER(child
->cgroups
, &init_css_set
);
5896 INIT_LIST_HEAD(&child
->cg_list
);
5899 static struct cgroup
*cgroup_get_from_file(struct file
*f
)
5901 struct cgroup_subsys_state
*css
;
5902 struct cgroup
*cgrp
;
5904 css
= css_tryget_online_from_dir(f
->f_path
.dentry
, NULL
);
5906 return ERR_CAST(css
);
5909 if (!cgroup_on_dfl(cgrp
)) {
5911 return ERR_PTR(-EBADF
);
5918 * cgroup_css_set_fork - find or create a css_set for a child process
5919 * @kargs: the arguments passed to create the child process
5921 * This functions finds or creates a new css_set which the child
5922 * process will be attached to in cgroup_post_fork(). By default,
5923 * the child process will be given the same css_set as its parent.
5925 * If CLONE_INTO_CGROUP is specified this function will try to find an
5926 * existing css_set which includes the requested cgroup and if not create
5927 * a new css_set that the child will be attached to later. If this function
5928 * succeeds it will hold cgroup_threadgroup_rwsem on return. If
5929 * CLONE_INTO_CGROUP is requested this function will grab cgroup mutex
5930 * before grabbing cgroup_threadgroup_rwsem and will hold a reference
5931 * to the target cgroup.
5933 static int cgroup_css_set_fork(struct kernel_clone_args
*kargs
)
5934 __acquires(&cgroup_mutex
) __acquires(&cgroup_threadgroup_rwsem
)
5937 struct cgroup
*dst_cgrp
= NULL
;
5938 struct css_set
*cset
;
5939 struct super_block
*sb
;
5942 if (kargs
->flags
& CLONE_INTO_CGROUP
)
5943 mutex_lock(&cgroup_mutex
);
5945 cgroup_threadgroup_change_begin(current
);
5947 spin_lock_irq(&css_set_lock
);
5948 cset
= task_css_set(current
);
5950 spin_unlock_irq(&css_set_lock
);
5952 if (!(kargs
->flags
& CLONE_INTO_CGROUP
)) {
5957 f
= fget_raw(kargs
->cgroup
);
5962 sb
= f
->f_path
.dentry
->d_sb
;
5964 dst_cgrp
= cgroup_get_from_file(f
);
5965 if (IS_ERR(dst_cgrp
)) {
5966 ret
= PTR_ERR(dst_cgrp
);
5971 if (cgroup_is_dead(dst_cgrp
)) {
5977 * Verify that we the target cgroup is writable for us. This is
5978 * usually done by the vfs layer but since we're not going through
5979 * the vfs layer here we need to do it "manually".
5981 ret
= cgroup_may_write(dst_cgrp
, sb
);
5985 ret
= cgroup_attach_permissions(cset
->dfl_cgrp
, dst_cgrp
, sb
,
5986 !(kargs
->flags
& CLONE_THREAD
));
5990 kargs
->cset
= find_css_set(cset
, dst_cgrp
);
5998 kargs
->cgrp
= dst_cgrp
;
6002 cgroup_threadgroup_change_end(current
);
6003 mutex_unlock(&cgroup_mutex
);
6007 cgroup_put(dst_cgrp
);
6010 put_css_set(kargs
->cset
);
6015 * cgroup_css_set_put_fork - drop references we took during fork
6016 * @kargs: the arguments passed to create the child process
6018 * Drop references to the prepared css_set and target cgroup if
6019 * CLONE_INTO_CGROUP was requested.
6021 static void cgroup_css_set_put_fork(struct kernel_clone_args
*kargs
)
6022 __releases(&cgroup_threadgroup_rwsem
) __releases(&cgroup_mutex
)
6024 cgroup_threadgroup_change_end(current
);
6026 if (kargs
->flags
& CLONE_INTO_CGROUP
) {
6027 struct cgroup
*cgrp
= kargs
->cgrp
;
6028 struct css_set
*cset
= kargs
->cset
;
6030 mutex_unlock(&cgroup_mutex
);
6045 * cgroup_can_fork - called on a new task before the process is exposed
6046 * @child: the child process
6048 * This prepares a new css_set for the child process which the child will
6049 * be attached to in cgroup_post_fork().
6050 * This calls the subsystem can_fork() callbacks. If the cgroup_can_fork()
6051 * callback returns an error, the fork aborts with that error code. This
6052 * allows for a cgroup subsystem to conditionally allow or deny new forks.
6054 int cgroup_can_fork(struct task_struct
*child
, struct kernel_clone_args
*kargs
)
6056 struct cgroup_subsys
*ss
;
6059 ret
= cgroup_css_set_fork(kargs
);
6063 do_each_subsys_mask(ss
, i
, have_canfork_callback
) {
6064 ret
= ss
->can_fork(child
, kargs
->cset
);
6067 } while_each_subsys_mask();
6072 for_each_subsys(ss
, j
) {
6075 if (ss
->cancel_fork
)
6076 ss
->cancel_fork(child
, kargs
->cset
);
6079 cgroup_css_set_put_fork(kargs
);
6085 * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork()
6086 * @child: the child process
6087 * @kargs: the arguments passed to create the child process
6089 * This calls the cancel_fork() callbacks if a fork failed *after*
6090 * cgroup_can_fork() succeded and cleans up references we took to
6091 * prepare a new css_set for the child process in cgroup_can_fork().
6093 void cgroup_cancel_fork(struct task_struct
*child
,
6094 struct kernel_clone_args
*kargs
)
6096 struct cgroup_subsys
*ss
;
6099 for_each_subsys(ss
, i
)
6100 if (ss
->cancel_fork
)
6101 ss
->cancel_fork(child
, kargs
->cset
);
6103 cgroup_css_set_put_fork(kargs
);
6107 * cgroup_post_fork - finalize cgroup setup for the child process
6108 * @child: the child process
6110 * Attach the child process to its css_set calling the subsystem fork()
6113 void cgroup_post_fork(struct task_struct
*child
,
6114 struct kernel_clone_args
*kargs
)
6115 __releases(&cgroup_threadgroup_rwsem
) __releases(&cgroup_mutex
)
6117 struct cgroup_subsys
*ss
;
6118 struct css_set
*cset
;
6124 spin_lock_irq(&css_set_lock
);
6126 /* init tasks are special, only link regular threads */
6127 if (likely(child
->pid
)) {
6128 WARN_ON_ONCE(!list_empty(&child
->cg_list
));
6130 css_set_move_task(child
, NULL
, cset
, false);
6137 * If the cgroup has to be frozen, the new task has too. Let's set
6138 * the JOBCTL_TRAP_FREEZE jobctl bit to get the task into the
6141 if (unlikely(cgroup_task_freeze(child
))) {
6142 spin_lock(&child
->sighand
->siglock
);
6143 WARN_ON_ONCE(child
->frozen
);
6144 child
->jobctl
|= JOBCTL_TRAP_FREEZE
;
6145 spin_unlock(&child
->sighand
->siglock
);
6148 * Calling cgroup_update_frozen() isn't required here,
6149 * because it will be called anyway a bit later from
6150 * do_freezer_trap(). So we avoid cgroup's transient switch
6151 * from the frozen state and back.
6155 spin_unlock_irq(&css_set_lock
);
6158 * Call ss->fork(). This must happen after @child is linked on
6159 * css_set; otherwise, @child might change state between ->fork()
6160 * and addition to css_set.
6162 do_each_subsys_mask(ss
, i
, have_fork_callback
) {
6164 } while_each_subsys_mask();
6166 /* Make the new cset the root_cset of the new cgroup namespace. */
6167 if (kargs
->flags
& CLONE_NEWCGROUP
) {
6168 struct css_set
*rcset
= child
->nsproxy
->cgroup_ns
->root_cset
;
6171 child
->nsproxy
->cgroup_ns
->root_cset
= cset
;
6175 cgroup_css_set_put_fork(kargs
);
6179 * cgroup_exit - detach cgroup from exiting task
6180 * @tsk: pointer to task_struct of exiting process
6182 * Description: Detach cgroup from @tsk.
6185 void cgroup_exit(struct task_struct
*tsk
)
6187 struct cgroup_subsys
*ss
;
6188 struct css_set
*cset
;
6191 spin_lock_irq(&css_set_lock
);
6193 WARN_ON_ONCE(list_empty(&tsk
->cg_list
));
6194 cset
= task_css_set(tsk
);
6195 css_set_move_task(tsk
, cset
, NULL
, false);
6196 list_add_tail(&tsk
->cg_list
, &cset
->dying_tasks
);
6199 WARN_ON_ONCE(cgroup_task_frozen(tsk
));
6200 if (unlikely(cgroup_task_freeze(tsk
)))
6201 cgroup_update_frozen(task_dfl_cgroup(tsk
));
6203 spin_unlock_irq(&css_set_lock
);
6205 /* see cgroup_post_fork() for details */
6206 do_each_subsys_mask(ss
, i
, have_exit_callback
) {
6208 } while_each_subsys_mask();
6211 void cgroup_release(struct task_struct
*task
)
6213 struct cgroup_subsys
*ss
;
6216 do_each_subsys_mask(ss
, ssid
, have_release_callback
) {
6218 } while_each_subsys_mask();
6220 spin_lock_irq(&css_set_lock
);
6221 css_set_skip_task_iters(task_css_set(task
), task
);
6222 list_del_init(&task
->cg_list
);
6223 spin_unlock_irq(&css_set_lock
);
6226 void cgroup_free(struct task_struct
*task
)
6228 struct css_set
*cset
= task_css_set(task
);
6232 static int __init
cgroup_disable(char *str
)
6234 struct cgroup_subsys
*ss
;
6238 while ((token
= strsep(&str
, ",")) != NULL
) {
6242 for_each_subsys(ss
, i
) {
6243 if (strcmp(token
, ss
->name
) &&
6244 strcmp(token
, ss
->legacy_name
))
6246 cgroup_disable_mask
|= 1 << i
;
6251 __setup("cgroup_disable=", cgroup_disable
);
6253 void __init __weak
enable_debug_cgroup(void) { }
6255 static int __init
enable_cgroup_debug(char *str
)
6257 cgroup_debug
= true;
6258 enable_debug_cgroup();
6261 __setup("cgroup_debug", enable_cgroup_debug
);
6264 * css_tryget_online_from_dir - get corresponding css from a cgroup dentry
6265 * @dentry: directory dentry of interest
6266 * @ss: subsystem of interest
6268 * If @dentry is a directory for a cgroup which has @ss enabled on it, try
6269 * to get the corresponding css and return it. If such css doesn't exist
6270 * or can't be pinned, an ERR_PTR value is returned.
6272 struct cgroup_subsys_state
*css_tryget_online_from_dir(struct dentry
*dentry
,
6273 struct cgroup_subsys
*ss
)
6275 struct kernfs_node
*kn
= kernfs_node_from_dentry(dentry
);
6276 struct file_system_type
*s_type
= dentry
->d_sb
->s_type
;
6277 struct cgroup_subsys_state
*css
= NULL
;
6278 struct cgroup
*cgrp
;
6280 /* is @dentry a cgroup dir? */
6281 if ((s_type
!= &cgroup_fs_type
&& s_type
!= &cgroup2_fs_type
) ||
6282 !kn
|| kernfs_type(kn
) != KERNFS_DIR
)
6283 return ERR_PTR(-EBADF
);
6288 * This path doesn't originate from kernfs and @kn could already
6289 * have been or be removed at any point. @kn->priv is RCU
6290 * protected for this access. See css_release_work_fn() for details.
6292 cgrp
= rcu_dereference(*(void __rcu __force
**)&kn
->priv
);
6294 css
= cgroup_css(cgrp
, ss
);
6296 if (!css
|| !css_tryget_online(css
))
6297 css
= ERR_PTR(-ENOENT
);
6304 * css_from_id - lookup css by id
6305 * @id: the cgroup id
6306 * @ss: cgroup subsys to be looked into
6308 * Returns the css if there's valid one with @id, otherwise returns NULL.
6309 * Should be called under rcu_read_lock().
6311 struct cgroup_subsys_state
*css_from_id(int id
, struct cgroup_subsys
*ss
)
6313 WARN_ON_ONCE(!rcu_read_lock_held());
6314 return idr_find(&ss
->css_idr
, id
);
6318 * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path
6319 * @path: path on the default hierarchy
6321 * Find the cgroup at @path on the default hierarchy, increment its
6322 * reference count and return it. Returns pointer to the found cgroup on
6323 * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR)
6324 * if @path points to a non-directory.
6326 struct cgroup
*cgroup_get_from_path(const char *path
)
6328 struct kernfs_node
*kn
;
6329 struct cgroup
*cgrp
;
6331 mutex_lock(&cgroup_mutex
);
6333 kn
= kernfs_walk_and_get(cgrp_dfl_root
.cgrp
.kn
, path
);
6335 if (kernfs_type(kn
) == KERNFS_DIR
) {
6337 cgroup_get_live(cgrp
);
6339 cgrp
= ERR_PTR(-ENOTDIR
);
6343 cgrp
= ERR_PTR(-ENOENT
);
6346 mutex_unlock(&cgroup_mutex
);
6349 EXPORT_SYMBOL_GPL(cgroup_get_from_path
);
6352 * cgroup_get_from_fd - get a cgroup pointer from a fd
6353 * @fd: fd obtained by open(cgroup2_dir)
6355 * Find the cgroup from a fd which should be obtained
6356 * by opening a cgroup directory. Returns a pointer to the
6357 * cgroup on success. ERR_PTR is returned if the cgroup
6360 struct cgroup
*cgroup_get_from_fd(int fd
)
6362 struct cgroup
*cgrp
;
6367 return ERR_PTR(-EBADF
);
6369 cgrp
= cgroup_get_from_file(f
);
6373 EXPORT_SYMBOL_GPL(cgroup_get_from_fd
);
6375 static u64
power_of_ten(int power
)
6384 * cgroup_parse_float - parse a floating number
6385 * @input: input string
6386 * @dec_shift: number of decimal digits to shift
6389 * Parse a decimal floating point number in @input and store the result in
6390 * @v with decimal point right shifted @dec_shift times. For example, if
6391 * @input is "12.3456" and @dec_shift is 3, *@v will be set to 12345.
6392 * Returns 0 on success, -errno otherwise.
6394 * There's nothing cgroup specific about this function except that it's
6395 * currently the only user.
6397 int cgroup_parse_float(const char *input
, unsigned dec_shift
, s64
*v
)
6399 s64 whole
, frac
= 0;
6400 int fstart
= 0, fend
= 0, flen
;
6402 if (!sscanf(input
, "%lld.%n%lld%n", &whole
, &fstart
, &frac
, &fend
))
6407 flen
= fend
> fstart
? fend
- fstart
: 0;
6408 if (flen
< dec_shift
)
6409 frac
*= power_of_ten(dec_shift
- flen
);
6411 frac
= DIV_ROUND_CLOSEST_ULL(frac
, power_of_ten(flen
- dec_shift
));
6413 *v
= whole
* power_of_ten(dec_shift
) + frac
;
6418 * sock->sk_cgrp_data handling. For more info, see sock_cgroup_data
6419 * definition in cgroup-defs.h.
6421 #ifdef CONFIG_SOCK_CGROUP_DATA
6423 #if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID)
6425 DEFINE_SPINLOCK(cgroup_sk_update_lock
);
6426 static bool cgroup_sk_alloc_disabled __read_mostly
;
6428 void cgroup_sk_alloc_disable(void)
6430 if (cgroup_sk_alloc_disabled
)
6432 pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n");
6433 cgroup_sk_alloc_disabled
= true;
6438 #define cgroup_sk_alloc_disabled false
6442 void cgroup_sk_alloc(struct sock_cgroup_data
*skcd
)
6444 if (cgroup_sk_alloc_disabled
) {
6445 skcd
->no_refcnt
= 1;
6449 /* Don't associate the sock with unrelated interrupted task's cgroup. */
6456 struct css_set
*cset
;
6458 cset
= task_css_set(current
);
6459 if (likely(cgroup_tryget(cset
->dfl_cgrp
))) {
6460 skcd
->val
= (unsigned long)cset
->dfl_cgrp
;
6461 cgroup_bpf_get(cset
->dfl_cgrp
);
6470 void cgroup_sk_clone(struct sock_cgroup_data
*skcd
)
6473 if (skcd
->no_refcnt
)
6476 * We might be cloning a socket which is left in an empty
6477 * cgroup and the cgroup might have already been rmdir'd.
6478 * Don't use cgroup_get_live().
6480 cgroup_get(sock_cgroup_ptr(skcd
));
6481 cgroup_bpf_get(sock_cgroup_ptr(skcd
));
6485 void cgroup_sk_free(struct sock_cgroup_data
*skcd
)
6487 struct cgroup
*cgrp
= sock_cgroup_ptr(skcd
);
6489 if (skcd
->no_refcnt
)
6491 cgroup_bpf_put(cgrp
);
6495 #endif /* CONFIG_SOCK_CGROUP_DATA */
6497 #ifdef CONFIG_CGROUP_BPF
6498 int cgroup_bpf_attach(struct cgroup
*cgrp
,
6499 struct bpf_prog
*prog
, struct bpf_prog
*replace_prog
,
6500 struct bpf_cgroup_link
*link
,
6501 enum bpf_attach_type type
,
6506 mutex_lock(&cgroup_mutex
);
6507 ret
= __cgroup_bpf_attach(cgrp
, prog
, replace_prog
, link
, type
, flags
);
6508 mutex_unlock(&cgroup_mutex
);
6512 int cgroup_bpf_detach(struct cgroup
*cgrp
, struct bpf_prog
*prog
,
6513 enum bpf_attach_type type
)
6517 mutex_lock(&cgroup_mutex
);
6518 ret
= __cgroup_bpf_detach(cgrp
, prog
, NULL
, type
);
6519 mutex_unlock(&cgroup_mutex
);
6523 int cgroup_bpf_query(struct cgroup
*cgrp
, const union bpf_attr
*attr
,
6524 union bpf_attr __user
*uattr
)
6528 mutex_lock(&cgroup_mutex
);
6529 ret
= __cgroup_bpf_query(cgrp
, attr
, uattr
);
6530 mutex_unlock(&cgroup_mutex
);
6533 #endif /* CONFIG_CGROUP_BPF */
6536 static ssize_t
show_delegatable_files(struct cftype
*files
, char *buf
,
6537 ssize_t size
, const char *prefix
)
6542 for (cft
= files
; cft
&& cft
->name
[0] != '\0'; cft
++) {
6543 if (!(cft
->flags
& CFTYPE_NS_DELEGATABLE
))
6547 ret
+= snprintf(buf
+ ret
, size
- ret
, "%s.", prefix
);
6549 ret
+= snprintf(buf
+ ret
, size
- ret
, "%s\n", cft
->name
);
6551 if (WARN_ON(ret
>= size
))
6558 static ssize_t
delegate_show(struct kobject
*kobj
, struct kobj_attribute
*attr
,
6561 struct cgroup_subsys
*ss
;
6565 ret
= show_delegatable_files(cgroup_base_files
, buf
, PAGE_SIZE
- ret
,
6568 for_each_subsys(ss
, ssid
)
6569 ret
+= show_delegatable_files(ss
->dfl_cftypes
, buf
+ ret
,
6571 cgroup_subsys_name
[ssid
]);
6575 static struct kobj_attribute cgroup_delegate_attr
= __ATTR_RO(delegate
);
6577 static ssize_t
features_show(struct kobject
*kobj
, struct kobj_attribute
*attr
,
6580 return snprintf(buf
, PAGE_SIZE
,
6582 "memory_localevents\n"
6583 "memory_recursiveprot\n");
6585 static struct kobj_attribute cgroup_features_attr
= __ATTR_RO(features
);
6587 static struct attribute
*cgroup_sysfs_attrs
[] = {
6588 &cgroup_delegate_attr
.attr
,
6589 &cgroup_features_attr
.attr
,
6593 static const struct attribute_group cgroup_sysfs_attr_group
= {
6594 .attrs
= cgroup_sysfs_attrs
,
6598 static int __init
cgroup_sysfs_init(void)
6600 return sysfs_create_group(kernel_kobj
, &cgroup_sysfs_attr_group
);
6602 subsys_initcall(cgroup_sysfs_init
);
6604 #endif /* CONFIG_SYSFS */