2 * lxc: linux Container library
4 * Copyright © 2016 Canonical Ltd.
7 * Serge Hallyn <serge.hallyn@ubuntu.com>
8 * Christian Brauner <christian.brauner@ubuntu.com>
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
15 * This library is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with this library; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
26 * cgfs-ng.c: this is a new, simplified implementation of a filesystem
27 * cgroup backend. The original cgfs.c was designed to be as flexible
28 * as possible. It would try to find cgroup filesystems no matter where
29 * or how you had them mounted, and deduce the most usable mount for
32 * This new implementation assumes that cgroup filesystems are mounted
33 * under /sys/fs/cgroup/clist where clist is either the controller, or
34 * a comman-separated list of controllers.
48 #include <linux/kdev_t.h>
49 #include <linux/types.h>
50 #include <sys/types.h>
54 #include "cgroup_utils.h"
58 #include "storage/storage.h"
61 lxc_log_define(lxc_cgfsng
, lxc
);
63 static struct cgroup_ops cgfsng_ops
;
65 /* A descriptor for a mounted hierarchy
69 * Either NULL, or a null-terminated list of all the co-mounted controllers.
71 * Either NULL, or a null-terminated list of all enabled controllers.
74 * - The mountpoint we will use.
76 * It will be either /sys/fs/cgroup/controller or
77 * /sys/fs/cgroup/controllerlist.
79 * It will either be /sys/fs/cgroup or /sys/fs/cgroup/<mountpoint-name>
80 * depending on whether this is a hybrid cgroup layout (mix of legacy and
81 * unified hierarchies) or a pure unified cgroup layout.
84 * - The cgroup under which the container cgroup path
85 * is created. This will be either the caller's cgroup (if not root), or
86 * init's cgroup (if root).
89 * - The full path to the containers cgroup.
93 * If the hierarchy is a legacy hierarchy this will be set to
96 * If the hierarchy is a legacy hierarchy this will be set to
97 * CGROUP2_SUPER_MAGIC.
107 /* The cgroup data which is attached to the lxc_handler.
110 * - A copy of lxc.cgroup.pattern.
113 * - If not null, the cgroup which was created for the container. For each
114 * hierarchy, it is created under the @hierarchy->base_cgroup directory.
115 * Relative to the base_cgroup it is the same for all hierarchies.
118 * - The name of the container.
121 * - A copy of the container's cgroup information. This overrides
125 * - What cgroup layout the container is running with.
126 * - CGROUP_LAYOUT_UNKNOWN
127 * The cgroup layout could not be determined. This should be treated as an
129 * - CGROUP_LAYOUT_LEGACY
130 * The container is running with all controllers mounted into legacy cgroup
132 * - CGROUP_LAYOUT_HYBRID
133 * The container is running with at least one controller mounted into a
134 * legacy cgroup hierarchy and a mountpoint for the unified hierarchy. The
135 * unified hierarchy can be empty (no controllers enabled) or non-empty
136 * (controllers enabled).
137 * - CGROUP_LAYOUT_UNIFIED
138 * The container is running on a pure unified cgroup hierarchy. The unified
139 * hierarchy can be empty (no controllers enabled) or non-empty (controllers
142 struct cgfsng_handler_data
{
143 char *cgroup_pattern
;
144 char *container_cgroup
; /* cgroup we created for the container */
145 char *name
; /* container name */
146 /* per-container cgroup information */
147 struct lxc_cgroup cgroup_meta
;
148 cgroup_layout_t cgroup_layout
;
152 * - A NULL-terminated array of struct hierarchy, one per legacy hierarchy. No
153 * duplicates. First sufficient, writeable mounted hierarchy wins.
155 struct hierarchy
**hierarchies
;
156 /* Pointer to the unified hierarchy in the null terminated list @hierarchies.
157 * This is merely a convenience for hybrid cgroup layouts to easily retrieve the
158 * unified hierarchy without iterating throught @hierarchies.
160 struct hierarchy
*unified
;
163 * - What cgroup layout the container is running with.
164 * - CGROUP_LAYOUT_UNKNOWN
165 * The cgroup layout could not be determined. This should be treated as an
167 * - CGROUP_LAYOUT_LEGACY
168 * The container is running with all controllers mounted into legacy cgroup
170 * - CGROUP_LAYOUT_HYBRID
171 * The container is running with at least one controller mounted into a
172 * legacy cgroup hierarchy and a mountpoint for the unified hierarchy. The
173 * unified hierarchy can be empty (no controllers enabled) or non-empty
174 * (controllers enabled).
175 * - CGROUP_LAYOUT_UNIFIED
176 * The container is running on a pure unified cgroup hierarchy. The unified
177 * hierarchy can be empty (no controllers enabled) or non-empty (controllers
180 cgroup_layout_t cgroup_layout
;
181 /* What controllers is the container supposed to use. */
185 * - Whether to print debug info to stdout for the cgfsng driver.
187 static bool lxc_cgfsng_debug
;
189 #define CGFSNG_DEBUG(format, ...) \
191 if (lxc_cgfsng_debug) \
192 printf("cgfsng: " format, ##__VA_ARGS__); \
195 static void free_string_list(char **clist
)
202 for (i
= 0; clist
[i
]; i
++)
208 /* Allocate a pointer, do not fail. */
209 static void *must_alloc(size_t sz
)
211 return must_realloc(NULL
, sz
);
214 /* Given a pointer to a null-terminated array of pointers, realloc to add one
215 * entry, and point the new entry to NULL. Do not fail. Return the index to the
216 * second-to-last entry - that is, the one which is now available for use
217 * (keeping the list null-terminated).
219 static int append_null_to_list(void ***list
)
224 for (; (*list
)[newentry
]; newentry
++)
227 *list
= must_realloc(*list
, (newentry
+ 2) * sizeof(void **));
228 (*list
)[newentry
+ 1] = NULL
;
232 /* Given a null-terminated array of strings, check whether @entry is one of the
235 static bool string_in_list(char **list
, const char *entry
)
242 for (i
= 0; list
[i
]; i
++)
243 if (strcmp(list
[i
], entry
) == 0)
249 /* Return a copy of @entry prepending "name=", i.e. turn "systemd" into
250 * "name=systemd". Do not fail.
252 static char *cg_legacy_must_prefix_named(char *entry
)
258 prefixed
= must_alloc(len
+ 6);
260 memcpy(prefixed
, "name=", sizeof("name="));
261 memcpy(prefixed
+ sizeof("name="), entry
, len
);
262 prefixed
[len
+ 5] = '\0';
266 /* Append an entry to the clist. Do not fail. @clist must be NULL the first time
269 * We also handle named subsystems here. Any controller which is not a kernel
270 * subsystem, we prefix "name=". Any which is both a kernel and named subsystem,
271 * we refuse to use because we're not sure which we have here.
272 * (TODO: We could work around this in some cases by just remounting to be
273 * unambiguous, or by comparing mountpoint contents with current cgroup.)
275 * The last entry will always be NULL.
277 static void must_append_controller(char **klist
, char **nlist
, char ***clist
,
283 if (string_in_list(klist
, entry
) && string_in_list(nlist
, entry
)) {
284 ERROR("Refusing to use ambiguous controller \"%s\"", entry
);
285 ERROR("It is both a named and kernel subsystem");
289 newentry
= append_null_to_list((void ***)clist
);
291 if (strncmp(entry
, "name=", 5) == 0)
292 copy
= must_copy_string(entry
);
293 else if (string_in_list(klist
, entry
))
294 copy
= must_copy_string(entry
);
296 copy
= cg_legacy_must_prefix_named(entry
);
298 (*clist
)[newentry
] = copy
;
301 static void free_handler_data(struct cgfsng_handler_data
*d
)
303 free(d
->cgroup_pattern
);
304 free(d
->container_cgroup
);
306 if (d
->cgroup_meta
.dir
)
307 free(d
->cgroup_meta
.dir
);
308 if (d
->cgroup_meta
.controllers
)
309 free(d
->cgroup_meta
.controllers
);
313 /* Given a handler's cgroup data, return the struct hierarchy for the controller
314 * @c, or NULL if there is none.
316 struct hierarchy
*get_hierarchy(const char *c
)
323 for (i
= 0; hierarchies
[i
]; i
++) {
325 /* This is the empty unified hierarchy. */
326 if (hierarchies
[i
]->controllers
&&
327 !hierarchies
[i
]->controllers
[0])
328 return hierarchies
[i
];
333 if (string_in_list(hierarchies
[i
]->controllers
, c
))
334 return hierarchies
[i
];
340 #define BATCH_SIZE 50
341 static void batch_realloc(char **mem
, size_t oldlen
, size_t newlen
)
343 int newbatches
= (newlen
/ BATCH_SIZE
) + 1;
344 int oldbatches
= (oldlen
/ BATCH_SIZE
) + 1;
346 if (!*mem
|| newbatches
> oldbatches
) {
347 *mem
= must_realloc(*mem
, newbatches
* BATCH_SIZE
);
351 static void append_line(char **dest
, size_t oldlen
, char *new, size_t newlen
)
353 size_t full
= oldlen
+ newlen
;
355 batch_realloc(dest
, oldlen
, full
+ 1);
357 memcpy(*dest
+ oldlen
, new, newlen
+ 1);
360 /* Slurp in a whole file */
361 static char *read_file(const char *fnam
)
364 char *line
= NULL
, *buf
= NULL
;
365 size_t len
= 0, fulllen
= 0;
368 f
= fopen(fnam
, "r");
371 while ((linelen
= getline(&line
, &len
, f
)) != -1) {
372 append_line(&buf
, fulllen
, line
, linelen
);
380 /* Taken over modified from the kernel sources. */
381 #define NBITS 32 /* bits in uint32_t */
382 #define DIV_ROUND_UP(n, d) (((n) + (d)-1) / (d))
383 #define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, NBITS)
385 static void set_bit(unsigned bit
, uint32_t *bitarr
)
387 bitarr
[bit
/ NBITS
] |= (1 << (bit
% NBITS
));
390 static void clear_bit(unsigned bit
, uint32_t *bitarr
)
392 bitarr
[bit
/ NBITS
] &= ~(1 << (bit
% NBITS
));
395 static bool is_set(unsigned bit
, uint32_t *bitarr
)
397 return (bitarr
[bit
/ NBITS
] & (1 << (bit
% NBITS
))) != 0;
400 /* Create cpumask from cpulist aka turn:
408 static uint32_t *lxc_cpumask(char *buf
, size_t nbits
)
413 char *saveptr
= NULL
;
415 arrlen
= BITS_TO_LONGS(nbits
);
416 bitarr
= calloc(arrlen
, sizeof(uint32_t));
420 for (; (token
= strtok_r(buf
, ",", &saveptr
)); buf
= NULL
) {
425 start
= strtoul(token
, NULL
, 0);
427 range
= strchr(token
, '-');
429 end
= strtoul(range
+ 1, NULL
, 0);
431 if (!(start
<= end
)) {
442 set_bit(start
++, bitarr
);
448 /* Turn cpumask into simple, comma-separated cpulist. */
449 static char *lxc_cpumask_to_cpulist(uint32_t *bitarr
, size_t nbits
)
453 char numstr
[LXC_NUMSTRLEN64
] = {0};
454 char **cpulist
= NULL
;
456 for (i
= 0; i
<= nbits
; i
++) {
457 if (is_set(i
, bitarr
)) {
458 ret
= snprintf(numstr
, LXC_NUMSTRLEN64
, "%zu", i
);
459 if (ret
< 0 || (size_t)ret
>= LXC_NUMSTRLEN64
) {
460 lxc_free_array((void **)cpulist
, free
);
463 if (lxc_append_string(&cpulist
, numstr
) < 0) {
464 lxc_free_array((void **)cpulist
, free
);
469 return lxc_string_join(",", (const char **)cpulist
, false);
472 static ssize_t
get_max_cpus(char *cpulist
)
475 char *maxcpus
= cpulist
;
478 c1
= strrchr(maxcpus
, ',');
482 c2
= strrchr(maxcpus
, '-');
492 else if (!c1
&& c2
) /* The reverse case is obvs. not needed. */
495 /* If the above logic is correct, c1 should always hold a valid string
500 cpus
= strtoul(c1
, NULL
, 0);
507 #define __ISOL_CPUS "/sys/devices/system/cpu/isolated"
508 static bool cg_legacy_filter_and_set_cpus(char *path
, bool am_initialized
)
510 char *lastslash
, *fpath
, oldv
;
514 ssize_t maxposs
= 0, maxisol
= 0;
515 char *cpulist
= NULL
, *posscpus
= NULL
, *isolcpus
= NULL
;
516 uint32_t *possmask
= NULL
, *isolmask
= NULL
;
517 bool bret
= false, flipped_bit
= false;
519 lastslash
= strrchr(path
, '/');
520 if (!lastslash
) { /* bug... this shouldn't be possible */
521 ERROR("Invalid path: %s.", path
);
526 fpath
= must_make_path(path
, "cpuset.cpus", NULL
);
527 posscpus
= read_file(fpath
);
529 SYSERROR("Could not read file: %s.\n", fpath
);
533 /* Get maximum number of cpus found in possible cpuset. */
534 maxposs
= get_max_cpus(posscpus
);
538 if (!file_exists(__ISOL_CPUS
)) {
539 /* This system doesn't expose isolated cpus. */
540 DEBUG("Path: "__ISOL_CPUS
" to read isolated cpus from does not exist.\n");
542 /* No isolated cpus but we weren't already initialized by
543 * someone. We should simply copy the parents cpuset.cpus
546 if (!am_initialized
) {
547 DEBUG("Copying cpuset of parent cgroup.");
550 /* No isolated cpus but we were already initialized by someone.
551 * Nothing more to do for us.
556 isolcpus
= read_file(__ISOL_CPUS
);
558 SYSERROR("Could not read file "__ISOL_CPUS
);
561 if (!isdigit(isolcpus
[0])) {
562 DEBUG("No isolated cpus detected.");
564 /* No isolated cpus but we weren't already initialized by
565 * someone. We should simply copy the parents cpuset.cpus
568 if (!am_initialized
) {
569 DEBUG("Copying cpuset of parent cgroup.");
572 /* No isolated cpus but we were already initialized by someone.
573 * Nothing more to do for us.
578 /* Get maximum number of cpus found in isolated cpuset. */
579 maxisol
= get_max_cpus(isolcpus
);
583 if (maxposs
< maxisol
)
587 possmask
= lxc_cpumask(posscpus
, maxposs
);
589 ERROR("Could not create cpumask for all possible cpus.\n");
593 isolmask
= lxc_cpumask(isolcpus
, maxposs
);
595 ERROR("Could not create cpumask for all isolated cpus.\n");
599 for (i
= 0; i
<= maxposs
; i
++) {
600 if (is_set(i
, isolmask
) && is_set(i
, possmask
)) {
602 clear_bit(i
, possmask
);
607 DEBUG("No isolated cpus present in cpuset.");
610 DEBUG("Removed isolated cpus from cpuset.");
612 cpulist
= lxc_cpumask_to_cpulist(possmask
, maxposs
);
614 ERROR("Could not create cpu list.\n");
621 fpath
= must_make_path(path
, "cpuset.cpus", NULL
);
622 ret
= lxc_write_to_file(fpath
, cpulist
, strlen(cpulist
), false);
624 SYSERROR("Could not write cpu list to: %s.\n", fpath
);
637 if (posscpus
!= cpulist
)
645 /* Copy contents of parent(@path)/@file to @path/@file */
646 static bool copy_parent_file(char *path
, char *file
)
648 char *lastslash
, *value
= NULL
, *fpath
, oldv
;
652 lastslash
= strrchr(path
, '/');
653 if (!lastslash
) { /* bug... this shouldn't be possible */
654 ERROR("cgfsng:copy_parent_file: bad path %s", path
);
659 fpath
= must_make_path(path
, file
, NULL
);
660 len
= lxc_read_from_file(fpath
, NULL
, 0);
663 value
= must_alloc(len
+ 1);
664 if (lxc_read_from_file(fpath
, value
, len
) != len
)
668 fpath
= must_make_path(path
, file
, NULL
);
669 ret
= lxc_write_to_file(fpath
, value
, len
, false);
671 SYSERROR("Unable to write %s to %s", value
, fpath
);
677 SYSERROR("Error reading '%s'", fpath
);
684 * Initialize the cpuset hierarchy in first directory of @gname and
685 * set cgroup.clone_children so that children inherit settings.
686 * Since the h->base_path is populated by init or ourselves, we know
687 * it is already initialized.
689 static bool cg_legacy_handle_cpuset_hierarchy(struct hierarchy
*h
, char *cgname
)
691 char *cgpath
, *clonechildrenpath
, v
, *slash
;
693 if (!string_in_list(h
->controllers
, "cpuset"))
698 slash
= strchr(cgname
, '/');
702 cgpath
= must_make_path(h
->mountpoint
, h
->base_cgroup
, cgname
, NULL
);
705 if (mkdir(cgpath
, 0755) < 0 && errno
!= EEXIST
) {
706 SYSERROR("Failed to create '%s'", cgpath
);
711 clonechildrenpath
= must_make_path(cgpath
, "cgroup.clone_children", NULL
);
712 /* unified hierarchy doesn't have clone_children */
713 if (!file_exists(clonechildrenpath
)) {
714 free(clonechildrenpath
);
718 if (lxc_read_from_file(clonechildrenpath
, &v
, 1) < 0) {
719 SYSERROR("Failed to read '%s'", clonechildrenpath
);
720 free(clonechildrenpath
);
725 /* Make sure any isolated cpus are removed from cpuset.cpus. */
726 if (!cg_legacy_filter_and_set_cpus(cgpath
, v
== '1')) {
727 SYSERROR("Failed to remove isolated cpus.");
728 free(clonechildrenpath
);
733 if (v
== '1') { /* already set for us by someone else */
734 DEBUG("\"cgroup.clone_children\" was already set to \"1\".");
735 free(clonechildrenpath
);
740 /* copy parent's settings */
741 if (!copy_parent_file(cgpath
, "cpuset.mems")) {
742 SYSERROR("Failed to copy \"cpuset.mems\" settings.");
744 free(clonechildrenpath
);
749 if (lxc_write_to_file(clonechildrenpath
, "1", 1, false) < 0) {
750 /* Set clone_children so children inherit our settings */
751 SYSERROR("Failed to write 1 to %s", clonechildrenpath
);
752 free(clonechildrenpath
);
755 free(clonechildrenpath
);
760 * Given two null-terminated lists of strings, return true if any string
763 static bool controller_lists_intersect(char **l1
, char **l2
)
770 for (i
= 0; l1
[i
]; i
++) {
771 if (string_in_list(l2
, l1
[i
]))
778 * For a null-terminated list of controllers @clist, return true if any of
779 * those controllers is already listed the null-terminated list of
780 * hierarchies @hlist. Realistically, if one is present, all must be present.
782 static bool controller_list_is_dup(struct hierarchy
**hlist
, char **clist
)
788 for (i
= 0; hlist
[i
]; i
++)
789 if (controller_lists_intersect(hlist
[i
]->controllers
, clist
))
796 * Return true if the controller @entry is found in the null-terminated
797 * list of hierarchies @hlist
799 static bool controller_found(struct hierarchy
**hlist
, char *entry
)
806 for (i
= 0; hlist
[i
]; i
++)
807 if (string_in_list(hlist
[i
]->controllers
, entry
))
814 * Return true if all of the controllers which we require have been found.
815 * The required list is freezer and anything in * lxc.cgroup.use.
817 static bool all_controllers_found(void)
819 char *p
, *saveptr
= NULL
;
820 struct hierarchy
** hlist
= hierarchies
;
822 if (!controller_found(hlist
, "freezer")) {
823 CGFSNG_DEBUG("No freezer controller mountpoint found\n");
830 for (p
= strtok_r(cgroup_use
, ",", &saveptr
); p
;
831 p
= strtok_r(NULL
, ",", &saveptr
)) {
832 if (!controller_found(hlist
, p
)) {
833 CGFSNG_DEBUG("No %s controller mountpoint found\n", p
);
842 * Get the controllers from a mountinfo line
843 * There are other ways we could get this info. For lxcfs, field 3
844 * is /cgroup/controller-list. For cgroupfs, we could parse the mount
845 * options. But we simply assume that the mountpoint must be
846 * /sys/fs/cgroup/controller-list
848 static char **cg_hybrid_get_controllers(char **klist
, char **nlist
, char *line
,
851 /* the fourth field is /sys/fs/cgroup/comma-delimited-controller-list */
853 char *dup
, *p2
, *tok
;
854 char *p
= line
, *saveptr
= NULL
, *sep
= ",";
857 for (i
= 0; i
< 4; i
++) {
864 /* note - if we change how mountinfo works, then our caller
865 * will need to verify /sys/fs/cgroup/ in this field */
866 if (strncmp(p
, "/sys/fs/cgroup/", 15)) {
867 CGFSNG_DEBUG("Found hierarchy not under /sys/fs/cgroup: \"%s\"\n", p
);
874 CGFSNG_DEBUG("Corrupt mountinfo\n");
879 if (type
== CGROUP_SUPER_MAGIC
) {
880 /* strdup() here for v1 hierarchies. Otherwise strtok_r() will
881 * destroy mountpoints such as "/sys/fs/cgroup/cpu,cpuacct".
887 for (tok
= strtok_r(dup
, sep
, &saveptr
); tok
;
888 tok
= strtok_r(NULL
, sep
, &saveptr
))
889 must_append_controller(klist
, nlist
, &aret
, tok
);
897 static char **cg_unified_make_empty_controller(void)
902 newentry
= append_null_to_list((void ***)&aret
);
903 aret
[newentry
] = NULL
;
907 static char **cg_unified_get_controllers(const char *file
)
910 char *saveptr
= NULL
, *sep
= " \t\n";
913 buf
= read_file(file
);
917 for (tok
= strtok_r(buf
, sep
, &saveptr
); tok
;
918 tok
= strtok_r(NULL
, sep
, &saveptr
)) {
922 newentry
= append_null_to_list((void ***)&aret
);
923 copy
= must_copy_string(tok
);
924 aret
[newentry
] = copy
;
931 static struct hierarchy
*add_hierarchy(char **clist
, char *mountpoint
,
932 char *base_cgroup
, int type
)
934 struct hierarchy
*new;
937 new = must_alloc(sizeof(*new));
938 new->controllers
= clist
;
939 new->mountpoint
= mountpoint
;
940 new->base_cgroup
= base_cgroup
;
941 new->fullcgpath
= NULL
;
944 newentry
= append_null_to_list((void ***)&hierarchies
);
945 hierarchies
[newentry
] = new;
950 * Get a copy of the mountpoint from @line, which is a line from
951 * /proc/self/mountinfo
953 static char *cg_hybrid_get_mountpoint(char *line
)
961 for (i
= 0; i
< 4; i
++) {
968 if (strncmp(p
, "/sys/fs/cgroup/", 15))
971 p2
= strchr(p
+ 15, ' ');
977 sret
= must_alloc(len
+ 1);
978 memcpy(sret
, p
, len
);
984 * Given a multi-line string, return a null-terminated copy of the
987 static char *copy_to_eol(char *p
)
989 char *p2
= strchr(p
, '\n'), *sret
;
996 sret
= must_alloc(len
+ 1);
997 memcpy(sret
, p
, len
);
1003 * cgline: pointer to character after the first ':' in a line in a
1004 * \n-terminated /proc/self/cgroup file. Check whether * controller c is
1007 static bool controller_in_clist(char *cgline
, char *c
)
1009 char *tok
, *saveptr
= NULL
, *eol
, *tmp
;
1012 eol
= strchr(cgline
, ':');
1017 tmp
= alloca(len
+ 1);
1018 memcpy(tmp
, cgline
, len
);
1021 for (tok
= strtok_r(tmp
, ",", &saveptr
); tok
;
1022 tok
= strtok_r(NULL
, ",", &saveptr
)) {
1023 if (strcmp(tok
, c
) == 0)
1031 * @basecginfo is a copy of /proc/$$/cgroup. Return the current
1032 * cgroup for @controller
1034 static char *cg_hybrid_get_current_cgroup(char *basecginfo
, char *controller
, int type
)
1036 char *p
= basecginfo
;
1039 bool is_cgv2_base_cgroup
= false;
1041 /* cgroup v2 entry in "/proc/<pid>/cgroup": "0::/some/path" */
1042 if ((type
== CGROUP2_SUPER_MAGIC
) && (*p
== '0'))
1043 is_cgv2_base_cgroup
= true;
1050 if (is_cgv2_base_cgroup
|| (controller
&& controller_in_clist(p
, controller
))) {
1055 return copy_to_eol(p
);
1058 p
= strchr(p
, '\n');
1065 static void must_append_string(char ***list
, char *entry
)
1067 int newentry
= append_null_to_list((void ***)list
);
1070 copy
= must_copy_string(entry
);
1071 (*list
)[newentry
] = copy
;
1074 static int get_existing_subsystems(char ***klist
, char ***nlist
)
1080 f
= fopen("/proc/self/cgroup", "r");
1084 while (getline(&line
, &len
, f
) != -1) {
1085 char *p
, *p2
, *tok
, *saveptr
= NULL
;
1086 p
= strchr(line
, ':');
1090 p2
= strchr(p
, ':');
1095 /* If the kernel has cgroup v2 support, then /proc/self/cgroup
1096 * contains an entry of the form:
1100 * In this case we use "cgroup2" as controller name.
1102 if ((p2
- p
) == 0) {
1103 must_append_string(klist
, "cgroup2");
1107 for (tok
= strtok_r(p
, ",", &saveptr
); tok
;
1108 tok
= strtok_r(NULL
, ",", &saveptr
)) {
1109 if (strncmp(tok
, "name=", 5) == 0)
1110 must_append_string(nlist
, tok
);
1112 must_append_string(klist
, tok
);
1121 static void trim(char *s
)
1123 size_t len
= strlen(s
);
1124 while ((len
> 1) && (s
[len
- 1] == '\n'))
1128 static void lxc_cgfsng_print_handler_data(const struct cgfsng_handler_data
*d
)
1130 printf("Cgroup information:\n");
1131 printf(" container name: %s\n", d
->name
? d
->name
: "(null)");
1132 printf(" lxc.cgroup.use: %s\n", cgroup_use
? cgroup_use
: "(null)");
1133 printf(" lxc.cgroup.pattern: %s\n",
1134 d
->cgroup_pattern
? d
->cgroup_pattern
: "(null)");
1135 printf(" lxc.cgroup.dir: %s\n",
1136 d
->cgroup_meta
.dir
? d
->cgroup_meta
.dir
: "(null)");
1137 printf(" cgroup: %s\n",
1138 d
->container_cgroup
? d
->container_cgroup
: "(null)");
1141 static void lxc_cgfsng_print_hierarchies()
1143 struct hierarchy
**it
;
1147 printf(" No hierarchies found\n");
1150 printf(" Hierarchies:\n");
1151 for (i
= 0, it
= hierarchies
; it
&& *it
; it
++, i
++) {
1154 printf(" %d: base_cgroup: %s\n", i
, (*it
)->base_cgroup
? (*it
)->base_cgroup
: "(null)");
1155 printf(" mountpoint: %s\n", (*it
)->mountpoint
? (*it
)->mountpoint
: "(null)");
1156 printf(" controllers:\n");
1157 for (j
= 0, cit
= (*it
)->controllers
; cit
&& *cit
; cit
++, j
++)
1158 printf(" %d: %s\n", j
, *cit
);
1162 static void lxc_cgfsng_print_basecg_debuginfo(char *basecginfo
, char **klist
,
1168 printf("basecginfo is:\n");
1169 printf("%s\n", basecginfo
);
1171 for (k
= 0, it
= klist
; it
&& *it
; it
++, k
++)
1172 printf("kernel subsystem %d: %s\n", k
, *it
);
1173 for (k
= 0, it
= nlist
; it
&& *it
; it
++, k
++)
1174 printf("named subsystem %d: %s\n", k
, *it
);
1177 static void lxc_cgfsng_print_debuginfo(const struct cgfsng_handler_data
*d
)
1179 lxc_cgfsng_print_handler_data(d
);
1180 lxc_cgfsng_print_hierarchies();
1184 * At startup, parse_hierarchies finds all the info we need about
1185 * cgroup mountpoints and current cgroups, and stores it in @d.
1187 static bool cg_hybrid_init(void)
1195 char **klist
= NULL
, **nlist
= NULL
;
1198 * Root spawned containers escape the current cgroup, so use init's
1199 * cgroups as our base in that case.
1201 will_escape
= (geteuid() == 0);
1203 basecginfo
= read_file("/proc/1/cgroup");
1205 basecginfo
= read_file("/proc/self/cgroup");
1209 ret
= get_existing_subsystems(&klist
, &nlist
);
1211 CGFSNG_DEBUG("Failed to retrieve available cgroup v1 controllers\n");
1216 f
= fopen("/proc/self/mountinfo", "r");
1218 CGFSNG_DEBUG("Failed to open \"/proc/self/mountinfo\"\n");
1223 if (lxc_cgfsng_debug
)
1224 lxc_cgfsng_print_basecg_debuginfo(basecginfo
, klist
, nlist
);
1226 while (getline(&line
, &len
, f
) != -1) {
1229 struct hierarchy
*new;
1230 char *mountpoint
= NULL
, *base_cgroup
= NULL
;
1231 char **controller_list
= NULL
;
1233 type
= get_cgroup_version(line
);
1237 if (type
== CGROUP2_SUPER_MAGIC
&& unified
)
1240 if (cgroup_layout
== CGROUP_LAYOUT_UNKNOWN
) {
1241 if (type
== CGROUP2_SUPER_MAGIC
)
1242 cgroup_layout
= CGROUP_LAYOUT_UNIFIED
;
1243 else if (type
== CGROUP_SUPER_MAGIC
)
1244 cgroup_layout
= CGROUP_LAYOUT_LEGACY
;
1245 } else if (cgroup_layout
== CGROUP_LAYOUT_UNIFIED
) {
1246 if (type
== CGROUP_SUPER_MAGIC
)
1247 cgroup_layout
= CGROUP_LAYOUT_HYBRID
;
1248 } else if (cgroup_layout
== CGROUP_LAYOUT_LEGACY
) {
1249 if (type
== CGROUP2_SUPER_MAGIC
)
1250 cgroup_layout
= CGROUP_LAYOUT_HYBRID
;
1253 controller_list
= cg_hybrid_get_controllers(klist
, nlist
, line
, type
);
1254 if (!controller_list
&& type
== CGROUP_SUPER_MAGIC
)
1257 if (type
== CGROUP_SUPER_MAGIC
)
1258 if (controller_list_is_dup(hierarchies
, controller_list
))
1261 mountpoint
= cg_hybrid_get_mountpoint(line
);
1263 CGFSNG_DEBUG("Failed parsing mountpoint from \"%s\"\n", line
);
1267 if (type
== CGROUP_SUPER_MAGIC
)
1268 base_cgroup
= cg_hybrid_get_current_cgroup(basecginfo
, controller_list
[0], CGROUP_SUPER_MAGIC
);
1270 base_cgroup
= cg_hybrid_get_current_cgroup(basecginfo
, NULL
, CGROUP2_SUPER_MAGIC
);
1272 CGFSNG_DEBUG("Failed to find current cgroup\n");
1277 prune_init_scope(base_cgroup
);
1278 if (type
== CGROUP2_SUPER_MAGIC
)
1279 writeable
= test_writeable_v2(mountpoint
, base_cgroup
);
1281 writeable
= test_writeable_v1(mountpoint
, base_cgroup
);
1285 if (type
== CGROUP2_SUPER_MAGIC
) {
1286 char *cgv2_ctrl_path
;
1288 cgv2_ctrl_path
= must_make_path(mountpoint
, base_cgroup
,
1289 "cgroup.controllers",
1292 controller_list
= cg_unified_get_controllers(cgv2_ctrl_path
);
1293 free(cgv2_ctrl_path
);
1294 if (!controller_list
)
1295 controller_list
= cg_unified_make_empty_controller();
1297 new = add_hierarchy(controller_list
, mountpoint
, base_cgroup
, type
);
1298 if (type
== CGROUP2_SUPER_MAGIC
&& !unified
)
1304 free_string_list(controller_list
);
1309 free_string_list(klist
);
1310 free_string_list(nlist
);
1317 if (lxc_cgfsng_debug
) {
1318 printf("writeable subsystems:\n");
1319 lxc_cgfsng_print_hierarchies();
1322 /* verify that all controllers in cgroup.use and all crucial
1323 * controllers are accounted for
1325 if (!all_controllers_found())
1331 static int cg_is_pure_unified(void) {
1336 ret
= statfs("/sys/fs/cgroup", &fs
);
1340 if (is_fs_type(&fs
, CGROUP2_SUPER_MAGIC
))
1341 return CGROUP2_SUPER_MAGIC
;
1346 /* Get current cgroup from /proc/self/cgroup for the cgroupfs v2 hierarchy. */
1347 static char *cg_unified_get_current_cgroup(void)
1354 will_escape
= (geteuid() == 0);
1356 basecginfo
= read_file("/proc/1/cgroup");
1358 basecginfo
= read_file("/proc/self/cgroup");
1362 base_cgroup
= strstr(basecginfo
, "0::/");
1364 goto cleanup_on_err
;
1366 base_cgroup
= base_cgroup
+ 3;
1367 copy
= copy_to_eol(base_cgroup
);
1369 goto cleanup_on_err
;
1379 static int cg_unified_init(void)
1382 char *mountpoint
, *subtree_path
;
1384 char *base_cgroup
= NULL
;
1386 ret
= cg_is_pure_unified();
1387 if (ret
== -ENOMEDIUM
)
1390 if (ret
!= CGROUP2_SUPER_MAGIC
)
1393 base_cgroup
= cg_unified_get_current_cgroup();
1396 prune_init_scope(base_cgroup
);
1398 /* We assume that we have already been given controllers to delegate
1399 * further down the hierarchy. If not it is up to the user to delegate
1402 mountpoint
= must_copy_string("/sys/fs/cgroup");
1403 subtree_path
= must_make_path(mountpoint
, base_cgroup
,
1404 "cgroup.subtree_control", NULL
);
1405 delegatable
= cg_unified_get_controllers(subtree_path
);
1408 delegatable
= cg_unified_make_empty_controller();
1409 if (!delegatable
[0])
1410 CGFSNG_DEBUG("No controllers are enabled for delegation\n");
1412 /* TODO: If the user requested specific controllers via lxc.cgroup.use
1413 * we should verify here. The reason I'm not doing it right is that I'm
1414 * not convinced that lxc.cgroup.use will be the future since it is a
1415 * global property. I much rather have an option that lets you request
1416 * controllers per container.
1419 add_hierarchy(delegatable
, mountpoint
, base_cgroup
, CGROUP2_SUPER_MAGIC
);
1420 unified
= hierarchies
[0];
1422 cgroup_layout
= CGROUP_LAYOUT_UNIFIED
;
1423 return CGROUP2_SUPER_MAGIC
;
1426 static bool cg_init(void)
1432 tmp
= lxc_global_config_value("lxc.cgroup.use");
1433 if (!cgroup_use
&& errno
!= 0) { /* lxc.cgroup.use can be NULL */
1434 CGFSNG_DEBUG("Failed to retrieve list of cgroups to use\n");
1437 cgroup_use
= must_copy_string(tmp
);
1439 ret
= cg_unified_init();
1443 if (ret
== CGROUP2_SUPER_MAGIC
)
1446 return cg_hybrid_init();
1449 static void *cgfsng_init(struct lxc_handler
*handler
)
1451 const char *cgroup_pattern
;
1452 struct cgfsng_handler_data
*d
;
1454 d
= must_alloc(sizeof(*d
));
1455 memset(d
, 0, sizeof(*d
));
1457 /* copy container name */
1458 d
->name
= must_copy_string(handler
->name
);
1460 /* copy per-container cgroup information */
1461 d
->cgroup_meta
.dir
= NULL
;
1462 d
->cgroup_meta
.controllers
= NULL
;
1463 if (handler
->conf
) {
1464 d
->cgroup_meta
.dir
= must_copy_string(handler
->conf
->cgroup_meta
.dir
);
1465 d
->cgroup_meta
.controllers
= must_copy_string(handler
->conf
->cgroup_meta
.controllers
);
1468 /* copy system-wide cgroup information */
1469 cgroup_pattern
= lxc_global_config_value("lxc.cgroup.pattern");
1470 if (!cgroup_pattern
) {
1471 /* lxc.cgroup.pattern is only NULL on error. */
1472 ERROR("Error getting cgroup pattern");
1475 d
->cgroup_pattern
= must_copy_string(cgroup_pattern
);
1477 d
->cgroup_layout
= cgroup_layout
;
1478 if (d
->cgroup_layout
== CGROUP_LAYOUT_LEGACY
)
1479 TRACE("Running with legacy cgroup layout");
1480 else if (d
->cgroup_layout
== CGROUP_LAYOUT_HYBRID
)
1481 TRACE("Running with hybrid cgroup layout");
1482 else if (d
->cgroup_layout
== CGROUP_LAYOUT_UNIFIED
)
1483 TRACE("Running with unified cgroup layout");
1485 WARN("Running with unknown cgroup layout");
1487 if (lxc_cgfsng_debug
)
1488 lxc_cgfsng_print_debuginfo(d
);
1493 free_handler_data(d
);
1497 static int recursive_destroy(char *dirname
)
1500 struct dirent
*direntp
;
1504 dir
= opendir(dirname
);
1508 while ((direntp
= readdir(dir
))) {
1512 if (!strcmp(direntp
->d_name
, ".") ||
1513 !strcmp(direntp
->d_name
, ".."))
1516 pathname
= must_make_path(dirname
, direntp
->d_name
, NULL
);
1518 ret
= lstat(pathname
, &mystat
);
1521 WARN("Failed to stat %s", pathname
);
1526 if (!S_ISDIR(mystat
.st_mode
))
1529 ret
= recursive_destroy(pathname
);
1536 ret
= rmdir(dirname
);
1539 WARN("%s - Failed to delete \"%s\"", strerror(errno
),
1544 ret
= closedir(dir
);
1547 WARN("%s - Failed to delete \"%s\"", strerror(errno
),
1555 static int cgroup_rmdir(char *container_cgroup
)
1559 if (!container_cgroup
|| !hierarchies
)
1562 for (i
= 0; hierarchies
[i
]; i
++) {
1564 struct hierarchy
*h
= hierarchies
[i
];
1569 ret
= recursive_destroy(h
->fullcgpath
);
1571 WARN("Failed to destroy \"%s\"", h
->fullcgpath
);
1573 free(h
->fullcgpath
);
1574 h
->fullcgpath
= NULL
;
1580 struct generic_userns_exec_data
{
1581 struct cgfsng_handler_data
*d
;
1582 struct lxc_conf
*conf
;
1583 uid_t origuid
; /* target uid in parent namespace */
1587 static int cgroup_rmdir_wrapper(void *data
)
1590 struct generic_userns_exec_data
*arg
= data
;
1591 uid_t nsuid
= (arg
->conf
->root_nsuid_map
!= NULL
) ? 0 : arg
->conf
->init_uid
;
1592 gid_t nsgid
= (arg
->conf
->root_nsgid_map
!= NULL
) ? 0 : arg
->conf
->init_gid
;
1594 ret
= setresgid(nsgid
, nsgid
, nsgid
);
1596 SYSERROR("Failed to setresgid(%d, %d, %d)", (int)nsgid
,
1597 (int)nsgid
, (int)nsgid
);
1601 ret
= setresuid(nsuid
, nsuid
, nsuid
);
1603 SYSERROR("Failed to setresuid(%d, %d, %d)", (int)nsuid
,
1604 (int)nsuid
, (int)nsuid
);
1608 ret
= setgroups(0, NULL
);
1609 if (ret
< 0 && errno
!= EPERM
) {
1610 SYSERROR("Failed to setgroups(0, NULL)");
1614 return cgroup_rmdir(arg
->d
->container_cgroup
);
1617 static void cgfsng_destroy(void *hdata
, struct lxc_conf
*conf
)
1620 struct cgfsng_handler_data
*d
= hdata
;
1621 struct generic_userns_exec_data wrap
;
1630 if (conf
&& !lxc_list_empty(&conf
->id_map
))
1631 ret
= userns_exec_1(conf
, cgroup_rmdir_wrapper
, &wrap
,
1632 "cgroup_rmdir_wrapper");
1634 ret
= cgroup_rmdir(d
->container_cgroup
);
1636 WARN("Failed to destroy cgroups");
1640 free_handler_data(d
);
1643 struct cgroup_ops
*cgfsng_ops_init(void)
1645 if (getenv("LXC_DEBUG_CGFSNG"))
1646 lxc_cgfsng_debug
= true;
1654 static bool cg_unified_create_cgroup(struct hierarchy
*h
, char *cgname
)
1657 size_t i
, parts_len
;
1658 size_t full_len
= 0;
1659 char *add_controllers
= NULL
, *cgroup
= NULL
;
1660 char **parts
= NULL
;
1663 if (h
->version
!= CGROUP2_SUPER_MAGIC
)
1666 if (!h
->controllers
)
1669 /* For now we simply enable all controllers that we have detected by
1670 * creating a string like "+memory +pids +cpu +io".
1671 * TODO: In the near future we might want to support "-<controller>"
1672 * etc. but whether supporting semantics like this make sense will need
1675 for (it
= h
->controllers
; it
&& *it
; it
++) {
1676 full_len
+= strlen(*it
) + 2;
1677 add_controllers
= must_realloc(add_controllers
, full_len
+ 1);
1678 if (h
->controllers
[0] == *it
)
1679 add_controllers
[0] = '\0';
1680 strcat(add_controllers
, "+");
1681 strcat(add_controllers
, *it
);
1682 if ((it
+ 1) && *(it
+ 1))
1683 strcat(add_controllers
, " ");
1686 parts
= lxc_string_split(cgname
, '/');
1689 parts_len
= lxc_array_len((void **)parts
);
1693 cgroup
= must_make_path(h
->mountpoint
, h
->base_cgroup
, NULL
);
1694 for (i
= 0; i
< parts_len
; i
++) {
1698 cgroup
= must_append_path(cgroup
, parts
[i
], NULL
);
1699 target
= must_make_path(cgroup
, "cgroup.subtree_control", NULL
);
1700 ret
= lxc_write_to_file(target
, add_controllers
, full_len
, false);
1703 SYSERROR("Could not enable \"%s\" controllers in the "
1704 "unified cgroup \"%s\"", add_controllers
, cgroup
);
1712 lxc_free_array((void **)parts
, free
);
1713 free(add_controllers
);
1718 static bool create_path_for_hierarchy(struct hierarchy
*h
, char *cgname
)
1722 h
->fullcgpath
= must_make_path(h
->mountpoint
, h
->base_cgroup
, cgname
, NULL
);
1723 if (dir_exists(h
->fullcgpath
)) { /* it must not already exist */
1724 ERROR("cgroup \"%s\" already existed", h
->fullcgpath
);
1728 if (!cg_legacy_handle_cpuset_hierarchy(h
, cgname
)) {
1729 ERROR("Failed to handle cgroupfs v1 cpuset controller");
1733 ret
= mkdir_p(h
->fullcgpath
, 0755);
1735 ERROR("Failed to create cgroup \"%s\"", h
->fullcgpath
);
1739 return cg_unified_create_cgroup(h
, cgname
);
1742 static void remove_path_for_hierarchy(struct hierarchy
*h
, char *cgname
)
1744 if (rmdir(h
->fullcgpath
) < 0)
1745 SYSERROR("Failed to clean up cgroup %s from failed creation attempt", h
->fullcgpath
);
1746 free(h
->fullcgpath
);
1747 h
->fullcgpath
= NULL
;
1751 * Try to create the same cgroup in all hierarchies.
1752 * Start with cgroup_pattern; next cgroup_pattern-1, -2, ..., -999
1754 static inline bool cgfsng_create(void *hdata
)
1758 char *container_cgroup
, *offset
, *tmp
;
1760 struct cgfsng_handler_data
*d
= hdata
;
1765 if (d
->container_cgroup
) {
1766 WARN("cgfsng_create called a second time");
1770 if (d
->cgroup_meta
.dir
)
1771 tmp
= lxc_string_join("/", (const char *[]){d
->cgroup_meta
.dir
, d
->name
, NULL
}, false);
1773 tmp
= lxc_string_replace("%n", d
->name
, d
->cgroup_pattern
);
1775 ERROR("Failed expanding cgroup name pattern");
1778 len
= strlen(tmp
) + 5; /* leave room for -NNN\0 */
1779 container_cgroup
= must_alloc(len
);
1780 strcpy(container_cgroup
, tmp
);
1782 offset
= container_cgroup
+ len
- 5;
1786 ERROR("Too many conflicting cgroup names");
1792 ret
= snprintf(offset
, 5, "-%d", idx
);
1793 if (ret
< 0 || (size_t)ret
>= 5) {
1794 FILE *f
= fopen("/dev/null", "w");
1796 fprintf(f
, "Workaround for GCC7 bug: "
1797 "https://gcc.gnu.org/bugzilla/"
1798 "show_bug.cgi?id=78969");
1803 for (i
= 0; hierarchies
[i
]; i
++) {
1804 if (!create_path_for_hierarchy(hierarchies
[i
], container_cgroup
)) {
1806 ERROR("Failed to create \"%s\"", hierarchies
[i
]->fullcgpath
);
1807 free(hierarchies
[i
]->fullcgpath
);
1808 hierarchies
[i
]->fullcgpath
= NULL
;
1809 for (j
= 0; j
< i
; j
++)
1810 remove_path_for_hierarchy(hierarchies
[j
], container_cgroup
);
1816 d
->container_cgroup
= container_cgroup
;
1820 free(container_cgroup
);
1824 static bool cgfsng_enter(void *hdata
, pid_t pid
)
1829 len
= snprintf(pidstr
, 25, "%d", pid
);
1830 if (len
< 0 || len
> 25)
1833 for (i
= 0; hierarchies
[i
]; i
++) {
1834 char *fullpath
= must_make_path(hierarchies
[i
]->fullcgpath
,
1835 "cgroup.procs", NULL
);
1836 if (lxc_write_to_file(fullpath
, pidstr
, len
, false) != 0) {
1837 SYSERROR("Failed to enter %s", fullpath
);
1847 static int chowmod(char *path
, uid_t chown_uid
, gid_t chown_gid
,
1852 ret
= chown(path
, chown_uid
, chown_gid
);
1854 WARN("%s - Failed to chown(%s, %d, %d)", strerror(errno
), path
,
1855 (int)chown_uid
, (int)chown_gid
);
1859 ret
= chmod(path
, chmod_mode
);
1861 WARN("%s - Failed to chmod(%s, %d)", strerror(errno
), path
,
1869 /* chgrp the container cgroups to container group. We leave
1870 * the container owner as cgroup owner. So we must make the
1871 * directories 775 so that the container can create sub-cgroups.
1873 * Also chown the tasks and cgroup.procs files. Those may not
1874 * exist depending on kernel version.
1876 static int chown_cgroup_wrapper(void *data
)
1880 struct generic_userns_exec_data
*arg
= data
;
1881 uid_t nsuid
= (arg
->conf
->root_nsuid_map
!= NULL
) ? 0 : arg
->conf
->init_uid
;
1882 gid_t nsgid
= (arg
->conf
->root_nsgid_map
!= NULL
) ? 0 : arg
->conf
->init_gid
;
1884 ret
= setresgid(nsgid
, nsgid
, nsgid
);
1886 SYSERROR("Failed to setresgid(%d, %d, %d)",
1887 (int)nsgid
, (int)nsgid
, (int)nsgid
);
1891 ret
= setresuid(nsuid
, nsuid
, nsuid
);
1893 SYSERROR("Failed to setresuid(%d, %d, %d)",
1894 (int)nsuid
, (int)nsuid
, (int)nsuid
);
1898 ret
= setgroups(0, NULL
);
1899 if (ret
< 0 && errno
!= EPERM
) {
1900 SYSERROR("Failed to setgroups(0, NULL)");
1904 destuid
= get_ns_uid(arg
->origuid
);
1906 for (i
= 0; hierarchies
[i
]; i
++) {
1908 char *path
= hierarchies
[i
]->fullcgpath
;
1910 ret
= chowmod(path
, destuid
, nsgid
, 0775);
1914 /* Failures to chown() these are inconvenient but not
1915 * detrimental We leave these owned by the container launcher,
1916 * so that container root can write to the files to attach. We
1917 * chmod() them 664 so that container systemd can write to the
1918 * files (which systemd in wily insists on doing).
1921 if (hierarchies
[i
]->version
== CGROUP_SUPER_MAGIC
) {
1922 fullpath
= must_make_path(path
, "tasks", NULL
);
1923 (void)chowmod(fullpath
, destuid
, nsgid
, 0664);
1927 fullpath
= must_make_path(path
, "cgroup.procs", NULL
);
1928 (void)chowmod(fullpath
, destuid
, 0, 0664);
1931 if (hierarchies
[i
]->version
!= CGROUP2_SUPER_MAGIC
)
1934 fullpath
= must_make_path(path
, "cgroup.subtree_control", NULL
);
1935 (void)chowmod(fullpath
, destuid
, nsgid
, 0664);
1938 fullpath
= must_make_path(path
, "cgroup.threads", NULL
);
1939 (void)chowmod(fullpath
, destuid
, nsgid
, 0664);
1946 static bool cgfsng_chown(void *hdata
, struct lxc_conf
*conf
)
1948 struct cgfsng_handler_data
*d
= hdata
;
1949 struct generic_userns_exec_data wrap
;
1954 if (lxc_list_empty(&conf
->id_map
))
1957 wrap
.origuid
= geteuid();
1962 if (userns_exec_1(conf
, chown_cgroup_wrapper
, &wrap
,
1963 "chown_cgroup_wrapper") < 0) {
1964 ERROR("Error requesting cgroup chown in new namespace");
1972 * We've safe-mounted a tmpfs as parent, so we don't need to protect against
1973 * symlinks any more - just use mount
1976 /* mount cgroup-full if requested */
1977 static int mount_cgroup_full(int type
, struct hierarchy
*h
, char *dest
,
1978 char *container_cgroup
)
1980 if (type
< LXC_AUTO_CGROUP_FULL_RO
|| type
> LXC_AUTO_CGROUP_FULL_MIXED
)
1982 if (mount(h
->mountpoint
, dest
, "cgroup", MS_BIND
, NULL
) < 0) {
1983 SYSERROR("Error bind-mounting %s cgroup onto %s", h
->mountpoint
,
1987 if (type
!= LXC_AUTO_CGROUP_FULL_RW
) {
1988 unsigned long flags
= MS_BIND
| MS_NOSUID
| MS_NOEXEC
| MS_NODEV
|
1989 MS_REMOUNT
| MS_RDONLY
;
1990 if (mount(NULL
, dest
, "cgroup", flags
, NULL
) < 0) {
1991 SYSERROR("Error remounting %s readonly", dest
);
1996 INFO("Bind mounted %s onto %s", h
->mountpoint
, dest
);
1997 if (type
!= LXC_AUTO_CGROUP_FULL_MIXED
)
2000 /* mount just the container path rw */
2001 char *source
= must_make_path(h
->mountpoint
, h
->base_cgroup
, container_cgroup
, NULL
);
2002 char *rwpath
= must_make_path(dest
, h
->base_cgroup
, container_cgroup
, NULL
);
2003 if (mount(source
, rwpath
, "cgroup", MS_BIND
, NULL
) < 0)
2004 WARN("Failed to mount %s read-write: %s", rwpath
,
2006 INFO("Made %s read-write", rwpath
);
2012 /* cgroup-full:* is done, no need to create subdirs */
2013 static bool cg_mount_needs_subdirs(int type
)
2015 if (type
>= LXC_AUTO_CGROUP_FULL_RO
)
2021 /* After $rootfs/sys/fs/container/controller/the/cg/path has been created,
2022 * remount controller ro if needed and bindmount the cgroupfs onto
2023 * controll/the/cg/path.
2025 static int do_secondstage_mounts_if_needed(int type
, struct hierarchy
*h
,
2026 char *controllerpath
, char *cgpath
,
2027 const char *container_cgroup
)
2029 int ret
, remount_flags
;
2031 int flags
= MS_BIND
;
2033 if (type
== LXC_AUTO_CGROUP_RO
|| type
== LXC_AUTO_CGROUP_MIXED
) {
2034 ret
= mount(controllerpath
, controllerpath
, "cgroup", MS_BIND
, NULL
);
2036 SYSERROR("Failed to bind mount \"%s\" onto \"%s\"",
2037 controllerpath
, controllerpath
);
2041 remount_flags
= add_required_remount_flags(controllerpath
,
2043 flags
| MS_REMOUNT
);
2044 ret
= mount(controllerpath
, controllerpath
, "cgroup",
2045 MS_REMOUNT
| MS_BIND
| MS_RDONLY
, NULL
);
2047 SYSERROR("Failed to remount \"%s\" ro", controllerpath
);
2051 INFO("Remounted %s read-only", controllerpath
);
2054 sourcepath
= must_make_path(h
->mountpoint
, h
->base_cgroup
,
2055 container_cgroup
, NULL
);
2056 if (type
== LXC_AUTO_CGROUP_RO
)
2059 ret
= mount(sourcepath
, cgpath
, "cgroup", flags
, NULL
);
2061 SYSERROR("Failed to mount \"%s\" onto \"%s\"", h
->controllers
[0], cgpath
);
2065 INFO("Mounted \"%s\" onto \"%s\"", h
->controllers
[0], cgpath
);
2067 if (flags
& MS_RDONLY
) {
2068 remount_flags
= add_required_remount_flags(sourcepath
, cgpath
,
2069 flags
| MS_REMOUNT
);
2070 ret
= mount(sourcepath
, cgpath
, "cgroup", remount_flags
, NULL
);
2072 SYSERROR("Failed to remount \"%s\" ro", cgpath
);
2076 INFO("Remounted %s read-only", cgpath
);
2080 INFO("Completed second stage cgroup automounts for \"%s\"", cgpath
);
2084 static int cg_mount_in_cgroup_namespace(int type
, struct hierarchy
*h
,
2085 const char *controllerpath
)
2088 char *controllers
= NULL
;
2089 char *fstype
= "cgroup2";
2090 unsigned long flags
= 0;
2095 flags
|= MS_RELATIME
;
2097 if (type
== LXC_AUTO_CGROUP_RO
|| type
== LXC_AUTO_CGROUP_FULL_RO
)
2100 if (h
->version
!= CGROUP2_SUPER_MAGIC
) {
2101 controllers
= lxc_string_join(",", (const char **)h
->controllers
, false);
2107 ret
= mount("cgroup", controllerpath
, fstype
, flags
, controllers
);
2110 SYSERROR("Failed to mount %s with cgroup filesystem type %s", controllerpath
, fstype
);
2114 DEBUG("Mounted %s with cgroup filesystem type %s", controllerpath
, fstype
);
2118 static bool cgfsng_mount(void *hdata
, const char *root
, int type
)
2121 char *tmpfspath
= NULL
;
2122 bool retval
= false;
2123 struct lxc_handler
*handler
= hdata
;
2124 struct cgfsng_handler_data
*d
= handler
->cgroup_data
;
2125 bool has_cgns
= false, wants_force_mount
= false;
2127 if ((type
& LXC_AUTO_CGROUP_MASK
) == 0)
2130 if (type
& LXC_AUTO_CGROUP_FORCE
) {
2131 type
&= ~LXC_AUTO_CGROUP_FORCE
;
2132 wants_force_mount
= true;
2135 if (!wants_force_mount
){
2136 if (!lxc_list_empty(&handler
->conf
->keepcaps
))
2137 wants_force_mount
= !in_caplist(CAP_SYS_ADMIN
, &handler
->conf
->keepcaps
);
2139 wants_force_mount
= in_caplist(CAP_SYS_ADMIN
, &handler
->conf
->caps
);
2142 has_cgns
= cgns_supported();
2143 if (has_cgns
&& !wants_force_mount
)
2146 if (type
== LXC_AUTO_CGROUP_NOSPEC
)
2147 type
= LXC_AUTO_CGROUP_MIXED
;
2148 else if (type
== LXC_AUTO_CGROUP_FULL_NOSPEC
)
2149 type
= LXC_AUTO_CGROUP_FULL_MIXED
;
2152 tmpfspath
= must_make_path(root
, "/sys/fs/cgroup", NULL
);
2153 ret
= safe_mount("cgroup_root", tmpfspath
, "tmpfs",
2154 MS_NOSUID
| MS_NODEV
| MS_NOEXEC
| MS_RELATIME
,
2155 "size=10240k,mode=755", root
);
2159 for (i
= 0; hierarchies
[i
]; i
++) {
2160 char *controllerpath
, *path2
;
2161 struct hierarchy
*h
= hierarchies
[i
];
2162 char *controller
= strrchr(h
->mountpoint
, '/');
2167 controllerpath
= must_make_path(tmpfspath
, controller
, NULL
);
2168 if (dir_exists(controllerpath
)) {
2169 free(controllerpath
);
2172 ret
= mkdir(controllerpath
, 0755);
2174 SYSERROR("Error creating cgroup path: %s", controllerpath
);
2175 free(controllerpath
);
2179 if (has_cgns
&& wants_force_mount
) {
2180 /* If cgroup namespaces are supported but the container
2181 * will not have CAP_SYS_ADMIN after it has started we
2182 * need to mount the cgroups manually.
2184 ret
= cg_mount_in_cgroup_namespace(type
, h
, controllerpath
);
2185 free(controllerpath
);
2192 ret
= mount_cgroup_full(type
, h
, controllerpath
, d
->container_cgroup
);
2194 free(controllerpath
);
2198 if (!cg_mount_needs_subdirs(type
)) {
2199 free(controllerpath
);
2203 path2
= must_make_path(controllerpath
, h
->base_cgroup
,
2204 d
->container_cgroup
, NULL
);
2205 ret
= mkdir_p(path2
, 0755);
2207 free(controllerpath
);
2212 ret
= do_secondstage_mounts_if_needed(
2213 type
, h
, controllerpath
, path2
, d
->container_cgroup
);
2214 free(controllerpath
);
2226 static int recursive_count_nrtasks(char *dirname
)
2228 struct dirent
*direntp
;
2233 dir
= opendir(dirname
);
2237 while ((direntp
= readdir(dir
))) {
2243 if (!strcmp(direntp
->d_name
, ".") ||
2244 !strcmp(direntp
->d_name
, ".."))
2247 path
= must_make_path(dirname
, direntp
->d_name
, NULL
);
2249 if (lstat(path
, &mystat
))
2252 if (!S_ISDIR(mystat
.st_mode
))
2255 count
+= recursive_count_nrtasks(path
);
2260 path
= must_make_path(dirname
, "cgroup.procs", NULL
);
2261 ret
= lxc_count_file_lines(path
);
2266 (void) closedir(dir
);
2271 static int cgfsng_nrtasks(void *hdata
) {
2272 struct cgfsng_handler_data
*d
= hdata
;
2276 if (!d
|| !d
->container_cgroup
|| !hierarchies
)
2279 path
= must_make_path(hierarchies
[0]->fullcgpath
, NULL
);
2280 count
= recursive_count_nrtasks(path
);
2285 /* Only root needs to escape to the cgroup of its init */
2286 static bool cgfsng_escape()
2293 for (i
= 0; hierarchies
[i
]; i
++) {
2294 char *fullpath
= must_make_path(hierarchies
[i
]->mountpoint
,
2295 hierarchies
[i
]->base_cgroup
,
2296 "cgroup.procs", NULL
);
2297 if (lxc_write_to_file(fullpath
, "0", 2, false) != 0) {
2298 SYSERROR("Failed to escape to %s", fullpath
);
2308 static int cgfsng_num_hierarchies(void)
2312 for (i
= 0; hierarchies
[i
]; i
++)
2318 static bool cgfsng_get_hierarchies(int n
, char ***out
)
2322 /* sanity check n */
2323 for (i
= 0; i
< n
; i
++)
2324 if (!hierarchies
[i
])
2327 *out
= hierarchies
[i
]->controllers
;
2332 #define THAWED "THAWED"
2333 #define THAWED_LEN (strlen(THAWED))
2335 /* TODO: If the unified cgroup hierarchy grows a freezer controller this needs
2338 static bool cgfsng_unfreeze(void *hdata
)
2342 struct hierarchy
*h
;
2344 h
= get_hierarchy("freezer");
2348 fullpath
= must_make_path(h
->fullcgpath
, "freezer.state", NULL
);
2349 ret
= lxc_write_to_file(fullpath
, THAWED
, THAWED_LEN
, false);
2357 static const char *cgfsng_get_cgroup(void *hdata
, const char *subsystem
)
2359 struct hierarchy
*h
;
2361 h
= get_hierarchy(subsystem
);
2365 return h
->fullcgpath
? h
->fullcgpath
+ strlen(h
->mountpoint
) : NULL
;
2369 * Given a cgroup path returned from lxc_cmd_get_cgroup_path, build a
2370 * full path, which must be freed by the caller.
2372 static char *build_full_cgpath_from_monitorpath(struct hierarchy
*h
,
2374 const char *filename
)
2376 return must_make_path(h
->mountpoint
, inpath
, filename
, NULL
);
2379 /* Technically, we're always at a delegation boundary here. (This is especially
2380 * true when cgroup namespaces are available.) The reasoning is that in order
2381 * for us to have been able to start a container in the first place the root
2382 * cgroup must have been a leaf node. Now, either the container's init system
2383 * has populated the cgroup and kept it as a leaf node or it has created
2384 * subtrees. In the former case we will simply attach to the leaf node we
2385 * created when we started the container in the latter case we create our own
2386 * cgroup for the attaching process.
2388 static int __cg_unified_attach(const struct hierarchy
*h
, const char *name
,
2389 const char *lxcpath
, const char *pidstr
,
2390 size_t pidstr_len
, const char *controller
)
2394 int fret
= -1, idx
= 0;
2395 char *base_path
= NULL
, *container_cgroup
= NULL
, *full_path
= NULL
;
2397 container_cgroup
= lxc_cmd_get_cgroup_path(name
, lxcpath
, controller
);
2399 if (!container_cgroup
)
2402 base_path
= must_make_path(h
->mountpoint
, container_cgroup
, NULL
);
2403 full_path
= must_make_path(base_path
, "cgroup.procs", NULL
);
2404 /* cgroup is populated */
2405 ret
= lxc_write_to_file(full_path
, pidstr
, pidstr_len
, false);
2406 if (ret
< 0 && errno
!= EBUSY
)
2414 len
= strlen(base_path
) + sizeof("/lxc-1000") - 1 +
2415 sizeof("/cgroup-procs") - 1;
2416 full_path
= must_alloc(len
+ 1);
2419 ret
= snprintf(full_path
, len
+ 1, "%s/lxc-%d",
2422 ret
= snprintf(full_path
, len
+ 1, "%s/lxc", base_path
);
2423 if (ret
< 0 || (size_t)ret
>= len
+ 1)
2426 ret
= mkdir_p(full_path
, 0755);
2427 if (ret
< 0 && errno
!= EEXIST
)
2430 strcat(full_path
, "/cgroup.procs");
2431 ret
= lxc_write_to_file(full_path
, pidstr
, len
, false);
2435 /* this is a non-leaf node */
2439 } while (++idx
> 0 && idx
< 1000);
2447 free(container_cgroup
);
2453 static bool cgfsng_attach(const char *name
, const char *lxcpath
, pid_t pid
)
2458 len
= snprintf(pidstr
, 25, "%d", pid
);
2459 if (len
< 0 || len
> 25)
2462 for (i
= 0; hierarchies
[i
]; i
++) {
2464 char *fullpath
= NULL
;
2465 struct hierarchy
*h
= hierarchies
[i
];
2467 if (h
->version
== CGROUP2_SUPER_MAGIC
) {
2468 ret
= __cg_unified_attach(h
, name
, lxcpath
, pidstr
, len
,
2476 path
= lxc_cmd_get_cgroup_path(name
, lxcpath
, h
->controllers
[0]);
2481 fullpath
= build_full_cgpath_from_monitorpath(h
, path
, "cgroup.procs");
2482 ret
= lxc_write_to_file(fullpath
, pidstr
, len
, false);
2484 SYSERROR("Failed to attach %d to %s", (int)pid
, fullpath
);
2495 * Called externally (i.e. from 'lxc-cgroup') to query cgroup limits.
2496 * Here we don't have a cgroup_data set up, so we ask the running
2497 * container through the commands API for the cgroup path
2499 static int cgfsng_get(const char *filename
, char *value
, size_t len
,
2500 const char *name
, const char *lxcpath
)
2503 size_t controller_len
;
2504 char *controller
, *p
, *path
;
2505 struct hierarchy
*h
;
2507 controller_len
= strlen(filename
);
2508 controller
= alloca(controller_len
+ 1);
2509 strcpy(controller
, filename
);
2510 p
= strchr(controller
, '.');
2514 path
= lxc_cmd_get_cgroup_path(name
, lxcpath
, controller
);
2519 h
= get_hierarchy(controller
);
2523 fullpath
= build_full_cgpath_from_monitorpath(h
, path
, filename
);
2524 ret
= lxc_read_from_file(fullpath
, value
, len
);
2533 * Called externally (i.e. from 'lxc-cgroup') to set new cgroup limits.
2534 * Here we don't have a cgroup_data set up, so we ask the running
2535 * container through the commands API for the cgroup path
2537 static int cgfsng_set(const char *filename
, const char *value
, const char *name
,
2538 const char *lxcpath
)
2541 size_t controller_len
;
2542 char *controller
, *p
, *path
;
2543 struct hierarchy
*h
;
2545 controller_len
= strlen(filename
);
2546 controller
= alloca(controller_len
+ 1);
2547 strcpy(controller
, filename
);
2548 p
= strchr(controller
, '.');
2552 path
= lxc_cmd_get_cgroup_path(name
, lxcpath
, controller
);
2557 h
= get_hierarchy(controller
);
2561 fullpath
= build_full_cgpath_from_monitorpath(h
, path
, filename
);
2562 ret
= lxc_write_to_file(fullpath
, value
, strlen(value
), false);
2571 * take devices cgroup line
2573 * and convert it to a valid
2574 * type major:minor mode
2575 * line. Return <0 on error. Dest is a preallocated buffer
2576 * long enough to hold the output.
2578 static int convert_devpath(const char *invalue
, char *dest
)
2581 char *p
, *path
, type
;
2583 unsigned long minor
, major
;
2587 path
= must_copy_string(invalue
);
2590 * read path followed by mode; ignore any trailing text.
2591 * A ' # comment' would be legal. Technically other text
2592 * is not legal, we could check for that if we cared to
2594 for (n_parts
= 1, p
= path
; *p
&& n_parts
< 3; p
++) {
2612 ret
= stat(path
, &sb
);
2616 mode_t m
= sb
.st_mode
& S_IFMT
;
2625 ERROR("Unsupported device type %i for %s", m
, path
);
2630 major
= MAJOR(sb
.st_rdev
);
2631 minor
= MINOR(sb
.st_rdev
);
2632 ret
= snprintf(dest
, 50, "%c %lu:%lu %s", type
, major
, minor
, mode
);
2633 if (ret
< 0 || ret
>= 50) {
2634 ERROR("Error on configuration value \"%c %lu:%lu %s\" (max 50 "
2635 "chars)", type
, major
, minor
, mode
);
2636 ret
= -ENAMETOOLONG
;
2647 * Called from setup_limits - here we have the container's cgroup_data because
2648 * we created the cgroups
2650 static int cg_legacy_set_data(const char *filename
, const char *value
,
2651 struct cgfsng_handler_data
*d
)
2655 /* "b|c <2^64-1>:<2^64-1> r|w|m" = 47 chars max */
2656 char converted_value
[50];
2657 struct hierarchy
*h
;
2659 char *controller
= NULL
;
2661 len
= strlen(filename
);
2662 controller
= alloca(len
+ 1);
2663 strcpy(controller
, filename
);
2664 p
= strchr(controller
, '.');
2668 if (strcmp("devices.allow", filename
) == 0 && value
[0] == '/') {
2669 ret
= convert_devpath(value
, converted_value
);
2672 value
= converted_value
;
2675 h
= get_hierarchy(controller
);
2677 ERROR("Failed to setup limits for the \"%s\" controller. "
2678 "The controller seems to be unused by \"cgfsng\" cgroup "
2679 "driver or not enabled on the cgroup hierarchy",
2685 fullpath
= must_make_path(h
->fullcgpath
, filename
, NULL
);
2686 ret
= lxc_write_to_file(fullpath
, value
, strlen(value
), false);
2691 static bool __cg_legacy_setup_limits(void *hdata
,
2692 struct lxc_list
*cgroup_settings
,
2695 struct cgfsng_handler_data
*d
= hdata
;
2696 struct lxc_list
*iterator
, *sorted_cgroup_settings
, *next
;
2697 struct lxc_cgroup
*cg
;
2700 if (lxc_list_empty(cgroup_settings
))
2703 sorted_cgroup_settings
= sort_cgroup_settings(cgroup_settings
);
2704 if (!sorted_cgroup_settings
)
2707 lxc_list_for_each(iterator
, sorted_cgroup_settings
) {
2708 cg
= iterator
->elem
;
2710 if (do_devices
== !strncmp("devices", cg
->subsystem
, 7)) {
2711 if (cg_legacy_set_data(cg
->subsystem
, cg
->value
, d
)) {
2712 if (do_devices
&& (errno
== EACCES
|| errno
== EPERM
)) {
2713 WARN("Error setting %s to %s for %s",
2714 cg
->subsystem
, cg
->value
, d
->name
);
2717 SYSERROR("Error setting %s to %s for %s",
2718 cg
->subsystem
, cg
->value
, d
->name
);
2721 DEBUG("cgroup '%s' set to '%s'", cg
->subsystem
, cg
->value
);
2726 INFO("Limits for the legacy cgroup hierarchies have been setup");
2728 lxc_list_for_each_safe(iterator
, sorted_cgroup_settings
, next
) {
2729 lxc_list_del(iterator
);
2732 free(sorted_cgroup_settings
);
2736 static bool __cg_unified_setup_limits(void *hdata
,
2737 struct lxc_list
*cgroup_settings
)
2739 struct lxc_list
*iterator
;
2740 struct hierarchy
*h
= unified
;
2742 if (lxc_list_empty(cgroup_settings
))
2748 lxc_list_for_each(iterator
, cgroup_settings
) {
2751 struct lxc_cgroup
*cg
= iterator
->elem
;
2753 fullpath
= must_make_path(h
->fullcgpath
, cg
->subsystem
, NULL
);
2754 ret
= lxc_write_to_file(fullpath
, cg
->value
, strlen(cg
->value
), false);
2757 SYSERROR("Failed to set \"%s\" to \"%s\"", cg
->subsystem
, cg
->value
);
2760 TRACE("Set \"%s\" to \"%s\"", cg
->subsystem
, cg
->value
);
2763 INFO("Limits for the unified cgroup hierarchy have been setup");
2767 static bool cgfsng_setup_limits(void *hdata
, struct lxc_conf
*conf
,
2772 bret
= __cg_legacy_setup_limits(hdata
, &conf
->cgroup
, do_devices
);
2776 return __cg_unified_setup_limits(hdata
, &conf
->cgroup2
);
2779 static struct cgroup_ops cgfsng_ops
= {
2780 .init
= cgfsng_init
,
2781 .destroy
= cgfsng_destroy
,
2782 .create
= cgfsng_create
,
2783 .enter
= cgfsng_enter
,
2784 .escape
= cgfsng_escape
,
2785 .num_hierarchies
= cgfsng_num_hierarchies
,
2786 .get_hierarchies
= cgfsng_get_hierarchies
,
2787 .get_cgroup
= cgfsng_get_cgroup
,
2790 .unfreeze
= cgfsng_unfreeze
,
2791 .setup_limits
= cgfsng_setup_limits
,
2792 .name
= "cgroupfs-ng",
2793 .attach
= cgfsng_attach
,
2794 .chown
= cgfsng_chown
,
2795 .mount_cgroup
= cgfsng_mount
,
2796 .nrtasks
= cgfsng_nrtasks
,
2800 .create_legacy
= NULL
,