]>
git.proxmox.com Git - pve-common.git/blob - src/PVE/CGroup.pm
3 # This package should deal with figuring out the right cgroup path for a
4 # container (via the command socket), reading and writing cgroup values, and
5 # handling cgroup v1 & v2 differences.
7 # Note that the long term plan is to have resource manage functions instead of
8 # dealing with cgroup files on the outside.
25 # We don't want to do a command socket round trip for every cgroup read/write,
26 # so any cgroup function needs to have the container's path cached, so this
27 # package has to be instantiated.
29 # LXC keeps separate paths by controller (although they're normally all the
30 # same, in our # case anyway), so we cache them by controller as well.
32 my ($class, $vmid) = @_;
34 my $self = { vmid
=> $vmid };
36 return bless $self, $class;
39 # Get the v1 controller list.
41 # Returns a set (hash mapping names to `1`) of cgroupv1 controllers, and an
42 # optional boolean whether a unified (cgroupv2) hierarchy exists.
43 my sub get_v1_controllers
{
46 my $data = PVE
::Tools
::file_get_contents
('/proc/self/cgroup');
47 while ($data =~ /^\d+:([^:\n]*):.*$/gm) {
50 $v1->{$_} = 1 foreach split(/,/, $type);
55 return wantarray ?
($v1, $v2) : $v1;
58 # Get the set v2 controller list from the `cgroup.controllers` file.
59 my sub get_v2_controllers
{
60 my $v2 = eval { file_get_contents
('/sys/fs/cgroup/cgroup.controllers') }
61 || eval { file_get_contents
('/sys/fs/cgroup/unified/cgroup.controllers') };
62 return undef if !defined $v2;
64 # It's a simple space separated list:
65 return { map { $_ => 1 } split(/\s+/, $v2) };
68 my $CGROUP_CONTROLLERS = undef;
69 # Get a list of controllers enabled in each cgroup subsystem.
71 # This is a more complete version of `PVE::LXC::get_cgroup_subsystems`.
73 # Returns 2 sets (hashes mapping controller names to `1`), one for each cgroup
75 sub get_cgroup_controllers
() {
76 if (!defined($CGROUP_CONTROLLERS)) {
77 my ($v1, undef) = get_v1_controllers
();
78 my $v2 = get_v2_controllers
();
80 $CGROUP_CONTROLLERS = [$v1, $v2];
83 return $CGROUP_CONTROLLERS->@*;
86 my $CGROUP_MODE = undef;
87 # Figure out which cgroup mode we're operating under:
89 # For this we check the file system type of `/sys/fs/cgroup` as it may well be possible that some
90 # additional cgroupv1 mount points have been created by tools such as `systemd-nspawn`, or
93 # Returns 1 for what we consider the hybrid layout, 2 for what we consider the unified layout.
95 # NOTE: To fully support a hybrid layout it is better to use functions like
96 # `cpuset_controller_path` and not rely on this value for anything involving paths.
98 # This is a function, not a method!
100 if (!defined($CGROUP_MODE)) {
101 my $mounts = PVE
::ProcFSTools
::parse_proc_mounts
();
102 for my $entry (@$mounts) {
103 my ($what, $dir, $fstype, $opts) = @$entry;
104 if ($dir eq '/sys/fs/cgroup') {
105 if ($fstype eq 'cgroup2') {
116 die "unknown cgroup mode\n" if !defined($CGROUP_MODE);
120 my $CGROUPV2_PATH = undef;
121 sub cgroupv2_base_path
() {
122 if (!defined($CGROUPV2_PATH)) {
123 if (cgroup_mode
() == 2) {
124 $CGROUPV2_PATH = '/sys/fs/cgroup';
126 $CGROUPV2_PATH = '/sys/fs/cgroup/unified';
129 return $CGROUPV2_PATH;
132 # Find a cgroup controller and return its path and version.
134 # LXC initializes the unified hierarchy first, so if a controller is
135 # available via both we favor cgroupv2 here as well.
137 # Returns nothing if the controller is not available.
139 sub find_cgroup_controller
($) {
140 my ($controller) = @_;
142 my ($v1, $v2) = get_cgroup_controllers
();
144 if (!defined($controller) || $v2->{$controller}) {
145 my $path = cgroupv2_base_path
();
146 return wantarray ?
($path, 2) : $path;
149 if (defined($controller) && $v1->{$controller}) {
150 my $path = "/sys/fs/cgroup/$controller";
151 return wantarray ?
($path, 1) : $path;
157 my $CG_PATH_CPUSET = undef;
158 my $CG_VER_CPUSET = undef;
159 # Find the cpuset cgroup controller.
161 # This is a function, not a method!
162 sub cpuset_controller_path
() {
163 if (!defined($CG_PATH_CPUSET)) {
164 ($CG_PATH_CPUSET, $CG_VER_CPUSET) = find_cgroup_controller
('cpuset')
165 or die "failed to find cpuset controller\n";
168 return wantarray ?
($CG_PATH_CPUSET, $CG_VER_CPUSET) : $CG_PATH_CPUSET;
171 # Get a subdirectory (without the cgroup mount point) for a controller.
173 my ($self, $controller, $limiting) = @_;
175 die "implement in subclass";
178 # Get path and version for a controller.
180 # `$controller` may be `undef`, see get_subdir above for details.
182 # Returns either just the path, or the path and cgroup version as a tuple.
184 my ($self, $controller, $limiting) = @_;
185 # Find the controller before querying the lxc monitor via a socket:
186 my ($cgpath, $ver) = find_cgroup_controller
($controller)
189 my $path = $self->get_subdir($controller, $limiting)
192 $path = "$cgpath/$path";
193 return wantarray ?
($path, $ver) : $path;
196 # Convenience method to get the path info if the first existing controller.
198 # Returns the same as `get_path`.
200 my ($self, $limiting, @controllers) = @_;
203 for my $c (@controllers) {
204 ($path, $ver) = $self->get_path($c, $limiting);
205 last if defined $path;
207 return wantarray ?
($path, $ver) : $path;
210 # Parse a 'Nested keyed' file:
212 # See kernel documentation `admin-guide/cgroup-v2.rst` 4.1.
213 my sub parse_nested_keyed_file
($) {
216 foreach my $line (split(/\n/, $data)) {
217 my ($key, @values) = split(/\s+/, $line);
219 my $d = ($res->{$key} = {});
221 foreach my $value (@values) {
222 if (my ($key, $value) = ($value =~ /^([^=]+)=(.*)$/)) {
225 warn "bad key=value pair in nested keyed file\n";
232 # Parse a 'Flat keyed' file:
234 # See kernel documentation `admin-guide/cgroup-v2.rst` 4.1.
235 my sub parse_flat_keyed_file
($) {
238 foreach my $line (split(/\n/, $data)) {
239 if (my ($key, $value) = ($line =~ /^(\S+)\s+(.*)$/)) {
240 $res->{$key} = $value;
242 warn "bad 'key value' pair in flat keyed file\n";
248 # Parse out 'diskread' and 'diskwrite' values from I/O stats for this container.
257 # With cgroupv1 we have a 'blkio' controller, with cgroupv2 it's just 'io':
258 my ($path, $ver) = $self->get_any_path(1, 'io', 'blkio');
259 if (!defined($path)) {
260 # container not running
262 } elsif ($ver == 2) {
263 # cgroupv2 environment, io controller enabled
264 my $io_stat = file_get_contents
("$path/io.stat");
266 my $data = parse_nested_keyed_file
($io_stat);
267 foreach my $dev (keys %$data) {
268 my $dev = $data->{$dev};
269 if (my $b = $dev->{rbytes
}) {
270 $res->{diskread
} += $b;
272 if (my $b = $dev->{wbytes
}) {
273 $res->{diskwrite
} += $b;
278 } elsif ($ver == 1) {
279 # cgroupv1 environment:
280 my $io = file_get_contents
("$path/blkio.throttle.io_service_bytes_recursive");
281 foreach my $line (split(/\n/, $io)) {
282 if (my ($type, $bytes) = ($line =~ /^\d+:\d+\s+(Read|Write)\s+(\d+)$/)) {
283 $res->{diskread
} += $bytes if $type eq 'Read';
284 $res->{diskwrite
} += $bytes if $type eq 'Write';
290 die "bad cgroup version: $ver\n";
293 # container not running
297 # Read utime and stime for this container from the cpuacct cgroup.
298 # Values are in milliseconds!
307 my ($path, $ver) = $self->get_any_path(1, 'cpuacct', 'cpu');
308 if (!defined($path)) {
309 # container not running
311 } elsif ($ver == 2) {
312 my $data = eval { file_get_contents
("$path/cpu.stat") };
314 # or no io controller available:
315 return undef if !defined($data);
317 $data = parse_flat_keyed_file
($data);
318 $res->{utime} = int($data->{user_usec
} / 1000);
319 $res->{stime
} = int($data->{system_usec
} / 1000);
320 } elsif ($ver == 1) {
321 # cgroupv1 environment:
322 my $clock_ticks = POSIX
::sysconf
(&POSIX
::_SC_CLK_TCK
);
323 my $clk_to_usec = 1000 / $clock_ticks;
325 my $data = parse_flat_keyed_file
(file_get_contents
("$path/cpuacct.stat"));
326 $res->{utime} = int($data->{user
} * $clk_to_usec);
327 $res->{stime
} = int($data->{system} * $clk_to_usec);
329 die "bad cgroup version: $ver\n";
335 # Parse some memory data from `memory.stat`
336 sub get_memory_stat
{
344 my ($path, $ver) = $self->get_path('memory', 1);
345 if (!defined($path)) {
346 # container most likely isn't running
348 } elsif ($ver == 2) {
349 my $mem = file_get_contents
("$path/memory.current");
350 my $swap = file_get_contents
("$path/memory.swap.current");
351 my $stat = parse_flat_keyed_file
(file_get_contents
("$path/memory.stat"));
355 $res->{mem
} = $mem - $stat->{file
};
356 $res->{swap
} = $swap;
357 } elsif ($ver == 1) {
358 # cgroupv1 environment:
359 my $stat = parse_flat_keyed_file
(file_get_contents
("$path/memory.stat"));
360 my $mem = file_get_contents
("$path/memory.usage_in_bytes");
361 my $memsw = file_get_contents
("$path/memory.memsw.usage_in_bytes");
362 chomp ($mem, $memsw);
364 $res->{mem
} = $mem - $stat->{total_cache
};
365 $res->{swap
} = $memsw - $mem;
367 die "bad cgroup version: $ver\n";
373 sub get_pressure_stat
{
378 some
=> { avg10
=> 0, avg60
=> 0, avg300
=> 0 }
381 some
=> { avg10
=> 0, avg60
=> 0, avg300
=> 0 },
382 full
=> { avg10
=> 0, avg60
=> 0, avg300
=> 0 }
385 some
=> { avg10
=> 0, avg60
=> 0, avg300
=> 0 },
386 full
=> { avg10
=> 0, avg60
=> 0, avg300
=> 0 }
390 my ($path, $version) = $self->get_path(undef, 1);
391 if (!defined($path)) {
392 return $res; # container or VM most likely isn't running, retrun zero stats
393 } elsif ($version == 1) {
394 return undef; # v1 controller does not provides pressure stat
395 } elsif ($version == 2) {
396 for my $type (qw(cpu memory io)) {
397 my $stats = PVE
::ProcFSTools
::parse_pressure
("$path/$type.pressure");
398 $res->{$type} = $stats if $stats;
401 die "bad cgroup version: $version\n";
407 # Change the memory limit for this container.
409 # Dies on error (including a not-running or currently-shutting-down guest).
410 sub change_memory_limit
{
411 my ($self, $mem_bytes, $swap_bytes, $mem_high_bytes) = @_;
413 my ($path, $ver) = $self->get_path('memory', 1);
414 if (!defined($path)) {
415 die "trying to change memory cgroup values: container not running\n";
416 } elsif ($ver == 2) {
417 PVE
::ProcFSTools
::write_proc_entry
("$path/memory.swap.max", $swap_bytes)
418 if defined($swap_bytes);
419 if (defined($mem_bytes)) {
420 # 'max' is the hard-limit (triggers OOM), while 'high' throttles & adds reclaim pressure
421 PVE
::ProcFSTools
::write_proc_entry
("$path/memory.high", $mem_high_bytes // 'max');
422 PVE
::ProcFSTools
::write_proc_entry
("$path/memory.max", $mem_bytes);
424 } elsif ($ver == 1) {
425 # With cgroupv1 we cannot control memory and swap limits separately.
426 # This also means that since the two values aren't independent, we need to handle
427 # growing and shrinking separately.
428 my $path_mem = "$path/memory.limit_in_bytes";
429 my $path_memsw = "$path/memory.memsw.limit_in_bytes";
431 my $old_mem_bytes = file_get_contents
($path_mem);
432 my $old_memsw_bytes = file_get_contents
($path_memsw);
433 chomp($old_mem_bytes, $old_memsw_bytes);
435 $mem_bytes //= $old_mem_bytes;
436 $swap_bytes //= $old_memsw_bytes - $old_mem_bytes;
437 my $memsw_bytes = $mem_bytes + $swap_bytes;
439 if ($memsw_bytes > $old_memsw_bytes) {
440 # Growing the limit means growing the combined limit first, then pulling the
442 PVE
::ProcFSTools
::write_proc_entry
($path_memsw, $memsw_bytes);
443 PVE
::ProcFSTools
::write_proc_entry
($path_mem, $mem_bytes);
445 # Shrinking means we first need to shrink the mem-only memsw cannot be
447 PVE
::ProcFSTools
::write_proc_entry
($path_mem, $mem_bytes);
448 PVE
::ProcFSTools
::write_proc_entry
($path_memsw, $memsw_bytes);
451 die "bad cgroup version: $ver\n";
454 # return a truth value
458 # Change the cpu quota for a container.
460 # Dies on error (including a not-running or currently-shutting-down guest).
461 sub change_cpu_quota
{
462 my ($self, $quota, $period) = @_;
464 die "quota without period not allowed\n" if !defined($period) && defined($quota);
466 my ($path, $ver) = $self->get_path('cpu', 1);
467 if (!defined($path)) {
468 die "trying to change cpu quota cgroup values: container not running\n";
469 } elsif ($ver == 2) {
470 # cgroupv2 environment, an undefined (unlimited) quota is defined as "max"
472 $quota //= 'max'; # unlimited
473 if (defined($quota)) {
474 PVE
::ProcFSTools
::write_proc_entry
("$path/cpu.max", "$quota $period");
476 # we're allowed to only write the quota:
477 PVE
::ProcFSTools
::write_proc_entry
("$path/cpu.max", 'max');
479 } elsif ($ver == 1) {
480 $quota //= -1; # default (unlimited)
481 $period //= 100_000; # default (100 ms)
482 PVE
::ProcFSTools
::write_proc_entry
("$path/cpu.cfs_period_us", $period);
483 PVE
::ProcFSTools
::write_proc_entry
("$path/cpu.cfs_quota_us", $quota);
485 die "bad cgroup version: $ver\n";
488 # return a truth value
492 # Clamp an integer to the supported range of CPU shares from the booted CGroup version
494 # Returns the default if called with an undefined value.
495 sub clamp_cpu_shares
{
498 my $is_cgroupv2 = cgroup_mode
() == 2;
500 return $is_cgroupv2 ?
100 : 1024 if !defined($shares);
503 $shares = 10000 if $shares >= 10000; # v1 can be higher, so clamp v2 there
505 $shares = 2 if $shares < 2; # v2 can be lower, so clamp v1 there
510 # Change the cpu "shares" for a container.
512 # In cgroupv1 we used a value in `[0..500000]` with a default of 1024.
514 # In cgroupv2 we do not have "shares", we have "weights" in the range
515 # of `[1..10000]` with a default of 100.
517 # Since the default values don't match when scaling linearly, we use the
518 # values we get as-is and simply error for values >10000 in cgroupv2.
520 # It is left to the user to figure this out for now.
522 # Dies on error (including a not-running or currently-shutting-down guest).
524 # NOTE: if you add a new param during 7.x you need to break older pve-container/qemu-server versions
525 # that previously passed a `$cgroupv1_default`, which got removed due to being ignored anyway.
526 # otherwise you risk that a old module bogusly passes some cgroup default as your new param.
527 sub change_cpu_shares
{
528 my ($self, $shares) = @_;
530 my ($path, $ver) = $self->get_path('cpu', 1);
531 if (!defined($path)) {
532 die "trying to change cpu shares/weight cgroup values: container not running\n";
533 } elsif ($ver == 2) {
534 # the cgroupv2 documentation defines the default to 100
536 die "cpu weight (shares) must be in range [1, 10000]\n" if $shares < 1 || $shares > 10000;
537 PVE
::ProcFSTools
::write_proc_entry
("$path/cpu.weight", $shares);
538 } elsif ($ver == 1) {
540 PVE
::ProcFSTools
::write_proc_entry
("$path/cpu.shares", $shares);
542 die "bad cgroup version: $ver\n";
545 # return a truth value
549 my sub v1_freeze_thaw
{
550 my ($self, $controller_path, $freeze) = @_;
551 my $path = $self->get_subdir('freezer', 1)
552 or die "trying to freeze container: container not running\n";
553 $path = "$controller_path/$path/freezer.state";
555 my $data = $freeze ?
'FROZEN' : 'THAWED';
556 PVE
::ProcFSTools
::write_proc_entry
($path, $data);
558 # Here we just poll the freezer.state once per second.
560 my $state = file_get_contents
($path);
562 last if $state eq $data;
566 my sub v2_freeze_thaw
{
567 my ($self, $controller_path, $freeze) = @_;
568 my $path = $self->get_subdir(undef, 1)
569 or die "trying to freeze container: container not running\n";
570 $path = "$controller_path/$path";
572 my $desired_state = $freeze ?
1 : 0;
574 # cgroupv2 supports poll events on cgroup.events which contains the frozen
576 my $fh = IO
::File-
>new("$path/cgroup.events", 'r')
577 or die "failed to open $path/cgroup.events file: $!\n";
578 my $select = IO
::Select-
>new();
581 PVE
::ProcFSTools
::write_proc_entry
("$path/cgroup.freeze", $desired_state);
587 $data = parse_flat_keyed_file
($data);
588 last if $data->{frozen
} == $desired_state;
589 my @handles = $select->has_exception();
592 or die "failed to rewind cgroup.events file: $!\n";
596 # Freeze or unfreeze a container.
598 # This will freeze the container at its outer (limiting) cgroup path. We use
599 # this instead of `lxc-freeze` as `lxc-freeze` from lxc4 will not be able to
600 # fetch the cgroup path from contaienrs still running on lxc3.
602 my ($self, $freeze) = @_;
604 my $controller_path = find_cgroup_controller
('freezer');
605 if (defined($controller_path)) {
606 return v1_freeze_thaw
($self, $controller_path, $freeze);
608 # cgroupv2 always has a freezer, there can be both cgv1 and cgv2
609 # freezers, but we'll prefer v1 when it's available as that's what lxc
611 return v2_freeze_thaw
($self, cgroupv2_base_path
(), $freeze);