src/PVE/CGroup.pm

   1 # cgroup handler
   2 #
   3 # This package should deal with figuring out the right cgroup path for a
   4 # container (via the command socket), reading and writing cgroup values, and
   5 # handling cgroup v1 & v2 differences.
   6 #
   7 # Note that the long term plan is to have resource manage functions instead of
   8 # dealing with cgroup files on the outside.
   9
  10 package PVE::CGroup;
  11
  12 use strict;
  13 use warnings;
  14
  15 use IO::File;
  16 use IO::Select;
  17 use POSIX qw();
  18
  19 use PVE::ProcFSTools;
  20 use PVE::Tools qw(
  21     file_get_contents
  22     file_read_firstline
  23 );
  24
  25 use PVE::LXC::Command;
  26
  27 # We don't want to do a command socket round trip for every cgroup read/write,
  28 # so any cgroup function needs to have the container's path cached, so this
  29 # package has to be instantiated.
  30 #
  31 # LXC keeps separate paths by controller (although they're normally all the
  32 # same, in our # case anyway), so we cache them by controller as well.
  33 sub new {
  34     my ($class, $vmid) = @_;
  35
  36     my $self = { vmid => $vmid };
  37
  38     return bless $self, $class;
  39 }
  40
  41 # Get the v1 controller list.
  42 #
  43 # Returns a set (hash mapping names to `1`) of cgroupv1 controllers, and an
  44 # optional boolean whether a unified (cgroupv2) hierarchy exists.
  45 #
  46 # Deprecated: Use `get_cgroup_controllers()` instead.
  47 sub get_v1_controllers {
  48     my $v1 = {};
  49     my $v2 = 0;
  50     my $data = PVE::Tools::file_get_contents('/proc/self/cgroup');
  51     while ($data =~ /^\d+:([^:\n]*):.*$/gm) {
  52         my $type = $1;
  53         if (length($type)) {
  54             $v1->{$_} = 1 foreach split(/,/, $type);
  55         } else {
  56             $v2 = 1;
  57         }
  58     }
  59     return wantarray ? ($v1, $v2) : $v1;
  60 }
  61
  62 # Get the set v2 controller list from the `cgroup.controllers` file.
  63 my sub get_v2_controllers {
  64     my $v2 = eval { file_get_contents('/sys/fs/cgroup/cgroup.controllers') }
  65         || eval { file_get_contents('/sys/fs/cgroup/unified/cgroup.controllers') };
  66     return undef if !defined $v2;
  67
  68     # It's a simple space separated list:
  69     return { map { $_ => 1 } split(/\s+/, $v2) };
  70 }
  71
  72 my $CGROUP_CONTROLLERS = undef;
  73 # Get a list of controllers enabled in each cgroup subsystem.
  74 #
  75 # This is a more complete version of `PVE::LXC::get_cgroup_subsystems`.
  76 #
  77 # Returns 2 sets (hashes mapping controller names to `1`), one for each cgroup
  78 # version.
  79 sub get_cgroup_controllers() {
  80     if (!defined($CGROUP_CONTROLLERS)) {
  81         my ($v1, undef) = get_v1_controllers();
  82         my $v2 = get_v2_controllers();
  83
  84         $CGROUP_CONTROLLERS = [$v1, $v2];
  85     }
  86
  87     return $CGROUP_CONTROLLERS->@*;
  88 }
  89
  90 my $CGROUP_MODE = undef;
  91 # Figure out which cgroup mode we're operating under:
  92 #
  93 # Returns 1 if cgroupv1 controllers exist (hybrid or legacy mode), and 2 in a
  94 # cgroupv2-only environment.
  95 #
  96 # NOTE: To fully support a hybrid layout it is better to use functions like
  97 # `cpuset_controller_path`.
  98 #
  99 # This is a function, not a method!
 100 sub cgroup_mode() {
 101     if (!defined($CGROUP_MODE)) {
 102         my ($v1, $v2) = get_cgroup_controllers();
 103         if (keys %$v1) {
 104             # hybrid or legacy mode
 105             $CGROUP_MODE = 1;
 106         } elsif ($v2) {
 107             $CGROUP_MODE = 2;
 108         }
 109     }
 110
 111     die "unknown cgroup mode\n" if !defined($CGROUP_MODE);
 112     return $CGROUP_MODE;
 113 }
 114
 115 my $CGROUPV2_PATH = undef;
 116 sub cgroupv2_base_path() {
 117     if (!defined($CGROUPV2_PATH)) {
 118         if (cgroup_mode() == 2) {
 119             $CGROUPV2_PATH = '/sys/fs/cgroup';
 120         } else {
 121             $CGROUPV2_PATH = '/sys/fs/cgroup/unified';
 122         }
 123     }
 124     return $CGROUPV2_PATH;
 125 }
 126
 127 # Find a cgroup controller and return its path and version.
 128 #
 129 # LXC initializes the unified hierarchy first, so if a controller is
 130 # available via both we favor cgroupv2 here as well.
 131 #
 132 # Returns nothing if the controller is not available.
 133 sub find_cgroup_controller($) {
 134     my ($controller) = @_;
 135
 136     my ($v1, $v2) = get_cgroup_controllers();
 137
 138     if (!defined($controller) || $v2->{$controller}) {
 139         my $path = cgroupv2_base_path();
 140         return wantarray ? ($path, 2) : $path;
 141     }
 142
 143     if (defined($controller) && $v1->{$controller}) {
 144         my $path = "/sys/fs/cgroup/$controller";
 145         return wantarray ? ($path, 1) : $path;
 146     }
 147
 148     return;
 149 }
 150
 151 my $CG_PATH_CPUSET = undef;
 152 my $CG_VER_CPUSET = undef;
 153 # Find the cpuset cgroup controller.
 154 #
 155 # This is a function, not a method!
 156 sub cpuset_controller_path() {
 157     if (!defined($CG_PATH_CPUSET)) {
 158         ($CG_PATH_CPUSET, $CG_VER_CPUSET) = find_cgroup_controller('cpuset')
 159             or die "failed to find cpuset controller\n";
 160     }
 161
 162     return wantarray ? ($CG_PATH_CPUSET, $CG_VER_CPUSET) : $CG_PATH_CPUSET;
 163 }
 164
 165 # Get a subdirectory (without the cgroup mount point) for a controller.
 166 #
 167 # If `$controller` is `undef`, get the unified (cgroupv2) path.
 168 #
 169 # Note that in cgroup v2, lxc uses the activated controller names
 170 # (`cgroup.controllers` file) as list of controllers for the unified hierarchy,
 171 # so this returns a result when a `controller` is provided even when using
 172 # a pure cgroupv2 setup.
 173 my sub get_subdir {
 174     my ($self, $controller, $limiting) = @_;
 175
 176     my $entry_name = $controller || 'unified';
 177     my $entry = ($self->{controllers}->{$entry_name} //= {});
 178
 179     my $kind = $limiting ? 'limit' : 'ns';
 180     my $path = $entry->{$kind};
 181
 182     return $path if defined $path;
 183
 184     $path = PVE::LXC::Command::get_cgroup_path(
 185         $self->{vmid},
 186         $controller,
 187         $limiting,
 188     ) or return undef;
 189
 190     # untaint:
 191     if ($path =~ /\.\./) {
 192         die "lxc returned suspicious path: '$path'\n";
 193     }
 194     ($path) = ($path =~ /^(.*)$/s);
 195
 196     $entry->{$kind} = $path;
 197
 198     return $path;
 199 }
 200
 201 # Get path and version for a controller.
 202 #
 203 # `$controller` may be `undef`, see get_subdir above for details.
 204 #
 205 # Returns either just the path, or the path and cgroup version as a tuple.
 206 sub get_path {
 207     my ($self, $controller, $limiting) = @_;
 208
 209     # Find the controller before querying the lxc monitor via a socket:
 210     my ($cgpath, $ver) = find_cgroup_controller($controller)
 211         or return undef;
 212
 213     my $path = get_subdir($self, $controller, $limiting)
 214         or return undef;
 215
 216     $path = "$cgpath/$path";
 217     return wantarray ? ($path, $ver) : $path;
 218 }
 219
 220 # Convenience method to get the path info if the first existing controller.
 221 #
 222 # Returns the same as `get_path`.
 223 sub get_any_path {
 224     my ($self, $limiting, @controllers) = @_;
 225
 226     my ($path, $ver);
 227     for my $c (@controllers) {
 228         ($path, $ver) = $self->get_path($c, $limiting);
 229         last if defined $path;
 230     }
 231     return wantarray ? ($path, $ver) : $path;
 232 }
 233
 234 # Parse a 'Nested keyed' file:
 235 #
 236 # See kernel documentation `admin-guide/cgroup-v2.rst` 4.1.
 237 my sub parse_nested_keyed_file($) {
 238     my ($data) = @_;
 239     my $res = {};
 240     foreach my $line (split(/\n/, $data)) {
 241         my ($key, @values) = split(/\s+/, $line);
 242
 243         my $d = ($res->{$key} = {});
 244
 245         foreach my $value (@values) {
 246             if (my ($key, $value) = ($value =~ /^([^=]+)=(.*)$/)) {
 247                 $d->{$key} = $value;
 248             } else {
 249                 warn "bad key=value pair in nested keyed file\n";
 250             }
 251         }
 252     }
 253     return $res;
 254 }
 255
 256 # Parse a 'Flat keyed' file:
 257 #
 258 # See kernel documentation `admin-guide/cgroup-v2.rst` 4.1.
 259 my sub parse_flat_keyed_file($) {
 260     my ($data) = @_;
 261     my $res = {};
 262     foreach my $line (split(/\n/, $data)) {
 263         if (my ($key, $value) = ($line =~ /^(\S+)\s+(.*)$/)) {
 264             $res->{$key} = $value;
 265         } else {
 266             warn "bad 'key value' pair in flat keyed file\n";
 267         }
 268     }
 269     return $res;
 270 }
 271
 272 # Parse out 'diskread' and 'diskwrite' values from I/O stats for this container.
 273 sub get_io_stats {
 274     my ($self) = @_;
 275
 276     my $res = {
 277         diskread => 0,
 278         diskwrite => 0,
 279     };
 280
 281     # With cgroupv1 we have a 'blkio' controller, with cgroupv2 it's just 'io':
 282     my ($path, $ver) = $self->get_any_path(1, 'io', 'blkio');
 283     if (!defined($path)) {
 284         # container not running
 285         return undef;
 286     } elsif ($ver == 2) {
 287         # cgroupv2 environment, io controller enabled
 288         my $io_stat = file_get_contents("$path/io.stat");
 289
 290         my $data = parse_nested_keyed_file($io_stat);
 291         foreach my $dev (keys %$data) {
 292             my $dev = $data->{$dev};
 293             if (my $b = $dev->{rbytes}) {
 294                 $res->{diskread} += $b;
 295             }
 296             if (my $b = $dev->{wbytes}) {
 297                 $res->{diskread} += $b;
 298             }
 299         }
 300
 301         return $res;
 302     } elsif ($ver == 1) {
 303         # cgroupv1 environment:
 304         my $io = file_get_contents("$path/blkio.throttle.io_service_bytes_recursive");
 305         foreach my $line (split(/\n/, $io)) {
 306             if (my ($type, $bytes) = ($line =~ /^\d+:\d+\s+(Read|Write)\s+(\d+)$/)) {
 307                 $res->{diskread} += $bytes if $type eq 'Read';
 308                 $res->{diskwrite} += $bytes if $type eq 'Write';
 309             }
 310         }
 311
 312         return $res;
 313     } else {
 314         die "bad cgroup version: $ver\n";
 315     }
 316
 317     # container not running
 318     return undef;
 319 }
 320
 321 # Read utime and stime for this container from the cpuacct cgroup.
 322 # Values are in milliseconds!
 323 sub get_cpu_stat {
 324     my ($self) = @_;
 325
 326     my $res = {
 327         utime => 0,
 328         stime => 0,
 329     };
 330
 331     my ($path, $ver) = $self->get_any_path(1, 'cpuacct', 'cpu');
 332     if (!defined($path)) {
 333         # container not running
 334         return undef;
 335     } elsif ($ver == 2) {
 336         my $data = eval { file_get_contents("$path/cpu.stat") };
 337
 338         # or no io controller available:
 339         return undef if !defined($data);
 340
 341         $data = parse_flat_keyed_file($data);
 342         $res->{utime} = int($data->{user_usec} / 1000);
 343         $res->{stime} = int($data->{system_usec} / 1000);
 344     } elsif ($ver == 1) {
 345         # cgroupv1 environment:
 346         my $clock_ticks = POSIX::sysconf(&POSIX::_SC_CLK_TCK);
 347         my $clk_to_usec = 1000 / $clock_ticks;
 348
 349         my $data = parse_flat_keyed_file(file_get_contents("$path/cpuacct.stat"));
 350         $res->{utime} = int($data->{user} * $clk_to_usec);
 351         $res->{stime} = int($data->{system} * $clk_to_usec);
 352     } else {
 353         die "bad cgroup version: $ver\n";
 354     }
 355
 356     return $res;
 357 }
 358
 359 # Parse some memory data from `memory.stat`
 360 sub get_memory_stat {
 361     my ($self) = @_;
 362
 363     my $res = {
 364         mem => 0,
 365         swap => 0,
 366     };
 367
 368     my ($path, $ver) = $self->get_path('memory', 1);
 369     if (!defined($path)) {
 370         # container most likely isn't running
 371         return undef;
 372     } elsif ($ver == 2) {
 373         my $mem = file_get_contents("$path/memory.current");
 374         my $swap = file_get_contents("$path/memory.swap.current");
 375
 376         chomp ($mem, $swap);
 377
 378         # FIXME: For the cgv1 equivalent of `total_cache` we may need to sum up
 379         # the values in `memory.stat`...
 380
 381         $res->{mem} = $mem;
 382         $res->{swap} = $swap;
 383     } elsif ($ver == 1) {
 384         # cgroupv1 environment:
 385         my $stat = parse_flat_keyed_file(file_get_contents("$path/memory.stat"));
 386         my $mem = file_get_contents("$path/memory.usage_in_bytes");
 387         my $memsw = file_get_contents("$path/memory.memsw.usage_in_bytes");
 388         chomp ($mem, $memsw);
 389
 390         $res->{mem} = $mem - $stat->{total_cache};
 391         $res->{swap} = $memsw - $mem;
 392     } else {
 393         die "bad cgroup version: $ver\n";
 394     }
 395
 396     return $res;
 397 }
 398
 399 # Change the memory limit for this container.
 400 #
 401 # Dies on error (including a not-running or currently-shutting-down guest).
 402 sub change_memory_limit {
 403     my ($self, $mem_bytes, $swap_bytes) = @_;
 404
 405     my ($path, $ver) = $self->get_path('memory', 1);
 406     if (!defined($path)) {
 407         die "trying to change memory cgroup values: container not running\n";
 408     } elsif ($ver == 2) {
 409         PVE::ProcFSTools::write_proc_entry("$path/memory.swap.max", $swap_bytes)
 410             if defined($swap_bytes);
 411         PVE::ProcFSTools::write_proc_entry("$path/memory.max", $mem_bytes)
 412             if defined($mem_bytes);
 413     } elsif ($ver == 1) {
 414         # With cgroupv1 we cannot control memory and swap limits separately.
 415         # This also means that since the two values aren't independent, we need to handle
 416         # growing and shrinking separately.
 417         my $path_mem = "$path/memory.limit_in_bytes";
 418         my $path_memsw = "$path/memory.memsw.limit_in_bytes";
 419
 420         my $old_mem_bytes = file_get_contents($path_mem);
 421         my $old_memsw_bytes = file_get_contents($path_memsw);
 422         chomp($old_mem_bytes, $old_memsw_bytes);
 423
 424         $mem_bytes //= $old_mem_bytes;
 425         $swap_bytes //= $old_memsw_bytes - $old_mem_bytes;
 426         my $memsw_bytes = $mem_bytes + $swap_bytes;
 427
 428         if ($memsw_bytes > $old_memsw_bytes) {
 429             # Growing the limit means growing the combined limit first, then pulling the
 430             # memory limitup.
 431             PVE::ProcFSTools::write_proc_entry($path_memsw, $memsw_bytes);
 432             PVE::ProcFSTools::write_proc_entry($path_mem, $mem_bytes);
 433         } else {
 434             # Shrinking means we first need to shrink the mem-only memsw cannot be
 435             # shrunk below it.
 436             PVE::ProcFSTools::write_proc_entry($path_mem, $mem_bytes);
 437             PVE::ProcFSTools::write_proc_entry($path_memsw, $memsw_bytes);
 438         }
 439     } else {
 440         die "bad cgroup version: $ver\n";
 441     }
 442
 443     # return a truth value
 444     return 1;
 445 }
 446
 447 # Change the cpu quota for a container.
 448 #
 449 # Dies on error (including a not-running or currently-shutting-down guest).
 450 sub change_cpu_quota {
 451     my ($self, $quota, $period) = @_;
 452
 453     die "quota without period not allowed\n" if !defined($period) && defined($quota);
 454
 455     my ($path, $ver) = $self->get_path('cpu', 1);
 456     if (!defined($path)) {
 457         die "trying to change cpu quota cgroup values: container not running\n";
 458     } elsif ($ver == 2) {
 459         # cgroupv2 environment, an undefined (unlimited) quota is defined as "max"
 460         # in this interface:
 461         $quota //= 'max'; # unlimited
 462         if (defined($quota)) {
 463             PVE::ProcFSTools::write_proc_entry("$path/cpu.max", "$quota $period");
 464         } else {
 465             # we're allowed to only write the quota:
 466             PVE::ProcFSTools::write_proc_entry("$path/cpu.max", 'max');
 467         }
 468     } elsif ($ver == 1) {
 469         $quota //= -1; # unlimited
 470         $period //= -1;
 471         PVE::ProcFSTools::write_proc_entry("$path/cpu.cfs_period_us", $period);
 472         PVE::ProcFSTools::write_proc_entry("$path/cpu.cfs_quota_us", $quota);
 473     } else {
 474         die "bad cgroup version: $ver\n";
 475     }
 476
 477     # return a truth value
 478     return 1;
 479 }
 480
 481 # Change the cpu "shares" for a container.
 482 #
 483 # In cgroupv1 we used a value in `[0..500000]` with a default of 1024.
 484 #
 485 # In cgroupv2 we do not have "shares", we have "weights" in the range
 486 # of `[1..10000]` with a default of 100.
 487 #
 488 # Since the default values don't match when scaling linearly, we use the
 489 # values we get as-is and simply error for values >10000 in cgroupv2.
 490 #
 491 # It is left to the user to figure this out for now.
 492 #
 493 # Dies on error (including a not-running or currently-shutting-down guest).
 494 sub change_cpu_shares {
 495     my ($self, $shares, $cgroupv1_default) = @_;
 496
 497     my ($path, $ver) = $self->get_path('cpu', 1);
 498     if (!defined($path)) {
 499         die "trying to change cpu shares/weight cgroup values: container not running\n";
 500     } elsif ($ver == 2) {
 501         # the cgroupv2 documentation defines the default to 100
 502         $shares //= 100;
 503         die "cpu weight (shares) must be in range [1, 10000]\n" if $shares < 1 || $shares > 10000;
 504         PVE::ProcFSTools::write_proc_entry("$path/cpu.weight", $shares);
 505     } elsif ($ver == 1) {
 506         $shares //= 100;
 507         PVE::ProcFSTools::write_proc_entry("$path/cpu.shares", $shares // $cgroupv1_default);
 508     } else {
 509         die "bad cgroup version: $ver\n";
 510     }
 511
 512     # return a truth value
 513     return 1;
 514 }
 515
 516 my sub v1_freeze_thaw {
 517     my ($self, $controller_path, $freeze) = @_;
 518     my $path = get_subdir($self, 'freezer', 1)
 519         or die "trying to freeze container: container not running\n";
 520     $path = "$controller_path/$path/freezer.state";
 521
 522     my $data = $freeze ? 'FROZEN' : 'THAWED';
 523     PVE::ProcFSTools::write_proc_entry($path, $data);
 524
 525     # Here we just poll the freezer.state once per second.
 526     while (1) {
 527         my $state = file_get_contents($path);
 528         chomp $state;
 529         last if $state eq $data;
 530     }
 531 }
 532
 533 my sub v2_freeze_thaw {
 534     my ($self, $controller_path, $freeze) = @_;
 535     my $path = get_subdir($self, undef, 1)
 536         or die "trying to freeze container: container not running\n";
 537     $path = "$controller_path/$path";
 538
 539     my $desired_state = $freeze ? 1 : 0;
 540
 541     # cgroupv2 supports poll events on cgroup.events which contains the frozen
 542     # state.
 543     my $fh = IO::File->new("$path/cgroup.events", 'r')
 544         or die "failed to open $path/cgroup.events file: $!\n";
 545     my $select = IO::Select->new();
 546     $select->add($fh);
 547
 548     PVE::ProcFSTools::write_proc_entry("$path/cgroup.freeze", $desired_state);
 549     while (1) {
 550         my $data = do {
 551             local $/ = undef;
 552             <$fh>
 553         };
 554         $data = parse_flat_keyed_file($data);
 555         last if $data->{frozen} == $desired_state;
 556         my @handles = $select->has_exception();
 557         next if !@handles;
 558         seek($fh, 0, 0)
 559             or die "failed to rewind cgroup.events file: $!\n";
 560     }
 561 }
 562
 563 # Freeze or unfreeze a container.
 564 #
 565 # This will freeze the container at its outer (limiting) cgroup path. We use
 566 # this instead of `lxc-freeze` as `lxc-freeze` from lxc4 will not be able to
 567 # fetch the cgroup path from contaienrs still running on lxc3.
 568 sub freeze_thaw {
 569     my ($self, $freeze) = @_;
 570
 571     my $controller_path = find_cgroup_controller('freezer');
 572     if (defined($controller_path)) {
 573         return v1_freeze_thaw($self, $controller_path, $freeze);
 574     } else {
 575         # cgroupv2 always has a freezer, there can be both cgv1 and cgv2
 576         # freezers, but we'll prefer v1 when it's available as that's what lxc
 577         # does as well...
 578         return v2_freeze_thaw($self, cgroupv2_base_path(), $freeze);
 579     }
 580 }
 581
 582 1;