src/PVE/CGroup.pm

   1 # cgroup handler
   2 #
   3 # This package should deal with figuring out the right cgroup path for a
   4 # container (via the command socket), reading and writing cgroup values, and
   5 # handling cgroup v1 & v2 differences.
   6 #
   7 # Note that the long term plan is to have resource manage functions instead of
   8 # dealing with cgroup files on the outside.
   9
  10 package PVE::CGroup;
  11
  12 use strict;
  13 use warnings;
  14
  15 use IO::File;
  16 use IO::Select;
  17 use POSIX qw();
  18
  19 use PVE::ProcFSTools;
  20 use PVE::Tools qw(
  21     file_get_contents
  22     file_read_firstline
  23 );
  24
  25 # We don't want to do a command socket round trip for every cgroup read/write,
  26 # so any cgroup function needs to have the container's path cached, so this
  27 # package has to be instantiated.
  28 #
  29 # LXC keeps separate paths by controller (although they're normally all the
  30 # same, in our # case anyway), so we cache them by controller as well.
  31 sub new {
  32     my ($class, $vmid) = @_;
  33
  34     my $self = { vmid => $vmid };
  35
  36     return bless $self, $class;
  37 }
  38
  39 # Get the v1 controller list.
  40 #
  41 # Returns a set (hash mapping names to `1`) of cgroupv1 controllers, and an
  42 # optional boolean whether a unified (cgroupv2) hierarchy exists.
  43 #
  44 # Deprecated: Use `get_cgroup_controllers()` instead.
  45 sub get_v1_controllers {
  46     my $v1 = {};
  47     my $v2 = 0;
  48     my $data = PVE::Tools::file_get_contents('/proc/self/cgroup');
  49     while ($data =~ /^\d+:([^:\n]*):.*$/gm) {
  50         my $type = $1;
  51         if (length($type)) {
  52             $v1->{$_} = 1 foreach split(/,/, $type);
  53         } else {
  54             $v2 = 1;
  55         }
  56     }
  57     return wantarray ? ($v1, $v2) : $v1;
  58 }
  59
  60 # Get the set v2 controller list from the `cgroup.controllers` file.
  61 my sub get_v2_controllers {
  62     my $v2 = eval { file_get_contents('/sys/fs/cgroup/cgroup.controllers') }
  63         || eval { file_get_contents('/sys/fs/cgroup/unified/cgroup.controllers') };
  64     return undef if !defined $v2;
  65
  66     # It's a simple space separated list:
  67     return { map { $_ => 1 } split(/\s+/, $v2) };
  68 }
  69
  70 my $CGROUP_CONTROLLERS = undef;
  71 # Get a list of controllers enabled in each cgroup subsystem.
  72 #
  73 # This is a more complete version of `PVE::LXC::get_cgroup_subsystems`.
  74 #
  75 # Returns 2 sets (hashes mapping controller names to `1`), one for each cgroup
  76 # version.
  77 sub get_cgroup_controllers() {
  78     if (!defined($CGROUP_CONTROLLERS)) {
  79         my ($v1, undef) = get_v1_controllers();
  80         my $v2 = get_v2_controllers();
  81
  82         $CGROUP_CONTROLLERS = [$v1, $v2];
  83     }
  84
  85     return $CGROUP_CONTROLLERS->@*;
  86 }
  87
  88 my $CGROUP_MODE = undef;
  89 # Figure out which cgroup mode we're operating under:
  90 #
  91 # Returns 1 if cgroupv1 controllers exist (hybrid or legacy mode), and 2 in a
  92 # cgroupv2-only environment.
  93 #
  94 # NOTE: To fully support a hybrid layout it is better to use functions like
  95 # `cpuset_controller_path`.
  96 #
  97 # This is a function, not a method!
  98 sub cgroup_mode() {
  99     if (!defined($CGROUP_MODE)) {
 100         my ($v1, $v2) = get_cgroup_controllers();
 101         if (keys %$v1) {
 102             # hybrid or legacy mode
 103             $CGROUP_MODE = 1;
 104         } elsif ($v2) {
 105             $CGROUP_MODE = 2;
 106         }
 107     }
 108
 109     die "unknown cgroup mode\n" if !defined($CGROUP_MODE);
 110     return $CGROUP_MODE;
 111 }
 112
 113 my $CGROUPV2_PATH = undef;
 114 sub cgroupv2_base_path() {
 115     if (!defined($CGROUPV2_PATH)) {
 116         if (cgroup_mode() == 2) {
 117             $CGROUPV2_PATH = '/sys/fs/cgroup';
 118         } else {
 119             $CGROUPV2_PATH = '/sys/fs/cgroup/unified';
 120         }
 121     }
 122     return $CGROUPV2_PATH;
 123 }
 124
 125 # Find a cgroup controller and return its path and version.
 126 #
 127 # LXC initializes the unified hierarchy first, so if a controller is
 128 # available via both we favor cgroupv2 here as well.
 129 #
 130 # Returns nothing if the controller is not available.
 131
 132 sub find_cgroup_controller($) {
 133     my ($controller) = @_;
 134
 135     my ($v1, $v2) = get_cgroup_controllers();
 136
 137     if (!defined($controller) || $v2->{$controller}) {
 138         my $path = cgroupv2_base_path();
 139         return wantarray ? ($path, 2) : $path;
 140     }
 141
 142     if (defined($controller) && $v1->{$controller}) {
 143         my $path = "/sys/fs/cgroup/$controller";
 144         return wantarray ? ($path, 1) : $path;
 145     }
 146
 147     return;
 148 }
 149
 150 my $CG_PATH_CPUSET = undef;
 151 my $CG_VER_CPUSET = undef;
 152 # Find the cpuset cgroup controller.
 153 #
 154 # This is a function, not a method!
 155 sub cpuset_controller_path() {
 156     if (!defined($CG_PATH_CPUSET)) {
 157         ($CG_PATH_CPUSET, $CG_VER_CPUSET) = find_cgroup_controller('cpuset')
 158             or die "failed to find cpuset controller\n";
 159     }
 160
 161     return wantarray ? ($CG_PATH_CPUSET, $CG_VER_CPUSET) : $CG_PATH_CPUSET;
 162 }
 163
 164 # Get a subdirectory (without the cgroup mount point) for a controller.
 165 sub get_subdir {
 166     my ($self, $controller, $limiting) = @_;
 167
 168     die "implement in subclass";
 169 }
 170
 171 # Get path and version for a controller.
 172 #
 173 # `$controller` may be `undef`, see get_subdir above for details.
 174 #
 175 # Returns either just the path, or the path and cgroup version as a tuple.
 176 sub get_path {
 177     my ($self, $controller, $limiting) = @_;
 178     # Find the controller before querying the lxc monitor via a socket:
 179     my ($cgpath, $ver) = find_cgroup_controller($controller)
 180         or return undef;
 181
 182     my $path = $self->get_subdir($controller, $limiting)
 183         or return undef;
 184
 185     $path = "$cgpath/$path";
 186     return wantarray ? ($path, $ver) : $path;
 187 }
 188
 189 # Convenience method to get the path info if the first existing controller.
 190 #
 191 # Returns the same as `get_path`.
 192 sub get_any_path {
 193     my ($self, $limiting, @controllers) = @_;
 194
 195     my ($path, $ver);
 196     for my $c (@controllers) {
 197         ($path, $ver) = $self->get_path($c, $limiting);
 198         last if defined $path;
 199     }
 200     return wantarray ? ($path, $ver) : $path;
 201 }
 202
 203 # Parse a 'Nested keyed' file:
 204 #
 205 # See kernel documentation `admin-guide/cgroup-v2.rst` 4.1.
 206 my sub parse_nested_keyed_file($) {
 207     my ($data) = @_;
 208     my $res = {};
 209     foreach my $line (split(/\n/, $data)) {
 210         my ($key, @values) = split(/\s+/, $line);
 211
 212         my $d = ($res->{$key} = {});
 213
 214         foreach my $value (@values) {
 215             if (my ($key, $value) = ($value =~ /^([^=]+)=(.*)$/)) {
 216                 $d->{$key} = $value;
 217             } else {
 218                 warn "bad key=value pair in nested keyed file\n";
 219             }
 220         }
 221     }
 222     return $res;
 223 }
 224
 225 # Parse a 'Flat keyed' file:
 226 #
 227 # See kernel documentation `admin-guide/cgroup-v2.rst` 4.1.
 228 my sub parse_flat_keyed_file($) {
 229     my ($data) = @_;
 230     my $res = {};
 231     foreach my $line (split(/\n/, $data)) {
 232         if (my ($key, $value) = ($line =~ /^(\S+)\s+(.*)$/)) {
 233             $res->{$key} = $value;
 234         } else {
 235             warn "bad 'key value' pair in flat keyed file\n";
 236         }
 237     }
 238     return $res;
 239 }
 240
 241 # Parse out 'diskread' and 'diskwrite' values from I/O stats for this container.
 242 sub get_io_stats {
 243     my ($self) = @_;
 244
 245     my $res = {
 246         diskread => 0,
 247         diskwrite => 0,
 248     };
 249
 250     # With cgroupv1 we have a 'blkio' controller, with cgroupv2 it's just 'io':
 251     my ($path, $ver) = $self->get_any_path(1, 'io', 'blkio');
 252     if (!defined($path)) {
 253         # container not running
 254         return undef;
 255     } elsif ($ver == 2) {
 256         # cgroupv2 environment, io controller enabled
 257         my $io_stat = file_get_contents("$path/io.stat");
 258
 259         my $data = parse_nested_keyed_file($io_stat);
 260         foreach my $dev (keys %$data) {
 261             my $dev = $data->{$dev};
 262             if (my $b = $dev->{rbytes}) {
 263                 $res->{diskread} += $b;
 264             }
 265             if (my $b = $dev->{wbytes}) {
 266                 $res->{diskread} += $b;
 267             }
 268         }
 269
 270         return $res;
 271     } elsif ($ver == 1) {
 272         # cgroupv1 environment:
 273         my $io = file_get_contents("$path/blkio.throttle.io_service_bytes_recursive");
 274         foreach my $line (split(/\n/, $io)) {
 275             if (my ($type, $bytes) = ($line =~ /^\d+:\d+\s+(Read|Write)\s+(\d+)$/)) {
 276                 $res->{diskread} += $bytes if $type eq 'Read';
 277                 $res->{diskwrite} += $bytes if $type eq 'Write';
 278             }
 279         }
 280
 281         return $res;
 282     } else {
 283         die "bad cgroup version: $ver\n";
 284     }
 285
 286     # container not running
 287     return undef;
 288 }
 289
 290 # Read utime and stime for this container from the cpuacct cgroup.
 291 # Values are in milliseconds!
 292 sub get_cpu_stat {
 293     my ($self) = @_;
 294
 295     my $res = {
 296         utime => 0,
 297         stime => 0,
 298     };
 299
 300     my ($path, $ver) = $self->get_any_path(1, 'cpuacct', 'cpu');
 301     if (!defined($path)) {
 302         # container not running
 303         return undef;
 304     } elsif ($ver == 2) {
 305         my $data = eval { file_get_contents("$path/cpu.stat") };
 306
 307         # or no io controller available:
 308         return undef if !defined($data);
 309
 310         $data = parse_flat_keyed_file($data);
 311         $res->{utime} = int($data->{user_usec} / 1000);
 312         $res->{stime} = int($data->{system_usec} / 1000);
 313     } elsif ($ver == 1) {
 314         # cgroupv1 environment:
 315         my $clock_ticks = POSIX::sysconf(&POSIX::_SC_CLK_TCK);
 316         my $clk_to_usec = 1000 / $clock_ticks;
 317
 318         my $data = parse_flat_keyed_file(file_get_contents("$path/cpuacct.stat"));
 319         $res->{utime} = int($data->{user} * $clk_to_usec);
 320         $res->{stime} = int($data->{system} * $clk_to_usec);
 321     } else {
 322         die "bad cgroup version: $ver\n";
 323     }
 324
 325     return $res;
 326 }
 327
 328 # Parse some memory data from `memory.stat`
 329 sub get_memory_stat {
 330     my ($self) = @_;
 331
 332     my $res = {
 333         mem => 0,
 334         swap => 0,
 335     };
 336
 337     my ($path, $ver) = $self->get_path('memory', 1);
 338     if (!defined($path)) {
 339         # container most likely isn't running
 340         return undef;
 341     } elsif ($ver == 2) {
 342         my $mem = file_get_contents("$path/memory.current");
 343         my $swap = file_get_contents("$path/memory.swap.current");
 344         my $stat = parse_flat_keyed_file(file_get_contents("$path/memory.stat"));
 345
 346         chomp ($mem, $swap);
 347
 348         $res->{mem} = $mem - $stat->{file};
 349         $res->{swap} = $swap;
 350     } elsif ($ver == 1) {
 351         # cgroupv1 environment:
 352         my $stat = parse_flat_keyed_file(file_get_contents("$path/memory.stat"));
 353         my $mem = file_get_contents("$path/memory.usage_in_bytes");
 354         my $memsw = file_get_contents("$path/memory.memsw.usage_in_bytes");
 355         chomp ($mem, $memsw);
 356
 357         $res->{mem} = $mem - $stat->{total_cache};
 358         $res->{swap} = $memsw - $mem;
 359     } else {
 360         die "bad cgroup version: $ver\n";
 361     }
 362
 363     return $res;
 364 }
 365
 366 sub get_pressure_stat {
 367     my ($self) = @_;
 368
 369     my $res = {
 370         cpu => {
 371             some => { avg10 => 0, avg60 => 0, avg300 => 0 }
 372         },
 373         memory => {
 374             some => { avg10 => 0, avg60 => 0, avg300 => 0 },
 375             full => { avg10 => 0, avg60 => 0, avg300 => 0 }
 376         },
 377         io => {
 378             some => { avg10 => 0, avg60 => 0, avg300 => 0 },
 379             full => { avg10 => 0, avg60 => 0, avg300 => 0 }
 380         },
 381     };
 382
 383     my ($path, $version) = $self->get_path(undef, 1);
 384     if (!defined($path)) {
 385         return $res; # container or VM most likely isn't running, retrun zero stats
 386     } elsif ($version == 1) {
 387         return undef; # v1 controller does not provides pressure stat
 388     } elsif ($version == 2) {
 389         for my $type (qw(cpu memory io)) {
 390             my $stats = PVE::ProcFSTools::parse_pressure("$path/$type.pressure");
 391             $res->{$type} = $stats if $stats;
 392         }
 393     } else {
 394         die "bad cgroup version: $version\n";
 395     }
 396
 397     return $res;
 398 }
 399
 400 # Change the memory limit for this container.
 401 #
 402 # Dies on error (including a not-running or currently-shutting-down guest).
 403 sub change_memory_limit {
 404     my ($self, $mem_bytes, $swap_bytes) = @_;
 405
 406     my ($path, $ver) = $self->get_path('memory', 1);
 407     if (!defined($path)) {
 408         die "trying to change memory cgroup values: container not running\n";
 409     } elsif ($ver == 2) {
 410         PVE::ProcFSTools::write_proc_entry("$path/memory.swap.max", $swap_bytes)
 411             if defined($swap_bytes);
 412         PVE::ProcFSTools::write_proc_entry("$path/memory.max", $mem_bytes)
 413             if defined($mem_bytes);
 414     } elsif ($ver == 1) {
 415         # With cgroupv1 we cannot control memory and swap limits separately.
 416         # This also means that since the two values aren't independent, we need to handle
 417         # growing and shrinking separately.
 418         my $path_mem = "$path/memory.limit_in_bytes";
 419         my $path_memsw = "$path/memory.memsw.limit_in_bytes";
 420
 421         my $old_mem_bytes = file_get_contents($path_mem);
 422         my $old_memsw_bytes = file_get_contents($path_memsw);
 423         chomp($old_mem_bytes, $old_memsw_bytes);
 424
 425         $mem_bytes //= $old_mem_bytes;
 426         $swap_bytes //= $old_memsw_bytes - $old_mem_bytes;
 427         my $memsw_bytes = $mem_bytes + $swap_bytes;
 428
 429         if ($memsw_bytes > $old_memsw_bytes) {
 430             # Growing the limit means growing the combined limit first, then pulling the
 431             # memory limitup.
 432             PVE::ProcFSTools::write_proc_entry($path_memsw, $memsw_bytes);
 433             PVE::ProcFSTools::write_proc_entry($path_mem, $mem_bytes);
 434         } else {
 435             # Shrinking means we first need to shrink the mem-only memsw cannot be
 436             # shrunk below it.
 437             PVE::ProcFSTools::write_proc_entry($path_mem, $mem_bytes);
 438             PVE::ProcFSTools::write_proc_entry($path_memsw, $memsw_bytes);
 439         }
 440     } else {
 441         die "bad cgroup version: $ver\n";
 442     }
 443
 444     # return a truth value
 445     return 1;
 446 }
 447
 448 # Change the cpu quota for a container.
 449 #
 450 # Dies on error (including a not-running or currently-shutting-down guest).
 451 sub change_cpu_quota {
 452     my ($self, $quota, $period) = @_;
 453
 454     die "quota without period not allowed\n" if !defined($period) && defined($quota);
 455
 456     my ($path, $ver) = $self->get_path('cpu', 1);
 457     if (!defined($path)) {
 458         die "trying to change cpu quota cgroup values: container not running\n";
 459     } elsif ($ver == 2) {
 460         # cgroupv2 environment, an undefined (unlimited) quota is defined as "max"
 461         # in this interface:
 462         $quota //= 'max'; # unlimited
 463         if (defined($quota)) {
 464             PVE::ProcFSTools::write_proc_entry("$path/cpu.max", "$quota $period");
 465         } else {
 466             # we're allowed to only write the quota:
 467             PVE::ProcFSTools::write_proc_entry("$path/cpu.max", 'max');
 468         }
 469     } elsif ($ver == 1) {
 470         $quota //= -1; # unlimited
 471         $period //= -1;
 472         PVE::ProcFSTools::write_proc_entry("$path/cpu.cfs_period_us", $period);
 473         PVE::ProcFSTools::write_proc_entry("$path/cpu.cfs_quota_us", $quota);
 474     } else {
 475         die "bad cgroup version: $ver\n";
 476     }
 477
 478     # return a truth value
 479     return 1;
 480 }
 481
 482 # Change the cpu "shares" for a container.
 483 #
 484 # In cgroupv1 we used a value in `[0..500000]` with a default of 1024.
 485 #
 486 # In cgroupv2 we do not have "shares", we have "weights" in the range
 487 # of `[1..10000]` with a default of 100.
 488 #
 489 # Since the default values don't match when scaling linearly, we use the
 490 # values we get as-is and simply error for values >10000 in cgroupv2.
 491 #
 492 # It is left to the user to figure this out for now.
 493 #
 494 # Dies on error (including a not-running or currently-shutting-down guest).
 495 sub change_cpu_shares {
 496     my ($self, $shares, $cgroupv1_default) = @_;
 497
 498     my ($path, $ver) = $self->get_path('cpu', 1);
 499     if (!defined($path)) {
 500         die "trying to change cpu shares/weight cgroup values: container not running\n";
 501     } elsif ($ver == 2) {
 502         # the cgroupv2 documentation defines the default to 100
 503         $shares //= 100;
 504         die "cpu weight (shares) must be in range [1, 10000]\n" if $shares < 1 || $shares > 10000;
 505         PVE::ProcFSTools::write_proc_entry("$path/cpu.weight", $shares);
 506     } elsif ($ver == 1) {
 507         $shares //= 1024;
 508         PVE::ProcFSTools::write_proc_entry("$path/cpu.shares", $shares // $cgroupv1_default);
 509     } else {
 510         die "bad cgroup version: $ver\n";
 511     }
 512
 513     # return a truth value
 514     return 1;
 515 }
 516
 517 my sub v1_freeze_thaw {
 518     my ($self, $controller_path, $freeze) = @_;
 519     my $path = $self->get_subdir('freezer', 1)
 520         or die "trying to freeze container: container not running\n";
 521     $path = "$controller_path/$path/freezer.state";
 522
 523     my $data = $freeze ? 'FROZEN' : 'THAWED';
 524     PVE::ProcFSTools::write_proc_entry($path, $data);
 525
 526     # Here we just poll the freezer.state once per second.
 527     while (1) {
 528         my $state = file_get_contents($path);
 529         chomp $state;
 530         last if $state eq $data;
 531     }
 532 }
 533
 534 my sub v2_freeze_thaw {
 535     my ($self, $controller_path, $freeze) = @_;
 536     my $path = $self->get_subdir(undef, 1)
 537         or die "trying to freeze container: container not running\n";
 538     $path = "$controller_path/$path";
 539
 540     my $desired_state = $freeze ? 1 : 0;
 541
 542     # cgroupv2 supports poll events on cgroup.events which contains the frozen
 543     # state.
 544     my $fh = IO::File->new("$path/cgroup.events", 'r')
 545         or die "failed to open $path/cgroup.events file: $!\n";
 546     my $select = IO::Select->new();
 547     $select->add($fh);
 548
 549     PVE::ProcFSTools::write_proc_entry("$path/cgroup.freeze", $desired_state);
 550     while (1) {
 551         my $data = do {
 552             local $/ = undef;
 553             <$fh>
 554         };
 555         $data = parse_flat_keyed_file($data);
 556         last if $data->{frozen} == $desired_state;
 557         my @handles = $select->has_exception();
 558         next if !@handles;
 559         seek($fh, 0, 0)
 560             or die "failed to rewind cgroup.events file: $!\n";
 561     }
 562 }
 563
 564 # Freeze or unfreeze a container.
 565 #
 566 # This will freeze the container at its outer (limiting) cgroup path. We use
 567 # this instead of `lxc-freeze` as `lxc-freeze` from lxc4 will not be able to
 568 # fetch the cgroup path from contaienrs still running on lxc3.
 569 sub freeze_thaw {
 570     my ($self, $freeze) = @_;
 571
 572     my $controller_path = find_cgroup_controller('freezer');
 573     if (defined($controller_path)) {
 574         return v1_freeze_thaw($self, $controller_path, $freeze);
 575     } else {
 576         # cgroupv2 always has a freezer, there can be both cgv1 and cgv2
 577         # freezers, but we'll prefer v1 when it's available as that's what lxc
 578         # does as well...
 579         return v2_freeze_thaw($self, cgroupv2_base_path(), $freeze);
 580     }
 581 }
 582
 583 1;