PVE/QemuServer/Memory.pm

   1 package PVE::QemuServer::Memory;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::JSONSchema qw(parse_property_string);
   7 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
   8 use PVE::Exception qw(raise raise_param_exc);
   9
  10 use PVE::QemuServer;
  11 use PVE::QemuServer::Helpers qw(parse_number_sets);
  12 use PVE::QemuServer::Monitor qw(mon_cmd);
  13
  14 our $MAX_NUMA = 8;
  15
  16 my $numa_fmt = {
  17     cpus => {
  18         type => "string",
  19         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
  20         description => "CPUs accessing this NUMA node.",
  21         format_description => "id[-id];...",
  22     },
  23     memory => {
  24         type => "number",
  25         description => "Amount of memory this NUMA node provides.",
  26         optional => 1,
  27     },
  28     hostnodes => {
  29         type => "string",
  30         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
  31         description => "Host NUMA nodes to use.",
  32         format_description => "id[-id];...",
  33         optional => 1,
  34     },
  35     policy => {
  36         type => 'string',
  37         enum => [qw(preferred bind interleave)],
  38         description => "NUMA allocation policy.",
  39         optional => 1,
  40     },
  41 };
  42 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
  43 our $numadesc = {
  44     optional => 1,
  45     type => 'string', format => $numa_fmt,
  46     description => "NUMA topology.",
  47 };
  48 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
  49
  50 sub parse_numa {
  51     my ($data) = @_;
  52
  53     my $res = parse_property_string($numa_fmt, $data);
  54     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
  55     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
  56     return $res;
  57 }
  58
  59 my $STATICMEM = 1024;
  60
  61 my $_host_bits;
  62 sub get_host_phys_address_bits {
  63     return $_host_bits if defined($_host_bits);
  64
  65     my $fh = IO::File->new ('/proc/cpuinfo', "r") or return;
  66     while (defined(my $line = <$fh>)) {
  67         # hopefully we never need to care about mixed (big.LITTLE) archs
  68         if ($line =~ m/^address sizes\s*:\s*(\d+)\s*bits physical/i) {
  69             $_host_bits = int($1);
  70             $fh->close();
  71             return $_host_bits;
  72         }
  73     }
  74     $fh->close();
  75     return; # undef, cannot really do anything..
  76 }
  77
  78 my sub get_max_mem {
  79     my ($conf) = @_;
  80
  81     my $cpu = {};
  82     if (my $cpu_prop_str = $conf->{cpu}) {
  83         $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str)
  84             or die "Cannot parse cpu description: $cpu_prop_str\n";
  85     }
  86     my $bits;
  87     if (my $phys_bits = $cpu->{'phys-bits'}) {
  88         if ($phys_bits eq 'host') {
  89             $bits = get_host_phys_address_bits();
  90         } elsif ($phys_bits =~ /^(\d+)$/) {
  91             $bits = int($phys_bits);
  92         }
  93     }
  94
  95     if (!defined($bits)) {
  96         my $host_bits = get_host_phys_address_bits() // 36; # fixme: what fallback?
  97         if ($cpu->{cputype} && $cpu->{cputype} =~ /^(host|max)$/) {
  98             $bits = $host_bits;
  99         } else {
 100             $bits = $host_bits > 40 ? 40 : $host_bits; # take the smaller one
 101         }
 102     }
 103
 104     $bits = $bits & ~1; # round down to nearest even as limit is lower with odd bit sizes
 105
 106     # heuristic: remove 20 bits to get MB and half that as QEMU needs some overhead
 107     my $bits_to_max_mem = int(1<<($bits - 21));
 108
 109     return $bits_to_max_mem > 4*1024*1024 ? 4*1024*1024 : $bits_to_max_mem;
 110 }
 111
 112 sub get_numa_node_list {
 113     my ($conf) = @_;
 114     my @numa_map;
 115     for (my $i = 0; $i < $MAX_NUMA; $i++) {
 116         my $entry = $conf->{"numa$i"} or next;
 117         my $numa = parse_numa($entry) or next;
 118         push @numa_map, $i;
 119     }
 120     return @numa_map if @numa_map;
 121     my $sockets = $conf->{sockets} || 1;
 122     return (0..($sockets-1));
 123 }
 124
 125 sub host_numanode_exists {
 126     my ($id) = @_;
 127
 128     return -d "/sys/devices/system/node/node$id/";
 129 }
 130
 131 # only valid when numa nodes map to a single host node
 132 sub get_numa_guest_to_host_map {
 133     my ($conf) = @_;
 134     my $map = {};
 135     for (my $i = 0; $i < $MAX_NUMA; $i++) {
 136         my $entry = $conf->{"numa$i"} or next;
 137         my $numa = parse_numa($entry) or next;
 138         $map->{$i} = print_numa_hostnodes($numa->{hostnodes});
 139     }
 140     return $map if %$map;
 141     my $sockets = $conf->{sockets} || 1;
 142     return {map { $_ => $_ } (0..($sockets-1))};
 143 }
 144
 145 sub foreach_dimm{
 146     my ($conf, $vmid, $memory, $static_memory, $func) = @_;
 147
 148     my $dimm_id = 0;
 149     my $current_size = $static_memory;
 150     my $dimm_size = 0;
 151
 152     if($conf->{hugepages} && $conf->{hugepages} == 1024) {
 153         $dimm_size = 1024;
 154     } else {
 155         $dimm_size = 512;
 156     }
 157
 158     return if $current_size == $memory;
 159
 160     my @numa_map = get_numa_node_list($conf);
 161
 162     for (my $j = 0; $j < 8; $j++) {
 163         for (my $i = 0; $i < 32; $i++) {
 164             my $name = "dimm${dimm_id}";
 165             $dimm_id++;
 166             my $numanode = $numa_map[$i % @numa_map];
 167             $current_size += $dimm_size;
 168             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
 169             return  $current_size if $current_size >= $memory;
 170         }
 171         $dimm_size *= 2;
 172     }
 173 }
 174
 175 sub qemu_memory_hotplug {
 176     my ($vmid, $conf, $defaults, $value) = @_;
 177
 178     return $value if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
 179
 180     my $sockets = $conf->{sockets} || 1;
 181
 182     my $memory = $conf->{memory} || $defaults->{memory};
 183     $value = $defaults->{memory} if !$value;
 184     return $value if $value == $memory;
 185
 186     my $static_memory = $STATICMEM;
 187     $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 188
 189     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
 190     my $MAX_MEM = get_max_mem($conf);
 191     die "you cannot add more memory than max mem $MAX_MEM MB!\n" if $value > $MAX_MEM;
 192
 193     if ($value > $memory) {
 194
 195         my $numa_hostmap;
 196
 197         foreach_dimm($conf, $vmid, $value, $static_memory, sub {
 198             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 199
 200                 return if $current_size <= $conf->{memory};
 201
 202                 if ($conf->{hugepages}) {
 203                     $numa_hostmap = get_numa_guest_to_host_map($conf) if !$numa_hostmap;
 204
 205                     my $hugepages_size = hugepages_size($conf, $dimm_size);
 206                     my $path = hugepages_mount_path($hugepages_size);
 207                     my $host_numanode = $numa_hostmap->{$numanode};
 208                     my $hugepages_topology->{$hugepages_size}->{$host_numanode} = hugepages_nr($dimm_size, $hugepages_size);
 209
 210                     my $code = sub {
 211                         my $hugepages_host_topology = hugepages_host_topology();
 212                         hugepages_allocate($hugepages_topology, $hugepages_host_topology);
 213
 214                         eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true ) };
 215                         if (my $err = $@) {
 216                             hugepages_reset($hugepages_host_topology);
 217                             die $err;
 218                         }
 219
 220                         hugepages_pre_deallocate($hugepages_topology);
 221                     };
 222                     eval { hugepages_update_locked($code); };
 223
 224                 } else {
 225                     eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", size => int($dimm_size*1024*1024) ) };
 226                 }
 227
 228                 if (my $err = $@) {
 229                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 230                     die $err;
 231                 }
 232
 233                 eval { mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
 234                 if (my $err = $@) {
 235                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 236                     die $err;
 237                 }
 238                 #update conf after each succesful module hotplug
 239                 $conf->{memory} = $current_size;
 240                 PVE::QemuConfig->write_config($vmid, $conf);
 241         });
 242
 243     } else {
 244
 245         my $dimms = qemu_memdevices_list($vmid, 'dimm');
 246
 247         my $current_size = $memory;
 248         for my $name (sort { ($b =~ /^dimm(\d+)$/)[0] <=> ($a =~ /^dimm(\d+)$/)[0] } keys %$dimms) {
 249
 250             my $dimm_size = $dimms->{$name}->{size} / 1024 / 1024;
 251
 252             last if $current_size <= $value;
 253
 254             print "try to unplug memory dimm $name\n";
 255
 256             my $retry = 0;
 257             while (1) {
 258                 eval { PVE::QemuServer::qemu_devicedel($vmid, $name) };
 259                 sleep 3;
 260                 my $dimm_list = qemu_memdevices_list($vmid, 'dimm');
 261                 last if !$dimm_list->{$name};
 262                 raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
 263                 $retry++;
 264             }
 265             $current_size -= $dimm_size;
 266             #update conf after each succesful module unplug
 267             $conf->{memory} = $current_size;
 268
 269             eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 270             PVE::QemuConfig->write_config($vmid, $conf);
 271         }
 272     }
 273     return $conf->{memory};
 274 }
 275
 276 sub qemu_memdevices_list {
 277     my ($vmid, $type) = @_;
 278
 279     my $dimmarray = mon_cmd($vmid, "query-memory-devices");
 280     my $dimms = {};
 281
 282     foreach my $dimm (@$dimmarray) {
 283         next if $type && $dimm->{data}->{id} !~ /^$type(\d+)$/;
 284         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
 285         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
 286         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
 287         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
 288         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
 289     }
 290     return $dimms;
 291 }
 292
 293 sub config {
 294     my ($conf, $vmid, $sockets, $cores, $defaults, $hotplug, $cmd) = @_;
 295
 296     my $memory = $conf->{memory} || $defaults->{memory};
 297     my $static_memory = 0;
 298
 299     if ($hotplug) {
 300         die "NUMA needs to be enabled for memory hotplug\n" if !$conf->{numa};
 301         my $MAX_MEM = get_max_mem($conf);
 302         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
 303
 304         for (my $i = 0; $i < $MAX_NUMA; $i++) {
 305             die "cannot enable memory hotplugging with custom NUMA topology\n"
 306                 if $conf->{"numa$i"};
 307         }
 308
 309         my $sockets = $conf->{sockets} || 1;
 310
 311         $static_memory = $STATICMEM;
 312         $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 313
 314         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
 315         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
 316
 317     } else {
 318
 319         $static_memory = $memory;
 320         push @$cmd, '-m', $static_memory;
 321     }
 322
 323     die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa};
 324
 325     if ($conf->{numa}) {
 326
 327         my $numa_totalmemory = undef;
 328         for (my $i = 0; $i < $MAX_NUMA; $i++) {
 329             next if !$conf->{"numa$i"};
 330             my $numa = parse_numa($conf->{"numa$i"});
 331             next if !$numa;
 332             # memory
 333             die "missing NUMA node$i memory value\n" if !$numa->{memory};
 334             my $numa_memory = $numa->{memory};
 335             $numa_totalmemory += $numa_memory;
 336
 337             my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
 338
 339             # cpus
 340             my $cpulists = $numa->{cpus};
 341             die "missing NUMA node$i cpus\n" if !defined($cpulists);
 342             my $cpus = join(',cpus=', map {
 343                 my ($start, $end) = @$_;
 344                 defined($end) ? "$start-$end" : $start
 345             } @$cpulists);
 346
 347             # hostnodes
 348             my $hostnodelists = $numa->{hostnodes};
 349             if (defined($hostnodelists)) {
 350
 351                 my $hostnodes = print_numa_hostnodes($hostnodelists);
 352
 353                 # policy
 354                 my $policy = $numa->{policy};
 355                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
 356                 $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
 357             } else {
 358                 die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
 359             }
 360
 361             push @$cmd, '-object', $mem_object;
 362             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 363         }
 364
 365         die "total memory for NUMA nodes must be equal to vm static memory\n"
 366             if $numa_totalmemory && $numa_totalmemory != $static_memory;
 367
 368         #if no custom tology, we split memory and cores across numa nodes
 369         if (!$numa_totalmemory) {
 370             my $numa_memory = ($static_memory / $sockets);
 371
 372             for (my $i = 0; $i < $sockets; $i++)  {
 373                 die "host NUMA node$i doesn't exist\n"
 374                     if !host_numanode_exists($i) && $conf->{hugepages};
 375
 376                 my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
 377                 push @$cmd, '-object', $mem_object;
 378
 379                 my $cpus = ($cores * $i);
 380                 $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1;
 381
 382                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 383             }
 384         }
 385     }
 386
 387     if ($hotplug) {
 388         foreach_dimm($conf, $vmid, $memory, $static_memory, sub {
 389             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 390
 391             my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size);
 392
 393             push @$cmd, "-object" , $mem_object;
 394             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
 395
 396             die "memory size ($memory) must be aligned to $dimm_size for hotplugging\n"
 397                 if $current_size > $memory;
 398         });
 399     }
 400 }
 401
 402 sub print_mem_object {
 403     my ($conf, $id, $size) = @_;
 404
 405     if ($conf->{hugepages}) {
 406
 407         my $hugepages_size = hugepages_size($conf, $size);
 408         my $path = hugepages_mount_path($hugepages_size);
 409
 410         return "memory-backend-file,id=$id,size=${size}M,mem-path=$path,share=on,prealloc=yes";
 411     } else {
 412         return "memory-backend-ram,id=$id,size=${size}M";
 413     }
 414
 415 }
 416
 417 sub print_numa_hostnodes {
 418     my ($hostnodelists) = @_;
 419
 420     my $hostnodes;
 421     foreach my $hostnoderange (@$hostnodelists) {
 422         my ($start, $end) = @$hostnoderange;
 423         $hostnodes .= ',' if $hostnodes;
 424         $hostnodes .= $start;
 425         $hostnodes .= "-$end" if defined($end);
 426         $end //= $start;
 427         for (my $i = $start; $i <= $end; ++$i ) {
 428             die "host NUMA node$i doesn't exist\n" if !host_numanode_exists($i);
 429         }
 430     }
 431     return $hostnodes;
 432 }
 433
 434 sub hugepages_mount {
 435
 436    my $mountdata = PVE::ProcFSTools::parse_proc_mounts();
 437
 438    foreach my $size (qw(2048 1048576)) {
 439         next if (! -d "/sys/kernel/mm/hugepages/hugepages-${size}kB");
 440
 441         my $path = "/run/hugepages/kvm/${size}kB";
 442
 443         my $found = grep {
 444             $_->[2] =~ /^hugetlbfs/ &&
 445             $_->[1] eq $path
 446         } @$mountdata;
 447
 448         if (!$found) {
 449
 450             File::Path::make_path($path) if (!-d $path);
 451             my $cmd = ['/bin/mount', '-t', 'hugetlbfs', '-o', "pagesize=${size}k", 'hugetlbfs', $path];
 452             run_command($cmd, errmsg => "hugepage mount error");
 453         }
 454    }
 455 }
 456
 457 sub hugepages_mount_path {
 458    my ($size) = @_;
 459
 460    $size = $size * 1024;
 461    return "/run/hugepages/kvm/${size}kB";
 462
 463 }
 464
 465 sub hugepages_nr {
 466   my ($size, $hugepages_size) = @_;
 467
 468   return $size / $hugepages_size;
 469 }
 470
 471 sub hugepages_chunk_size_supported {
 472     my ($size) = @_;
 473
 474     return -d "/sys/kernel/mm/hugepages/hugepages-". ($size * 1024) ."kB";
 475 }
 476
 477 sub hugepages_size {
 478     my ($conf, $size) = @_;
 479     die "hugepages option is not enabled" if !$conf->{hugepages};
 480     die "memory size '$size' is not a positive even integer; cannot use for hugepages\n"
 481         if $size <= 0 || $size & 1;
 482
 483     die "your system doesn't support hugepages\n"
 484         if !hugepages_chunk_size_supported(2) && !hugepages_chunk_size_supported(1024);
 485
 486     if ($conf->{hugepages} eq 'any') {
 487
 488         # try to use 1GB if available && memory size is matching
 489         if (hugepages_chunk_size_supported(1024) && ($size & 1023) == 0) {
 490             return 1024;
 491         } elsif (hugepages_chunk_size_supported(2)) {
 492             return 2;
 493         } else {
 494             die "host only supports 1024 GB hugepages, but requested size '$size' is not a multiple of 1024 MB\n"
 495         }
 496     } else {
 497
 498         my $hugepagesize = $conf->{hugepages};
 499
 500         if (!hugepages_chunk_size_supported($hugepagesize)) {
 501             die "your system doesn't support hugepages of $hugepagesize MB\n";
 502         } elsif (($size % $hugepagesize) != 0) {
 503             die "Memory size $size is not a multiple of the requested hugepages size $hugepagesize\n";
 504         }
 505
 506         return $hugepagesize
 507     }
 508 }
 509
 510 sub hugepages_topology {
 511     my ($conf, $hotplug) = @_;
 512
 513     my $hugepages_topology = {};
 514
 515     return if !$conf->{numa};
 516
 517     my $defaults = PVE::QemuServer::load_defaults();
 518     my $memory = $conf->{memory} || $defaults->{memory};
 519     my $static_memory = 0;
 520     my $sockets = $conf->{sockets} || 1;
 521     my $numa_custom_topology = undef;
 522
 523     if ($hotplug) {
 524         $static_memory = $STATICMEM;
 525         $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 526     } else {
 527         $static_memory = $memory;
 528     }
 529
 530     #custom numa topology
 531     for (my $i = 0; $i < $MAX_NUMA; $i++) {
 532         next if !$conf->{"numa$i"};
 533         my $numa = parse_numa($conf->{"numa$i"});
 534         next if !$numa;
 535
 536         $numa_custom_topology = 1;
 537         my $numa_memory = $numa->{memory};
 538         my $hostnodelists = $numa->{hostnodes};
 539         my $hostnodes = print_numa_hostnodes($hostnodelists);
 540
 541         die "more than 1 hostnode value in numa node is not supported when hugepages are enabled" if $hostnodes !~ m/^(\d)$/;
 542         my $hugepages_size = hugepages_size($conf, $numa_memory);
 543         $hugepages_topology->{$hugepages_size}->{$hostnodes} += hugepages_nr($numa_memory, $hugepages_size);
 544
 545     }
 546
 547     #if no custom numa tology, we split memory and cores across numa nodes
 548     if(!$numa_custom_topology) {
 549
 550         my $numa_memory = ($static_memory / $sockets);
 551
 552         for (my $i = 0; $i < $sockets; $i++)  {
 553
 554             my $hugepages_size = hugepages_size($conf, $numa_memory);
 555             $hugepages_topology->{$hugepages_size}->{$i} += hugepages_nr($numa_memory, $hugepages_size);
 556         }
 557     }
 558
 559     if ($hotplug) {
 560         my $numa_hostmap = get_numa_guest_to_host_map($conf);
 561
 562         foreach_dimm($conf, undef, $memory, $static_memory, sub {
 563             my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 564
 565             $numanode = $numa_hostmap->{$numanode};
 566
 567             my $hugepages_size = hugepages_size($conf, $dimm_size);
 568             $hugepages_topology->{$hugepages_size}->{$numanode} += hugepages_nr($dimm_size, $hugepages_size);
 569         });
 570     }
 571
 572     return $hugepages_topology;
 573 }
 574
 575 sub hugepages_host_topology {
 576
 577     #read host hugepages
 578     my $hugepages_host_topology = {};
 579
 580     dir_glob_foreach("/sys/devices/system/node/", 'node(\d+)', sub {
 581         my ($nodepath, $numanode) = @_;
 582
 583         dir_glob_foreach("/sys/devices/system/node/$nodepath/hugepages/", 'hugepages\-(\d+)kB', sub {
 584             my ($hugepages_path, $hugepages_size) = @_;
 585
 586             $hugepages_size = $hugepages_size / 1024;
 587             my $hugepages_nr = PVE::Tools::file_read_firstline("/sys/devices/system/node/$nodepath/hugepages/$hugepages_path/nr_hugepages");
 588             $hugepages_host_topology->{$hugepages_size}->{$numanode} = $hugepages_nr;
 589         });
 590     });
 591
 592     return $hugepages_host_topology;
 593 }
 594
 595 sub hugepages_allocate {
 596     my ($hugepages_topology, $hugepages_host_topology) = @_;
 597
 598     #allocate new hupages if needed
 599     foreach my $size (sort keys %$hugepages_topology) {
 600
 601         my $nodes = $hugepages_topology->{$size};
 602
 603         foreach my $numanode (keys %$nodes) {
 604
 605             my $hugepages_size = $size * 1024;
 606             my $hugepages_requested = $hugepages_topology->{$size}->{$numanode};
 607             my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/";
 608             my $hugepages_free = PVE::Tools::file_read_firstline($path."free_hugepages");
 609             my $hugepages_nr = PVE::Tools::file_read_firstline($path."nr_hugepages");
 610
 611             if ($hugepages_requested > $hugepages_free) {
 612                 my $hugepages_needed = $hugepages_requested - $hugepages_free;
 613                 PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr + $hugepages_needed);
 614                 #verify that is correctly allocated
 615                 $hugepages_free = PVE::Tools::file_read_firstline($path."free_hugepages");
 616                 if ($hugepages_free < $hugepages_requested) {
 617                     #rollback to initial host config
 618                     hugepages_reset($hugepages_host_topology);
 619                     die "hugepage allocation failed";
 620                 }
 621             }
 622
 623         }
 624     }
 625
 626 }
 627
 628 sub hugepages_default_nr_hugepages {
 629     my ($size) = @_;
 630
 631     my $cmdline = PVE::Tools::file_read_firstline("/proc/cmdline");
 632     my $args = PVE::Tools::split_args($cmdline);
 633
 634     my $parsed_size = 2; # default is 2M
 635
 636     foreach my $arg (@$args) {
 637         if ($arg eq "hugepagesz=2M") {
 638             $parsed_size = 2;
 639         } elsif ($arg eq "hugepagesz=1G") {
 640             $parsed_size = 1024;
 641         } elsif ($arg =~ m/^hugepages=(\d+)?$/) {
 642             if ($parsed_size == $size) {
 643                 return $1;
 644             }
 645         }
 646     }
 647
 648     return 0;
 649 }
 650
 651 sub hugepages_pre_deallocate {
 652     my ($hugepages_topology) = @_;
 653
 654     foreach my $size (sort keys %$hugepages_topology) {
 655
 656         my $hugepages_size = $size * 1024;
 657         my $path = "/sys/kernel/mm/hugepages/hugepages-${hugepages_size}kB/";
 658         my $hugepages_nr = hugepages_default_nr_hugepages($size);
 659         PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr);
 660     }
 661 }
 662
 663 sub hugepages_reset {
 664     my ($hugepages_topology) = @_;
 665
 666     foreach my $size (sort keys %$hugepages_topology) {
 667
 668         my $nodes = $hugepages_topology->{$size};
 669         foreach my $numanode (keys %$nodes) {
 670
 671             my $hugepages_nr = $hugepages_topology->{$size}->{$numanode};
 672             my $hugepages_size = $size * 1024;
 673             my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/";
 674
 675             PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr);
 676         }
 677     }
 678 }
 679
 680 sub hugepages_update_locked {
 681     my ($code, @param) = @_;
 682
 683     my $timeout = 60; #could be long if a lot of hugepages need to be alocated
 684
 685     my $lock_filename = "/var/lock/hugepages.lck";
 686
 687     my $res = lock_file($lock_filename, $timeout, $code, @param);
 688     die $@ if $@;
 689
 690     return $res;
 691 }
 692 1;
 693