PVE/QemuServer/Memory.pm

   1 package PVE::QemuServer::Memory;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
   7 use PVE::Exception qw(raise raise_param_exc);
   8
   9 use PVE::QemuServer;
  10 use PVE::QemuServer::Monitor qw(mon_cmd);
  11
  12 my $MAX_NUMA = 8;
  13 my $STATICMEM = 1024;
  14
  15 my $_host_bits;
  16 sub get_host_phys_address_bits {
  17     return $_host_bits if defined($_host_bits);
  18
  19     my $fh = IO::File->new ('/proc/cpuinfo', "r") or return;
  20     while (defined(my $line = <$fh>)) {
  21         # hopefully we never need to care about mixed (big.LITTLE) archs
  22         if ($line =~ m/^address sizes\s*:\s*(\d+)\s*bits physical/i) {
  23             $_host_bits = int($1);
  24             $fh->close();
  25             return $_host_bits;
  26         }
  27     }
  28     $fh->close();
  29     return; # undef, cannot really do anything..
  30 }
  31
  32 my sub get_max_mem {
  33     my ($conf) = @_;
  34
  35     my $cpu = {};
  36     if (my $cpu_prop_str = $conf->{cpu}) {
  37         $cpu = PVE::JSONSchema::parse_property_string('pve-vm-cpu-conf', $cpu_prop_str)
  38             or die "Cannot parse cpu description: $cpu_prop_str\n";
  39     }
  40     my $bits;
  41     if (my $phys_bits = $cpu->{'phys-bits'}) {
  42         if ($phys_bits eq 'host') {
  43             $bits = get_host_phys_address_bits();
  44         } elsif ($phys_bits =~ /^(\d+)$/) {
  45             $bits = int($phys_bits);
  46         }
  47     }
  48
  49     if (!defined($bits)) {
  50         my $host_bits = get_host_phys_address_bits() // 36; # fixme: what fallback?
  51         if ($cpu->{cputype} && $cpu->{cputype} =~ /^(host|max)$/) {
  52             $bits = $host_bits;
  53         } else {
  54             $bits = $host_bits > 40 ? 40 : $host_bits; # take the smaller one
  55         }
  56     }
  57
  58     $bits = $bits & ~1; # round down to nearest even as limit is lower with odd bit sizes
  59
  60     # heuristic: remove 20 bits to get MB and half that as QEMU needs some overhead
  61     my $bits_to_max_mem = int(1<<($bits - 21));
  62
  63     return $bits_to_max_mem > 4*1024*1024 ? 4*1024*1024 : $bits_to_max_mem;
  64 }
  65
  66 sub get_numa_node_list {
  67     my ($conf) = @_;
  68     my @numa_map;
  69     for (my $i = 0; $i < $MAX_NUMA; $i++) {
  70         my $entry = $conf->{"numa$i"} or next;
  71         my $numa = PVE::QemuServer::parse_numa($entry) or next;
  72         push @numa_map, $i;
  73     }
  74     return @numa_map if @numa_map;
  75     my $sockets = $conf->{sockets} || 1;
  76     return (0..($sockets-1));
  77 }
  78
  79 sub host_numanode_exists {
  80     my ($id) = @_;
  81
  82     return -d "/sys/devices/system/node/node$id/";
  83 }
  84
  85 # only valid when numa nodes map to a single host node
  86 sub get_numa_guest_to_host_map {
  87     my ($conf) = @_;
  88     my $map = {};
  89     for (my $i = 0; $i < $MAX_NUMA; $i++) {
  90         my $entry = $conf->{"numa$i"} or next;
  91         my $numa = PVE::QemuServer::parse_numa($entry) or next;
  92         $map->{$i} = print_numa_hostnodes($numa->{hostnodes});
  93     }
  94     return $map if %$map;
  95     my $sockets = $conf->{sockets} || 1;
  96     return {map { $_ => $_ } (0..($sockets-1))};
  97 }
  98
  99 sub foreach_dimm{
 100     my ($conf, $vmid, $memory, $sockets, $func) = @_;
 101
 102     my $dimm_id = 0;
 103     my $current_size = 0;
 104     my $dimm_size = 0;
 105
 106     if($conf->{hugepages} && $conf->{hugepages} == 1024) {
 107         $current_size = 1024 * $sockets;
 108         $dimm_size = 1024;
 109     } else {
 110         $current_size = 1024;
 111         $dimm_size = 512;
 112     }
 113
 114     return if $current_size == $memory;
 115
 116     my @numa_map = get_numa_node_list($conf);
 117
 118     for (my $j = 0; $j < 8; $j++) {
 119         for (my $i = 0; $i < 32; $i++) {
 120             my $name = "dimm${dimm_id}";
 121             $dimm_id++;
 122             my $numanode = $numa_map[$i % @numa_map];
 123             $current_size += $dimm_size;
 124             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
 125             return  $current_size if $current_size >= $memory;
 126         }
 127         $dimm_size *= 2;
 128     }
 129 }
 130
 131 sub foreach_reverse_dimm {
 132     my ($conf, $vmid, $memory, $sockets, $func) = @_;
 133
 134     my $dimm_id = 253;
 135     my $current_size = 0;
 136     my $dimm_size = 0;
 137
 138     if($conf->{hugepages} && $conf->{hugepages} == 1024) {
 139         $current_size = 8355840;
 140         $dimm_size = 131072;
 141     } else {
 142         $current_size = 4177920;
 143         $dimm_size = 65536;
 144     }
 145
 146     return if $current_size == $memory;
 147
 148     my @numa_map = get_numa_node_list($conf);
 149
 150     for (my $j = 0; $j < 8; $j++) {
 151         for (my $i = 0; $i < 32; $i++) {
 152             my $name = "dimm${dimm_id}";
 153             $dimm_id--;
 154             my $numanode = $numa_map[(31-$i) % @numa_map];
 155             $current_size -= $dimm_size;
 156             &$func($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory);
 157             return  $current_size if $current_size <= $memory;
 158         }
 159         $dimm_size /= 2;
 160     }
 161 }
 162
 163 sub qemu_memory_hotplug {
 164     my ($vmid, $conf, $defaults, $value) = @_;
 165
 166     return $value if !PVE::QemuServer::check_running($vmid);
 167
 168     my $sockets = $conf->{sockets} || 1;
 169
 170     my $memory = $conf->{memory} || $defaults->{memory};
 171     $value = $defaults->{memory} if !$value;
 172     return $value if $value == $memory;
 173
 174     my $static_memory = $STATICMEM;
 175     $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 176
 177     die "memory can't be lower than $static_memory MB" if $value < $static_memory;
 178     my $MAX_MEM = get_max_mem($conf);
 179     die "you cannot add more memory than max mem $MAX_MEM MB!\n" if $value > $MAX_MEM;
 180
 181     if ($value > $memory) {
 182
 183         my $numa_hostmap;
 184
 185         foreach_dimm($conf, $vmid, $value, $sockets, sub {
 186             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 187
 188                 return if $current_size <= $conf->{memory};
 189
 190                 if ($conf->{hugepages}) {
 191                     $numa_hostmap = get_numa_guest_to_host_map($conf) if !$numa_hostmap;
 192
 193                     my $hugepages_size = hugepages_size($conf, $dimm_size);
 194                     my $path = hugepages_mount_path($hugepages_size);
 195                     my $host_numanode = $numa_hostmap->{$numanode};
 196                     my $hugepages_topology->{$hugepages_size}->{$host_numanode} = hugepages_nr($dimm_size, $hugepages_size);
 197
 198                     my $code = sub {
 199                         my $hugepages_host_topology = hugepages_host_topology();
 200                         hugepages_allocate($hugepages_topology, $hugepages_host_topology);
 201
 202                         eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-file", id => "mem-$name", size => int($dimm_size*1024*1024), 'mem-path' => $path, share => JSON::true, prealloc => JSON::true ) };
 203                         if (my $err = $@) {
 204                             hugepages_reset($hugepages_host_topology);
 205                             die $err;
 206                         }
 207
 208                         hugepages_pre_deallocate($hugepages_topology);
 209                     };
 210                     eval { hugepages_update_locked($code); };
 211
 212                 } else {
 213                     eval { mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", size => int($dimm_size*1024*1024) ) };
 214                 }
 215
 216                 if (my $err = $@) {
 217                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 218                     die $err;
 219                 }
 220
 221                 eval { mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
 222                 if (my $err = $@) {
 223                     eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 224                     die $err;
 225                 }
 226                 #update conf after each succesful module hotplug
 227                 $conf->{memory} = $current_size;
 228                 PVE::QemuConfig->write_config($vmid, $conf);
 229         });
 230
 231     } else {
 232
 233         foreach_reverse_dimm($conf, $vmid, $value, $sockets, sub {
 234             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 235
 236                 return if $current_size >= $conf->{memory};
 237                 print "try to unplug memory dimm $name\n";
 238
 239                 my $retry = 0;
 240                 while (1) {
 241                     eval { PVE::QemuServer::qemu_devicedel($vmid, $name) };
 242                     sleep 3;
 243                     my $dimm_list = qemu_memdevices_list($vmid, 'dimm');
 244                     last if !$dimm_list->{$name};
 245                     raise_param_exc({ $name => "error unplug memory module" }) if $retry > 5;
 246                     $retry++;
 247                 }
 248
 249                 #update conf after each succesful module unplug
 250                 $conf->{memory} = $current_size;
 251
 252                 eval { PVE::QemuServer::qemu_objectdel($vmid, "mem-$name"); };
 253                 PVE::QemuConfig->write_config($vmid, $conf);
 254         });
 255     }
 256 }
 257
 258 sub qemu_memdevices_list {
 259     my ($vmid, $type) = @_;
 260
 261     my $dimmarray = mon_cmd($vmid, "query-memory-devices");
 262     my $dimms = {};
 263
 264     foreach my $dimm (@$dimmarray) {
 265         next if $type && $dimm->{data}->{id} !~ /^$type(\d+)$/;
 266         $dimms->{$dimm->{data}->{id}}->{id} = $dimm->{data}->{id};
 267         $dimms->{$dimm->{data}->{id}}->{node} = $dimm->{data}->{node};
 268         $dimms->{$dimm->{data}->{id}}->{addr} = $dimm->{data}->{addr};
 269         $dimms->{$dimm->{data}->{id}}->{size} = $dimm->{data}->{size};
 270         $dimms->{$dimm->{data}->{id}}->{slot} = $dimm->{data}->{slot};
 271     }
 272     return $dimms;
 273 }
 274
 275 sub config {
 276     my ($conf, $vmid, $sockets, $cores, $defaults, $hotplug, $cmd) = @_;
 277
 278     my $memory = $conf->{memory} || $defaults->{memory};
 279     my $static_memory = 0;
 280
 281     if ($hotplug) {
 282         die "NUMA needs to be enabled for memory hotplug\n" if !$conf->{numa};
 283         my $MAX_MEM = get_max_mem($conf);
 284         die "Total memory is bigger than ${MAX_MEM}MB\n" if $memory > $MAX_MEM;
 285
 286         for (my $i = 0; $i < $MAX_NUMA; $i++) {
 287             die "cannot enable memory hotplugging with custom NUMA topology\n"
 288                 if $conf->{"numa$i"};
 289         }
 290
 291         my $sockets = $conf->{sockets} || 1;
 292
 293         $static_memory = $STATICMEM;
 294         $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 295
 296         die "minimum memory must be ${static_memory}MB\n" if($memory < $static_memory);
 297         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
 298
 299     } else {
 300
 301         $static_memory = $memory;
 302         push @$cmd, '-m', $static_memory;
 303     }
 304
 305     die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa};
 306
 307     if ($conf->{numa}) {
 308
 309         my $numa_totalmemory = undef;
 310         for (my $i = 0; $i < $MAX_NUMA; $i++) {
 311             next if !$conf->{"numa$i"};
 312             my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
 313             next if !$numa;
 314             # memory
 315             die "missing NUMA node$i memory value\n" if !$numa->{memory};
 316             my $numa_memory = $numa->{memory};
 317             $numa_totalmemory += $numa_memory;
 318
 319             my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
 320
 321             # cpus
 322             my $cpulists = $numa->{cpus};
 323             die "missing NUMA node$i cpus\n" if !defined($cpulists);
 324             my $cpus = join(',cpus=', map {
 325                 my ($start, $end) = @$_;
 326                 defined($end) ? "$start-$end" : $start
 327             } @$cpulists);
 328
 329             # hostnodes
 330             my $hostnodelists = $numa->{hostnodes};
 331             if (defined($hostnodelists)) {
 332
 333                 my $hostnodes = print_numa_hostnodes($hostnodelists);
 334
 335                 # policy
 336                 my $policy = $numa->{policy};
 337                 die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
 338                 $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
 339             } else {
 340                 die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
 341             }
 342
 343             push @$cmd, '-object', $mem_object;
 344             push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 345         }
 346
 347         die "total memory for NUMA nodes must be equal to vm static memory\n"
 348             if $numa_totalmemory && $numa_totalmemory != $static_memory;
 349
 350         #if no custom tology, we split memory and cores across numa nodes
 351         if (!$numa_totalmemory) {
 352             my $numa_memory = ($static_memory / $sockets);
 353
 354             for (my $i = 0; $i < $sockets; $i++)  {
 355                 die "host NUMA node$i doesn't exist\n"
 356                     if !host_numanode_exists($i) && $conf->{hugepages};
 357
 358                 my $mem_object = print_mem_object($conf, "ram-node$i", $numa_memory);
 359                 push @$cmd, '-object', $mem_object;
 360
 361                 my $cpus = ($cores * $i);
 362                 $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1;
 363
 364                 push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 365             }
 366         }
 367     }
 368
 369     if ($hotplug) {
 370         foreach_dimm($conf, $vmid, $memory, $sockets, sub {
 371             my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 372
 373             my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size);
 374
 375             push @$cmd, "-object" , $mem_object;
 376             push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
 377
 378             die "memory size ($memory) must be aligned to $dimm_size for hotplugging\n"
 379                 if $current_size > $memory;
 380         });
 381     }
 382 }
 383
 384 sub print_mem_object {
 385     my ($conf, $id, $size) = @_;
 386
 387     if ($conf->{hugepages}) {
 388
 389         my $hugepages_size = hugepages_size($conf, $size);
 390         my $path = hugepages_mount_path($hugepages_size);
 391
 392         return "memory-backend-file,id=$id,size=${size}M,mem-path=$path,share=on,prealloc=yes";
 393     } else {
 394         return "memory-backend-ram,id=$id,size=${size}M";
 395     }
 396
 397 }
 398
 399 sub print_numa_hostnodes {
 400     my ($hostnodelists) = @_;
 401
 402     my $hostnodes;
 403     foreach my $hostnoderange (@$hostnodelists) {
 404         my ($start, $end) = @$hostnoderange;
 405         $hostnodes .= ',' if $hostnodes;
 406         $hostnodes .= $start;
 407         $hostnodes .= "-$end" if defined($end);
 408         $end //= $start;
 409         for (my $i = $start; $i <= $end; ++$i ) {
 410             die "host NUMA node$i doesn't exist\n" if !host_numanode_exists($i);
 411         }
 412     }
 413     return $hostnodes;
 414 }
 415
 416 sub hugepages_mount {
 417
 418    my $mountdata = PVE::ProcFSTools::parse_proc_mounts();
 419
 420    foreach my $size (qw(2048 1048576)) {
 421         next if (! -d "/sys/kernel/mm/hugepages/hugepages-${size}kB");
 422
 423         my $path = "/run/hugepages/kvm/${size}kB";
 424
 425         my $found = grep {
 426             $_->[2] =~ /^hugetlbfs/ &&
 427             $_->[1] eq $path
 428         } @$mountdata;
 429
 430         if (!$found) {
 431
 432             File::Path::make_path($path) if (!-d $path);
 433             my $cmd = ['/bin/mount', '-t', 'hugetlbfs', '-o', "pagesize=${size}k", 'hugetlbfs', $path];
 434             run_command($cmd, errmsg => "hugepage mount error");
 435         }
 436    }
 437 }
 438
 439 sub hugepages_mount_path {
 440    my ($size) = @_;
 441
 442    $size = $size * 1024;
 443    return "/run/hugepages/kvm/${size}kB";
 444
 445 }
 446
 447 sub hugepages_nr {
 448   my ($size, $hugepages_size) = @_;
 449
 450   return $size / $hugepages_size;
 451 }
 452
 453 sub hugepages_chunk_size_supported {
 454     my ($size) = @_;
 455
 456     return -d "/sys/kernel/mm/hugepages/hugepages-". ($size * 1024) ."kB";
 457 }
 458
 459 sub hugepages_size {
 460     my ($conf, $size) = @_;
 461     die "hugepages option is not enabled" if !$conf->{hugepages};
 462     die "memory size '$size' is not a positive even integer; cannot use for hugepages\n"
 463         if $size <= 0 || $size & 1;
 464
 465     die "your system doesn't support hugepages\n"
 466         if !hugepages_chunk_size_supported(2) && !hugepages_chunk_size_supported(1024);
 467
 468     if ($conf->{hugepages} eq 'any') {
 469
 470         # try to use 1GB if available && memory size is matching
 471         if (hugepages_chunk_size_supported(1024) && ($size & 1023) == 0) {
 472             return 1024;
 473         } elsif (hugepages_chunk_size_supported(2)) {
 474             return 2;
 475         } else {
 476             die "host only supports 1024 GB hugepages, but requested size '$size' is not a multiple of 1024 MB\n"
 477         }
 478     } else {
 479
 480         my $hugepagesize = $conf->{hugepages};
 481
 482         if (!hugepages_chunk_size_supported($hugepagesize)) {
 483             die "your system doesn't support hugepages of $hugepagesize MB\n";
 484         } elsif (($size % $hugepagesize) != 0) {
 485             die "Memory size $size is not a multiple of the requested hugepages size $hugepagesize\n";
 486         }
 487
 488         return $hugepagesize
 489     }
 490 }
 491
 492 sub hugepages_topology {
 493     my ($conf, $hotplug) = @_;
 494
 495     my $hugepages_topology = {};
 496
 497     return if !$conf->{numa};
 498
 499     my $defaults = PVE::QemuServer::load_defaults();
 500     my $memory = $conf->{memory} || $defaults->{memory};
 501     my $static_memory = 0;
 502     my $sockets = $conf->{sockets} || 1;
 503     my $numa_custom_topology = undef;
 504
 505     if ($hotplug) {
 506         $static_memory = $STATICMEM;
 507         $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 508     } else {
 509         $static_memory = $memory;
 510     }
 511
 512     #custom numa topology
 513     for (my $i = 0; $i < $MAX_NUMA; $i++) {
 514         next if !$conf->{"numa$i"};
 515         my $numa = PVE::QemuServer::parse_numa($conf->{"numa$i"});
 516         next if !$numa;
 517
 518         $numa_custom_topology = 1;
 519         my $numa_memory = $numa->{memory};
 520         my $hostnodelists = $numa->{hostnodes};
 521         my $hostnodes = print_numa_hostnodes($hostnodelists);
 522
 523         die "more than 1 hostnode value in numa node is not supported when hugepages are enabled" if $hostnodes !~ m/^(\d)$/;
 524         my $hugepages_size = hugepages_size($conf, $numa_memory);
 525         $hugepages_topology->{$hugepages_size}->{$hostnodes} += hugepages_nr($numa_memory, $hugepages_size);
 526
 527     }
 528
 529     #if no custom numa tology, we split memory and cores across numa nodes
 530     if(!$numa_custom_topology) {
 531
 532         my $numa_memory = ($static_memory / $sockets);
 533
 534         for (my $i = 0; $i < $sockets; $i++)  {
 535
 536             my $hugepages_size = hugepages_size($conf, $numa_memory);
 537             $hugepages_topology->{$hugepages_size}->{$i} += hugepages_nr($numa_memory, $hugepages_size);
 538         }
 539     }
 540
 541     if ($hotplug) {
 542         my $numa_hostmap = get_numa_guest_to_host_map($conf);
 543
 544         foreach_dimm($conf, undef, $memory, $sockets, sub {
 545             my ($conf, undef, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
 546
 547             $numanode = $numa_hostmap->{$numanode};
 548
 549             my $hugepages_size = hugepages_size($conf, $dimm_size);
 550             $hugepages_topology->{$hugepages_size}->{$numanode} += hugepages_nr($dimm_size, $hugepages_size);
 551         });
 552     }
 553
 554     return $hugepages_topology;
 555 }
 556
 557 sub hugepages_host_topology {
 558
 559     #read host hugepages
 560     my $hugepages_host_topology = {};
 561
 562     dir_glob_foreach("/sys/devices/system/node/", 'node(\d+)', sub {
 563         my ($nodepath, $numanode) = @_;
 564
 565         dir_glob_foreach("/sys/devices/system/node/$nodepath/hugepages/", 'hugepages\-(\d+)kB', sub {
 566             my ($hugepages_path, $hugepages_size) = @_;
 567
 568             $hugepages_size = $hugepages_size / 1024;
 569             my $hugepages_nr = PVE::Tools::file_read_firstline("/sys/devices/system/node/$nodepath/hugepages/$hugepages_path/nr_hugepages");
 570             $hugepages_host_topology->{$hugepages_size}->{$numanode} = $hugepages_nr;
 571         });
 572     });
 573
 574     return $hugepages_host_topology;
 575 }
 576
 577 sub hugepages_allocate {
 578     my ($hugepages_topology, $hugepages_host_topology) = @_;
 579
 580     #allocate new hupages if needed
 581     foreach my $size (sort keys %$hugepages_topology) {
 582
 583         my $nodes = $hugepages_topology->{$size};
 584
 585         foreach my $numanode (keys %$nodes) {
 586
 587             my $hugepages_size = $size * 1024;
 588             my $hugepages_requested = $hugepages_topology->{$size}->{$numanode};
 589             my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/";
 590             my $hugepages_free = PVE::Tools::file_read_firstline($path."free_hugepages");
 591             my $hugepages_nr = PVE::Tools::file_read_firstline($path."nr_hugepages");
 592
 593             if ($hugepages_requested > $hugepages_free) {
 594                 my $hugepages_needed = $hugepages_requested - $hugepages_free;
 595                 PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr + $hugepages_needed);
 596                 #verify that is correctly allocated
 597                 $hugepages_free = PVE::Tools::file_read_firstline($path."free_hugepages");
 598                 if ($hugepages_free < $hugepages_requested) {
 599                     #rollback to initial host config
 600                     hugepages_reset($hugepages_host_topology);
 601                     die "hugepage allocation failed";
 602                 }
 603             }
 604
 605         }
 606     }
 607
 608 }
 609
 610 sub hugepages_default_nr_hugepages {
 611     my ($size) = @_;
 612
 613     my $cmdline = PVE::Tools::file_read_firstline("/proc/cmdline");
 614     my $args = PVE::Tools::split_args($cmdline);
 615
 616     my $parsed_size = 2; # default is 2M
 617
 618     foreach my $arg (@$args) {
 619         if ($arg eq "hugepagesz=2M") {
 620             $parsed_size = 2;
 621         } elsif ($arg eq "hugepagesz=1G") {
 622             $parsed_size = 1024;
 623         } elsif ($arg =~ m/^hugepages=(\d+)?$/) {
 624             if ($parsed_size == $size) {
 625                 return $1;
 626             }
 627         }
 628     }
 629
 630     return 0;
 631 }
 632
 633 sub hugepages_pre_deallocate {
 634     my ($hugepages_topology) = @_;
 635
 636     foreach my $size (sort keys %$hugepages_topology) {
 637
 638         my $hugepages_size = $size * 1024;
 639         my $path = "/sys/kernel/mm/hugepages/hugepages-${hugepages_size}kB/";
 640         my $hugepages_nr = hugepages_default_nr_hugepages($size);
 641         PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr);
 642     }
 643 }
 644
 645 sub hugepages_reset {
 646     my ($hugepages_topology) = @_;
 647
 648     foreach my $size (sort keys %$hugepages_topology) {
 649
 650         my $nodes = $hugepages_topology->{$size};
 651         foreach my $numanode (keys %$nodes) {
 652
 653             my $hugepages_nr = $hugepages_topology->{$size}->{$numanode};
 654             my $hugepages_size = $size * 1024;
 655             my $path = "/sys/devices/system/node/node${numanode}/hugepages/hugepages-${hugepages_size}kB/";
 656
 657             PVE::ProcFSTools::write_proc_entry($path."nr_hugepages", $hugepages_nr);
 658         }
 659     }
 660 }
 661
 662 sub hugepages_update_locked {
 663     my ($code, @param) = @_;
 664
 665     my $timeout = 60; #could be long if a lot of hugepages need to be alocated
 666
 667     my $lock_filename = "/var/lock/hugepages.lck";
 668
 669     my $res = lock_file($lock_filename, $timeout, $code, @param);
 670     die $@ if $@;
 671
 672     return $res;
 673 }
 674 1;
 675