PVE/QemuServer/CPUConfig.pm

   1 package PVE::QemuServer::CPUConfig;
   2
   3 use strict;
   4 use warnings;
   5
   6 use PVE::JSONSchema;
   7 use PVE::Cluster qw(cfs_register_file cfs_read_file);
   8 use PVE::QemuServer::Helpers qw(min_version);
   9
  10 use base qw(PVE::SectionConfig Exporter);
  11
  12 our @EXPORT_OK = qw(
  13 print_cpu_device
  14 get_cpu_options
  15 );
  16
  17 # under certain race-conditions, this module might be loaded before pve-cluster
  18 # has started completely, so ensure we don't prevent the FUSE mount with our dir
  19 if (PVE::Cluster::check_cfs_is_mounted(1)) {
  20     mkdir "/etc/pve/virtual-guest";
  21 }
  22
  23 my $default_filename = "virtual-guest/cpu-models.conf";
  24 cfs_register_file($default_filename,
  25                   sub { PVE::QemuServer::CPUConfig->parse_config(@_); },
  26                   sub { PVE::QemuServer::CPUConfig->write_config(@_); });
  27
  28 sub load_custom_model_conf {
  29     return cfs_read_file($default_filename);
  30 }
  31
  32 my $cpu_vendor_list = {
  33     # Intel CPUs
  34     486 => 'GenuineIntel',
  35     pentium => 'GenuineIntel',
  36     pentium2 => 'GenuineIntel',
  37     pentium3 => 'GenuineIntel',
  38     coreduo => 'GenuineIntel',
  39     core2duo => 'GenuineIntel',
  40     Conroe => 'GenuineIntel',
  41     Penryn => 'GenuineIntel',
  42     Nehalem => 'GenuineIntel',
  43     'Nehalem-IBRS' => 'GenuineIntel',
  44     Westmere => 'GenuineIntel',
  45     'Westmere-IBRS' => 'GenuineIntel',
  46     SandyBridge => 'GenuineIntel',
  47     'SandyBridge-IBRS' => 'GenuineIntel',
  48     IvyBridge => 'GenuineIntel',
  49     'IvyBridge-IBRS' => 'GenuineIntel',
  50     Haswell => 'GenuineIntel',
  51     'Haswell-IBRS' => 'GenuineIntel',
  52     'Haswell-noTSX' => 'GenuineIntel',
  53     'Haswell-noTSX-IBRS' => 'GenuineIntel',
  54     Broadwell => 'GenuineIntel',
  55     'Broadwell-IBRS' => 'GenuineIntel',
  56     'Broadwell-noTSX' => 'GenuineIntel',
  57     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
  58     'Skylake-Client' => 'GenuineIntel',
  59     'Skylake-Client-IBRS' => 'GenuineIntel',
  60     'Skylake-Client-noTSX-IBRS' => 'GenuineIntel',
  61     'Skylake-Server' => 'GenuineIntel',
  62     'Skylake-Server-IBRS' => 'GenuineIntel',
  63     'Skylake-Server-noTSX-IBRS' => 'GenuineIntel',
  64     'Cascadelake-Server' => 'GenuineIntel',
  65     'Cascadelake-Server-noTSX' => 'GenuineIntel',
  66     KnightsMill => 'GenuineIntel',
  67     'Icelake-Client' => 'GenuineIntel',
  68     'Icelake-Client-noTSX' => 'GenuineIntel',
  69     'Icelake-Server' => 'GenuineIntel',
  70     'Icelake-Server-noTSX' => 'GenuineIntel',
  71
  72     # AMD CPUs
  73     athlon => 'AuthenticAMD',
  74     phenom => 'AuthenticAMD',
  75     Opteron_G1 => 'AuthenticAMD',
  76     Opteron_G2 => 'AuthenticAMD',
  77     Opteron_G3 => 'AuthenticAMD',
  78     Opteron_G4 => 'AuthenticAMD',
  79     Opteron_G5 => 'AuthenticAMD',
  80     EPYC => 'AuthenticAMD',
  81     'EPYC-IBPB' => 'AuthenticAMD',
  82
  83     # generic types, use vendor from host node
  84     host => 'default',
  85     kvm32 => 'default',
  86     kvm64 => 'default',
  87     qemu32 => 'default',
  88     qemu64 => 'default',
  89     max => 'default',
  90 };
  91
  92 my @supported_cpu_flags = (
  93     'pcid',
  94     'spec-ctrl',
  95     'ibpb',
  96     'ssbd',
  97     'virt-ssbd',
  98     'amd-ssbd',
  99     'amd-no-ssb',
 100     'pdpe1gb',
 101     'md-clear',
 102     'hv-tlbflush',
 103     'hv-evmcs',
 104     'aes'
 105 );
 106 my $cpu_flag_supported_re = qr/([+-])(@{[join('|', @supported_cpu_flags)]})/;
 107 my $cpu_flag_any_re = qr/([+-])([a-zA-Z0-9\-_\.]+)/;
 108
 109 our $qemu_cmdline_cpu_re = qr/^((?>[+-]?[\w\-_=]+,?)+)$/;
 110
 111 my $cpu_fmt = {
 112     cputype => {
 113         description => "Emulated CPU type. Can be default or custom name (custom model names must be prefixed with 'custom-').",
 114         type => 'string',
 115         format_description => 'string',
 116         default => 'kvm64',
 117         default_key => 1,
 118         optional => 1,
 119     },
 120     'reported-model' => {
 121         description => "CPU model and vendor to report to the guest. Must be a QEMU/KVM supported model."
 122                      . " Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.",
 123         type => 'string',
 124         enum => [ sort { lc("$a") cmp lc("$b") } keys %$cpu_vendor_list ],
 125         default => 'kvm64',
 126         optional => 1,
 127     },
 128     hidden => {
 129         description => "Do not identify as a KVM virtual machine.",
 130         type => 'boolean',
 131         optional => 1,
 132         default => 0
 133     },
 134     'hv-vendor-id' => {
 135         type => 'string',
 136         pattern => qr/[a-zA-Z0-9]{1,12}/,
 137         format_description => 'vendor-id',
 138         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 139         optional => 1,
 140     },
 141     flags => {
 142         description => "List of additional CPU flags separated by ';'."
 143                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 144                      . " Custom CPU models can specify any flag supported by"
 145                      . " QEMU/KVM, VM-specific flags must be from the following"
 146                      . " set for security reasons: @{[join(', ', @supported_cpu_flags)]}.",
 147         format_description => '+FLAG[;-FLAG...]',
 148         type => 'string',
 149         pattern => qr/$cpu_flag_any_re(;$cpu_flag_any_re)*/,
 150         optional => 1,
 151     },
 152 };
 153
 154 # $cpu_fmt describes both the CPU config passed as part of a VM config, as well
 155 # as the definition of a custom CPU model. There are some slight differences
 156 # though, which we catch in the custom verification function below.
 157 PVE::JSONSchema::register_format('pve-cpu-conf', \&parse_cpu_conf_basic);
 158 sub parse_cpu_conf_basic {
 159     my ($cpu_str, $noerr) = @_;
 160
 161     my $cpu = eval { PVE::JSONSchema::parse_property_string($cpu_fmt, $cpu_str) };
 162     if ($@) {
 163         die $@ if !$noerr;
 164         return undef;
 165     }
 166
 167     # required, but can't be forced in schema since it's encoded in section
 168     # header for custom models
 169     if (!$cpu->{cputype}) {
 170         die "CPU is missing cputype\n" if !$noerr;
 171         return undef;
 172     }
 173
 174     return $cpu;
 175 }
 176
 177 PVE::JSONSchema::register_format('pve-vm-cpu-conf', \&parse_vm_cpu_conf);
 178 sub parse_vm_cpu_conf {
 179     my ($cpu_str, $noerr) = @_;
 180
 181     my $cpu = parse_cpu_conf_basic($cpu_str, $noerr);
 182     return undef if !$cpu;
 183
 184     my $cputype = $cpu->{cputype};
 185
 186     # a VM-specific config is only valid if the cputype exists
 187     if (is_custom_model($cputype)) {
 188         eval { get_custom_model($cputype); };
 189         if ($@) {
 190             die $@ if !$noerr;
 191             return undef;
 192         }
 193     } else {
 194         if (!defined($cpu_vendor_list->{$cputype})) {
 195             die "Built-in cputype '$cputype' is not defined (missing 'custom-' prefix?)\n" if !$noerr;
 196             return undef;
 197         }
 198     }
 199
 200     # in a VM-specific config, certain properties are limited/forbidden
 201
 202     if ($cpu->{flags} && $cpu->{flags} !~ m/$cpu_flag_supported_re(;$cpu_flag_supported_re)*/) {
 203         die "VM-specific CPU flags must be a subset of: @{[join(', ', @supported_cpu_flags)]}\n"
 204             if !$noerr;
 205         return undef;
 206     }
 207
 208     die "Property 'reported-model' not allowed in VM-specific CPU config.\n"
 209         if defined($cpu->{'reported-model'});
 210
 211     return $cpu;
 212 }
 213
 214 # Section config settings
 215 my $defaultData = {
 216     # shallow copy, since SectionConfig modifies propertyList internally
 217     propertyList => { %$cpu_fmt },
 218 };
 219
 220 sub private {
 221     return $defaultData;
 222 }
 223
 224 sub options {
 225     return { %$cpu_fmt };
 226 }
 227
 228 sub type {
 229     return 'cpu-model';
 230 }
 231
 232 sub parse_section_header {
 233     my ($class, $line) = @_;
 234
 235     my ($type, $sectionId, $errmsg, $config) =
 236         $class->SUPER::parse_section_header($line);
 237
 238     return undef if !$type;
 239     return ($type, $sectionId, $errmsg, {
 240         # name is given by section header, and we can always prepend 'custom-'
 241         # since we're reading the custom CPU file
 242         cputype => "custom-$sectionId",
 243     });
 244 }
 245
 246 sub write_config {
 247     my ($class, $filename, $cfg) = @_;
 248
 249     mkdir "/etc/pve/virtual-guest";
 250
 251     for my $model (keys %{$cfg->{ids}}) {
 252         my $model_conf = $cfg->{ids}->{$model};
 253
 254         die "internal error: tried saving built-in CPU model (or missing prefix): $model_conf->{cputype}\n"
 255             if !is_custom_model($model_conf->{cputype});
 256
 257         die "internal error: tried saving custom cpumodel with cputype (ignoring prefix: $model_conf->{cputype}) not equal to \$cfg->ids entry ($model)\n"
 258             if "custom-$model" ne $model_conf->{cputype};
 259
 260         # saved in section header
 261         delete $model_conf->{cputype};
 262     }
 263
 264     $class->SUPER::write_config($filename, $cfg);
 265 }
 266
 267 sub is_custom_model {
 268     my ($cputype) = @_;
 269     return $cputype =~ m/^custom-/;
 270 }
 271
 272 # Use this to get a single model in the format described by $cpu_fmt.
 273 # Allows names with and without custom- prefix.
 274 sub get_custom_model {
 275     my ($name, $noerr) = @_;
 276
 277     $name =~ s/^custom-//;
 278     my $conf = load_custom_model_conf();
 279
 280     my $entry = $conf->{ids}->{$name};
 281     if (!defined($entry)) {
 282         die "Custom cputype '$name' not found\n" if !$noerr;
 283         return undef;
 284     }
 285
 286     my $model = {};
 287     for my $property (keys %$cpu_fmt) {
 288         if (my $value = $entry->{$property}) {
 289             $model->{$property} = $value;
 290         }
 291     }
 292
 293     return $model;
 294 }
 295
 296 # Print a QEMU device node for a given VM configuration for hotplugging CPUs
 297 sub print_cpu_device {
 298     my ($conf, $id) = @_;
 299
 300     my $kvm = $conf->{kvm} // 1;
 301     my $cpu = $kvm ? "kvm64" : "qemu64";
 302     if (my $cputype = $conf->{cpu}) {
 303         my $cpuconf = parse_cpu_conf_basic($cputype)
 304             or die "Cannot parse cpu description: $cputype\n";
 305         $cpu = $cpuconf->{cputype};
 306
 307         if (is_custom_model($cpu)) {
 308             my $custom_cpu = get_custom_model($cpu);
 309
 310             $cpu = $custom_cpu->{'reported-model'} //
 311                 $cpu_fmt->{'reported-model'}->{default};
 312         }
 313     }
 314
 315     my $cores = $conf->{cores} || 1;
 316
 317     my $current_core = ($id - 1) % $cores;
 318     my $current_socket = int(($id - 1 - $current_core)/$cores);
 319
 320     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
 321 }
 322
 323 # Resolves multiple arrays of hashes representing CPU flags with metadata to a
 324 # single string in QEMU "-cpu" compatible format. Later arrays have higher
 325 # priority.
 326 #
 327 # Hashes take the following format:
 328 # {
 329 #     aes => {
 330 #         op => "+", # defaults to "" if undefined
 331 #         reason => "to support AES acceleration", # for override warnings
 332 #         value => "" # needed for kvm=off (value: off) etc...
 333 #     },
 334 #     ...
 335 # }
 336 sub resolve_cpu_flags {
 337     my $flags = {};
 338
 339     for my $hash (@_) {
 340         for my $flag_name (keys %$hash) {
 341             my $flag = $hash->{$flag_name};
 342             my $old_flag = $flags->{$flag_name};
 343
 344             $flag->{op} //= "";
 345             $flag->{reason} //= "unknown origin";
 346
 347             if ($old_flag) {
 348                 my $value_changed = (defined($flag->{value}) != defined($old_flag->{value})) ||
 349                                     (defined($flag->{value}) && $flag->{value} ne $old_flag->{value});
 350
 351                 if ($old_flag->{op} eq $flag->{op} && !$value_changed) {
 352                     $flags->{$flag_name}->{reason} .= " & $flag->{reason}";
 353                     next;
 354                 }
 355
 356                 my $old = print_cpuflag_hash($flag_name, $flags->{$flag_name});
 357                 my $new = print_cpuflag_hash($flag_name, $flag);
 358                 warn "warning: CPU flag/setting $new overwrites $old\n";
 359             }
 360
 361             $flags->{$flag_name} = $flag;
 362         }
 363     }
 364
 365     my $flag_str = '';
 366     # sort for command line stability
 367     for my $flag_name (sort keys %$flags) {
 368         $flag_str .= ',';
 369         $flag_str .= $flags->{$flag_name}->{op};
 370         $flag_str .= $flag_name;
 371         $flag_str .= "=$flags->{$flag_name}->{value}"
 372             if $flags->{$flag_name}->{value};
 373     }
 374
 375     return $flag_str;
 376 }
 377
 378 sub print_cpuflag_hash {
 379     my ($flag_name, $flag) = @_;
 380     my $formatted = "'$flag->{op}$flag_name";
 381     $formatted .= "=$flag->{value}" if defined($flag->{value});
 382     $formatted .= "'";
 383     $formatted .= " ($flag->{reason})" if defined($flag->{reason});
 384     return $formatted;
 385 }
 386
 387 sub parse_cpuflag_list {
 388     my ($re, $reason, $flaglist) = @_;
 389
 390     my $res = {};
 391     return $res if !$flaglist;
 392
 393     foreach my $flag (split(";", $flaglist)) {
 394         if ($flag =~ $re) {
 395             $res->{$2} = { op => $1, reason => $reason };
 396         }
 397     }
 398
 399     return $res;
 400 }
 401
 402 # Calculate QEMU's '-cpu' argument from a given VM configuration
 403 sub get_cpu_options {
 404     my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
 405
 406     my $cputype = $kvm ? "kvm64" : "qemu64";
 407     if ($arch eq 'aarch64') {
 408         $cputype = 'cortex-a57';
 409     }
 410
 411     my $cpu = {};
 412     my $custom_cpu;
 413     my $hv_vendor_id;
 414     if (my $cpu_prop_str = $conf->{cpu}) {
 415         $cpu = parse_vm_cpu_conf($cpu_prop_str)
 416             or die "Cannot parse cpu description: $cpu_prop_str\n";
 417
 418         $cputype = $cpu->{cputype};
 419
 420         if (is_custom_model($cputype)) {
 421             $custom_cpu = get_custom_model($cputype);
 422
 423             $cputype = $custom_cpu->{'reported-model'} //
 424                 $cpu_fmt->{'reported-model'}->{default};
 425             $kvm_off = $custom_cpu->{hidden}
 426                 if defined($custom_cpu->{hidden});
 427             $hv_vendor_id = $custom_cpu->{'hv-vendor-id'};
 428         }
 429
 430         # VM-specific settings override custom CPU config
 431         $kvm_off = $cpu->{hidden}
 432             if defined($cpu->{hidden});
 433         $hv_vendor_id = $cpu->{'hv-vendor-id'}
 434             if defined($cpu->{'hv-vendor-id'});
 435     }
 436
 437     my $pve_flags = get_pve_cpu_flags($conf, $kvm, $cputype, $arch,
 438                                       $machine_version);
 439
 440     my $hv_flags = get_hyperv_enlightenments($winversion, $machine_version,
 441         $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
 442
 443     my $custom_cputype_flags = parse_cpuflag_list($cpu_flag_any_re,
 444         "set by custom CPU model", $custom_cpu->{flags});
 445
 446     my $vm_flags = parse_cpuflag_list($cpu_flag_supported_re,
 447         "manually set for VM", $cpu->{flags});
 448
 449     my $pve_forced_flags = {};
 450     $pve_forced_flags->{'enforce'} = {
 451         reason => "error if requested CPU settings not available",
 452     } if $cputype ne 'host' && $kvm && $arch eq 'x86_64';
 453     $pve_forced_flags->{'kvm'} = {
 454         value => "off",
 455         reason => "hide KVM virtualization from guest",
 456     } if $kvm_off;
 457
 458     # $cputype is the "reported-model" for custom types, so we can just look up
 459     # the vendor in the default list
 460     my $cpu_vendor = $cpu_vendor_list->{$cputype};
 461     if ($cpu_vendor) {
 462         $pve_forced_flags->{'vendor'} = {
 463             value => $cpu_vendor,
 464         } if $cpu_vendor ne 'default';
 465     } elsif ($arch ne 'aarch64') {
 466         die "internal error"; # should not happen
 467     }
 468
 469     my $cpu_str = $cputype;
 470
 471     # will be resolved in parameter order
 472     $cpu_str .= resolve_cpu_flags($pve_flags, $hv_flags, $custom_cputype_flags,
 473                               $vm_flags, $pve_forced_flags);
 474
 475     return ('-cpu', $cpu_str);
 476 }
 477
 478 # Some hardcoded flags required by certain configurations
 479 sub get_pve_cpu_flags {
 480     my ($conf, $kvm, $cputype, $arch, $machine_version) = @_;
 481
 482     my $pve_flags = {};
 483     my $pve_msg = "set by PVE;";
 484
 485     $pve_flags->{'lahf_lm'} = {
 486         op => '+',
 487         reason => "$pve_msg to support Windows 8.1+",
 488     } if $cputype eq 'kvm64' && $arch eq 'x86_64';
 489
 490     $pve_flags->{'x2apic'} = {
 491         op => '-',
 492         reason => "$pve_msg incompatible with Solaris",
 493     } if $conf->{ostype} && $conf->{ostype} eq 'solaris';
 494
 495     $pve_flags->{'sep'} = {
 496         op => '+',
 497         reason => "$pve_msg to support Windows 8+ and improve Windows XP+",
 498     } if $cputype eq 'kvm64' || $cputype eq 'kvm32';
 499
 500     $pve_flags->{'rdtscp'} = {
 501         op => '-',
 502         reason => "$pve_msg broken on AMD Opteron",
 503     } if $cputype =~ m/^Opteron/;
 504
 505     if (min_version($machine_version, 2, 3) && $kvm && $arch eq 'x86_64') {
 506         $pve_flags->{'kvm_pv_unhalt'} = {
 507             op => '+',
 508             reason => "$pve_msg to improve Linux guest spinlock performance",
 509         };
 510         $pve_flags->{'kvm_pv_eoi'} = {
 511             op => '+',
 512             reason => "$pve_msg to improve Linux guest interrupt performance",
 513         };
 514     }
 515
 516     return $pve_flags;
 517 }
 518
 519 sub get_hyperv_enlightenments {
 520     my ($winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
 521
 522     return if $winversion < 6;
 523     return if $bios && $bios eq 'ovmf' && $winversion < 8;
 524
 525     my $flags = {};
 526     my $default_reason = "automatic Hyper-V enlightenment for Windows";
 527     my $flagfn = sub {
 528         my ($flag, $value, $reason) = @_;
 529         $flags->{$flag} = {
 530             reason => $reason // $default_reason,
 531             value => $value,
 532         }
 533     };
 534
 535     my $hv_vendor_set = defined($hv_vendor_id);
 536     if ($gpu_passthrough || $hv_vendor_set) {
 537         $hv_vendor_id //= 'proxmox';
 538         $flagfn->('hv_vendor_id', $hv_vendor_id, $hv_vendor_set ?
 539             "custom hv_vendor_id set" : "NVIDIA workaround for GPU passthrough");
 540     }
 541
 542     if (min_version($machine_version, 2, 3)) {
 543         $flagfn->('hv_spinlocks', '0x1fff');
 544         $flagfn->('hv_vapic');
 545         $flagfn->('hv_time');
 546     } else {
 547         $flagfn->('hv_spinlocks', '0xffff');
 548     }
 549
 550     if (min_version($machine_version, 2, 6)) {
 551         $flagfn->('hv_reset');
 552         $flagfn->('hv_vpindex');
 553         $flagfn->('hv_runtime');
 554     }
 555
 556     if ($winversion >= 7) {
 557         my $win7_reason = $default_reason . " 7 and higher";
 558         $flagfn->('hv_relaxed', undef, $win7_reason);
 559
 560         if (min_version($machine_version, 2, 12)) {
 561             $flagfn->('hv_synic', undef, $win7_reason);
 562             $flagfn->('hv_stimer', undef, $win7_reason);
 563         }
 564
 565         if (min_version($machine_version, 3, 1)) {
 566             $flagfn->('hv_ipi', undef, $win7_reason);
 567         }
 568     }
 569
 570     return $flags;
 571 }
 572
 573 sub get_cpu_from_running_vm {
 574     my ($pid) = @_;
 575
 576     my $cmdline = PVE::QemuServer::Helpers::parse_cmdline($pid);
 577     die "could not read commandline of running machine\n"
 578         if !$cmdline->{cpu}->{value};
 579
 580     # sanitize and untaint value
 581     $cmdline->{cpu}->{value} =~ $qemu_cmdline_cpu_re;
 582     return $1;
 583 }
 584
 585 __PACKAGE__->register();
 586 __PACKAGE__->init();
 587
 588 1;