1 package PVE
::QemuServer
::CPUConfig
;
7 use PVE
::Cluster
qw(cfs_register_file cfs_read_file);
8 use PVE
::QemuServer
::Helpers
qw(min_version);
10 use base
qw(PVE::SectionConfig Exporter);
17 # under certain race-conditions, this module might be loaded before pve-cluster
18 # has started completely, so ensure we don't prevent the FUSE mount with our dir
19 if (PVE
::Cluster
::check_cfs_is_mounted
(1)) {
20 mkdir "/etc/pve/virtual-guest";
23 my $default_filename = "virtual-guest/cpu-models.conf";
26 sub { PVE
::QemuServer
::CPUConfig-
>parse_config(@_); },
27 sub { PVE
::QemuServer
::CPUConfig-
>write_config(@_); },
30 sub load_custom_model_conf
{
31 return cfs_read_file
($default_filename);
34 #builtin models : reported-model is mandatory
35 my $builtin_models = {
37 'reported-model' => 'qemu64',
38 flags
=> "+popcnt;+pni;+sse4.1;+sse4.2;+ssse3",
41 'reported-model' => 'qemu64',
42 flags
=> "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3",
45 'reported-model' => 'qemu64',
46 flags
=> "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave",
49 'reported-model' => 'qemu64',
50 flags
=> "+aes;+popcnt;+pni;+sse4.1;+sse4.2;+ssse3;+avx;+avx2;+bmi1;+bmi2;+f16c;+fma;+abm;+movbe;+xsave;+avx512f;+avx512bw;+avx512cd;+avx512dq;+avx512vl",
54 my $depreacated_cpu_map = {
55 # there never was such a client CPU, so map it to the server one for backward compat
56 'Icelake-Client' => 'Icelake-Server',
57 'Icelake-Client-noTSX' => 'Icelake-Server-noTSX',
60 my $cpu_vendor_list = {
62 486 => 'GenuineIntel',
63 pentium
=> 'GenuineIntel',
64 pentium2
=> 'GenuineIntel',
65 pentium3
=> 'GenuineIntel',
66 coreduo
=> 'GenuineIntel',
67 core2duo
=> 'GenuineIntel',
68 Conroe
=> 'GenuineIntel',
69 Penryn
=> 'GenuineIntel',
70 Nehalem
=> 'GenuineIntel',
71 'Nehalem-IBRS' => 'GenuineIntel',
72 Westmere
=> 'GenuineIntel',
73 'Westmere-IBRS' => 'GenuineIntel',
74 SandyBridge
=> 'GenuineIntel',
75 'SandyBridge-IBRS' => 'GenuineIntel',
76 IvyBridge
=> 'GenuineIntel',
77 'IvyBridge-IBRS' => 'GenuineIntel',
78 Haswell
=> 'GenuineIntel',
79 'Haswell-IBRS' => 'GenuineIntel',
80 'Haswell-noTSX' => 'GenuineIntel',
81 'Haswell-noTSX-IBRS' => 'GenuineIntel',
82 Broadwell
=> 'GenuineIntel',
83 'Broadwell-IBRS' => 'GenuineIntel',
84 'Broadwell-noTSX' => 'GenuineIntel',
85 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
86 'Skylake-Client' => 'GenuineIntel',
87 'Skylake-Client-IBRS' => 'GenuineIntel',
88 'Skylake-Client-noTSX-IBRS' => 'GenuineIntel',
89 'Skylake-Client-v4' => 'GenuineIntel',
90 'Skylake-Server' => 'GenuineIntel',
91 'Skylake-Server-IBRS' => 'GenuineIntel',
92 'Skylake-Server-noTSX-IBRS' => 'GenuineIntel',
93 'Skylake-Server-v4' => 'GenuineIntel',
94 'Skylake-Server-v5' => 'GenuineIntel',
95 'Cascadelake-Server' => 'GenuineIntel',
96 'Cascadelake-Server-v2' => 'GenuineIntel',
97 'Cascadelake-Server-noTSX' => 'GenuineIntel',
98 'Cascadelake-Server-v4' => 'GenuineIntel',
99 'Cascadelake-Server-v5' => 'GenuineIntel',
100 'Cooperlake' => 'GenuineIntel',
101 'Cooperlake-v2' => 'GenuineIntel',
102 KnightsMill
=> 'GenuineIntel',
103 'Icelake-Client' => 'GenuineIntel', # depreacated, removed with QEMU 7.1
104 'Icelake-Client-noTSX' => 'GenuineIntel', # depreacated, removed with QEMU 7.1
105 'Icelake-Server' => 'GenuineIntel',
106 'Icelake-Server-noTSX' => 'GenuineIntel',
107 'Icelake-Server-v3' => 'GenuineIntel',
108 'Icelake-Server-v4' => 'GenuineIntel',
109 'Icelake-Server-v5' => 'GenuineIntel',
110 'Icelake-Server-v6' => 'GenuineIntel',
111 'SapphireRapids' => 'GenuineIntel',
112 'SapphireRapids-v2' => 'GenuineIntel',
113 'GraniteRapids' => 'GenuineIntel',
116 athlon
=> 'AuthenticAMD',
117 phenom
=> 'AuthenticAMD',
118 Opteron_G1
=> 'AuthenticAMD',
119 Opteron_G2
=> 'AuthenticAMD',
120 Opteron_G3
=> 'AuthenticAMD',
121 Opteron_G4
=> 'AuthenticAMD',
122 Opteron_G5
=> 'AuthenticAMD',
123 EPYC
=> 'AuthenticAMD',
124 'EPYC-IBPB' => 'AuthenticAMD',
125 'EPYC-v3' => 'AuthenticAMD',
126 'EPYC-v4' => 'AuthenticAMD',
127 'EPYC-Rome' => 'AuthenticAMD',
128 'EPYC-Rome-v2' => 'AuthenticAMD',
129 'EPYC-Rome-v3' => 'AuthenticAMD',
130 'EPYC-Rome-v4' => 'AuthenticAMD',
131 'EPYC-Milan' => 'AuthenticAMD',
132 'EPYC-Milan-v2' => 'AuthenticAMD',
133 'EPYC-Genoa' => 'AuthenticAMD',
135 # generic types, use vendor from host node
144 my @supported_cpu_flags = (
158 my $cpu_flag_supported_re = qr/([+-])(@{[join('|', @supported_cpu_flags)]})/;
159 my $cpu_flag_any_re = qr/([+-])([a-zA-Z0-9\-_\.]+)/;
161 our $qemu_cmdline_cpu_re = qr/^((?>[+-]?[\w\-\._=]+,?)+)$/;
165 description
=> "Emulated CPU type. Can be default or custom name (custom model names must be prefixed with 'custom-').",
167 format_description
=> 'string',
172 'reported-model' => {
173 description
=> "CPU model and vendor to report to the guest. Must be a QEMU/KVM supported model."
174 ." Only valid for custom CPU model definitions, default models will always report themselves to the guest OS.",
176 enum
=> [ sort { lc("$a") cmp lc("$b") } keys %$cpu_vendor_list ],
181 description
=> "Do not identify as a KVM virtual machine.",
188 pattern
=> qr/[a-zA-Z0-9]{1,12}/,
189 format_description
=> 'vendor-id',
190 description
=> 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
194 description
=> "List of additional CPU flags separated by ';'. Use '+FLAG' to enable,"
195 ." '-FLAG' to disable a flag. Custom CPU models can specify any flag supported by"
196 ." QEMU/KVM, VM-specific flags must be from the following set for security reasons: "
197 . join(', ', @supported_cpu_flags),
198 format_description
=> '+FLAG[;-FLAG...]',
200 pattern
=> qr/$cpu_flag_any_re(;$cpu_flag_any_re)*/,
205 format
=> 'pve-phys-bits',
206 format_description
=> '8-64|host',
207 description
=> "The physical memory address bits that are reported to the guest OS. Should"
208 ." be smaller or equal to the host's. Set to 'host' to use value from host CPU, but"
209 ." note that doing so will break live migration to CPUs with other values.",
214 PVE
::JSONSchema
::register_format
('pve-phys-bits', \
&parse_phys_bits
);
215 sub parse_phys_bits
{
216 my ($str, $noerr) = @_;
218 my $err_msg = "value must be an integer between 8 and 64 or 'host'\n";
220 if ($str !~ m/^(host|\d{1,2})$/) {
221 die $err_msg if !$noerr;
225 if ($str =~ m/^\d+$/ && (int($str) < 8 || int($str) > 64)) {
226 die $err_msg if !$noerr;
233 # $cpu_fmt describes both the CPU config passed as part of a VM config, as well
234 # as the definition of a custom CPU model. There are some slight differences
235 # though, which we catch in the custom validation functions below.
236 PVE
::JSONSchema
::register_format
('pve-cpu-conf', $cpu_fmt, \
&validate_cpu_conf
);
237 sub validate_cpu_conf
{
239 # required, but can't be forced in schema since it's encoded in section header for custom models
240 die "CPU is missing cputype\n" if !$cpu->{cputype
};
243 PVE
::JSONSchema
::register_format
('pve-vm-cpu-conf', $cpu_fmt, \
&validate_vm_cpu_conf
);
244 sub validate_vm_cpu_conf
{
247 validate_cpu_conf
($cpu);
249 my $cputype = $cpu->{cputype
};
251 # a VM-specific config is only valid if the cputype exists
252 if (is_custom_model
($cputype)) {
253 # dies on unknown model
254 get_custom_model
($cputype);
256 die "Built-in cputype '$cputype' is not defined (missing 'custom-' prefix?)\n"
257 if !defined($cpu_vendor_list->{$cputype}) && !defined($builtin_models->{$cputype});
260 # in a VM-specific config, certain properties are limited/forbidden
262 die "VM-specific CPU flags must be a subset of: @{[join(', ', @supported_cpu_flags)]}\n"
263 if ($cpu->{flags
} && $cpu->{flags
} !~ m/^$cpu_flag_supported_re(;$cpu_flag_supported_re)*$/);
265 die "Property 'reported-model' not allowed in VM-specific CPU config.\n"
266 if defined($cpu->{'reported-model'});
271 # Section config settings
273 # shallow copy, since SectionConfig modifies propertyList internally
274 propertyList
=> { %$cpu_fmt },
282 return { %$cpu_fmt };
289 sub parse_section_header
{
290 my ($class, $line) = @_;
292 my ($type, $sectionId, $errmsg, $config) =
293 $class->SUPER::parse_section_header
($line);
296 return ($type, $sectionId, $errmsg, {
297 # name is given by section header, and we can always prepend 'custom-'
298 # since we're reading the custom CPU file
299 cputype
=> "custom-$sectionId",
304 my ($class, $filename, $cfg) = @_;
306 mkdir "/etc/pve/virtual-guest";
308 for my $model (keys %{$cfg->{ids
}}) {
309 my $model_conf = $cfg->{ids
}->{$model};
311 die "internal error: tried saving built-in CPU model (or missing prefix): $model_conf->{cputype}\n"
312 if !is_custom_model
($model_conf->{cputype
});
314 die "internal error: tried saving custom cpumodel with cputype (ignoring prefix: $model_conf->{cputype}) not equal to \$cfg->ids entry ($model)\n"
315 if "custom-$model" ne $model_conf->{cputype
};
317 # saved in section header
318 delete $model_conf->{cputype
};
321 $class->SUPER::write_config
($filename, $cfg);
324 sub add_cpu_json_properties
{
327 foreach my $opt (keys %$cpu_fmt) {
328 $prop->{$opt} = $cpu_fmt->{$opt};
335 my ($include_custom) = @_;
339 for my $default_model (keys %{$cpu_vendor_list}) {
341 name
=> $default_model,
343 vendor
=> $cpu_vendor_list->{$default_model},
347 for my $model (keys %{$builtin_models}) {
348 my $reported_model = $builtin_models->{$model}->{'reported-model'};
349 my $vendor = $cpu_vendor_list->{$reported_model};
357 return $models if !$include_custom;
359 my $conf = load_custom_model_conf
();
360 for my $custom_model (keys %{$conf->{ids
}}) {
361 my $reported_model = $conf->{ids
}->{$custom_model}->{'reported-model'};
362 $reported_model //= $cpu_fmt->{'reported-model'}->{default};
363 my $vendor = $cpu_vendor_list->{$reported_model};
365 name
=> "custom-$custom_model",
374 sub is_custom_model
{
376 return $cputype =~ m/^custom-/;
379 # Use this to get a single model in the format described by $cpu_fmt.
380 # Allows names with and without custom- prefix.
381 sub get_custom_model
{
382 my ($name, $noerr) = @_;
384 $name =~ s/^custom-//;
385 my $conf = load_custom_model_conf
();
387 my $entry = $conf->{ids
}->{$name};
388 if (!defined($entry)) {
389 die "Custom cputype '$name' not found\n" if !$noerr;
394 for my $property (keys %$cpu_fmt) {
395 if (my $value = $entry->{$property}) {
396 $model->{$property} = $value;
403 # Print a QEMU device node for a given VM configuration for hotplugging CPUs
404 sub print_cpu_device
{
405 my ($conf, $id) = @_;
407 my $kvm = $conf->{kvm
} // 1;
408 my $cpu = get_default_cpu_type
('x86_64', $kvm);
409 if (my $cputype = $conf->{cpu
}) {
410 my $cpuconf = PVE
::JSONSchema
::parse_property_string
('pve-vm-cpu-conf', $cputype)
411 or die "Cannot parse cpu description: $cputype\n";
412 $cpu = $cpuconf->{cputype
};
414 if (my $model = $builtin_models->{$cpu}) {
415 $cpu = $model->{'reported-model'};
416 } elsif (is_custom_model
($cputype)) {
417 my $custom_cpu = get_custom_model
($cpu);
419 $cpu = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default};
421 if (my $replacement_type = $depreacated_cpu_map->{$cpu}) {
422 $cpu = $replacement_type;
426 my $cores = $conf->{cores
} || 1;
428 my $current_core = ($id - 1) % $cores;
429 my $current_socket = int(($id - 1 - $current_core)/$cores);
431 # FIXME: hot plugging other architectures like our unofficial arch64 support?
432 return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
435 # Resolves multiple arrays of hashes representing CPU flags with metadata to a
436 # single string in QEMU "-cpu" compatible format. Later arrays have higher
439 # Hashes take the following format:
442 # op => "+", # defaults to "" if undefined
443 # reason => "to support AES acceleration", # for override warnings
444 # value => "" # needed for kvm=off (value: off) etc...
448 sub resolve_cpu_flags
{
452 for my $flag_name (keys %$hash) {
453 my $flag = $hash->{$flag_name};
454 my $old_flag = $flags->{$flag_name};
457 $flag->{reason
} //= "unknown origin";
460 my $value_changed = (defined($flag->{value
}) != defined($old_flag->{value
})) ||
461 (defined($flag->{value
}) && $flag->{value
} ne $old_flag->{value
});
463 if ($old_flag->{op
} eq $flag->{op
} && !$value_changed) {
464 $flags->{$flag_name}->{reason
} .= " & $flag->{reason}";
468 my $old = print_cpuflag_hash
($flag_name, $flags->{$flag_name});
469 my $new = print_cpuflag_hash
($flag_name, $flag);
470 warn "warning: CPU flag/setting $new overwrites $old\n";
473 $flags->{$flag_name} = $flag;
478 # sort for command line stability
479 for my $flag_name (sort keys %$flags) {
481 $flag_str .= $flags->{$flag_name}->{op
};
482 $flag_str .= $flag_name;
483 $flag_str .= "=$flags->{$flag_name}->{value}"
484 if $flags->{$flag_name}->{value
};
490 sub print_cpuflag_hash
{
491 my ($flag_name, $flag) = @_;
492 my $formatted = "'$flag->{op}$flag_name";
493 $formatted .= "=$flag->{value}" if defined($flag->{value
});
495 $formatted .= " ($flag->{reason})" if defined($flag->{reason
});
499 sub parse_cpuflag_list
{
500 my ($re, $reason, $flaglist) = @_;
503 return $res if !$flaglist;
505 foreach my $flag (split(";", $flaglist)) {
506 if ($flag =~ m/^$re$/) {
507 $res->{$2} = { op
=> $1, reason
=> $reason };
514 # Calculate QEMU's '-cpu' argument from a given VM configuration
515 sub get_cpu_options
{
516 my ($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough) = @_;
518 my $cputype = get_default_cpu_type
($arch, $kvm);
524 if (my $cpu_prop_str = $conf->{cpu
}) {
525 $cpu = PVE
::JSONSchema
::parse_property_string
('pve-vm-cpu-conf', $cpu_prop_str)
526 or die "Cannot parse cpu description: $cpu_prop_str\n";
528 $cputype = $cpu->{cputype
};
529 if (my $model = $builtin_models->{$cputype}) {
530 $cputype = $model->{'reported-model'};
531 $builtin_cpu->{flags
} = $model->{'flags'};
532 } elsif (is_custom_model
($cputype)) {
533 $custom_cpu = get_custom_model
($cputype);
535 $cputype = $custom_cpu->{'reported-model'} // $cpu_fmt->{'reported-model'}->{default};
536 $kvm_off = $custom_cpu->{hidden
} if defined($custom_cpu->{hidden
});
537 $hv_vendor_id = $custom_cpu->{'hv-vendor-id'};
540 if (my $replacement_type = $depreacated_cpu_map->{$cputype}) {
541 $cputype = $replacement_type;
544 # VM-specific settings override custom CPU config
545 $kvm_off = $cpu->{hidden
} if defined($cpu->{hidden
});
546 $hv_vendor_id = $cpu->{'hv-vendor-id'} if defined($cpu->{'hv-vendor-id'});
549 my $pve_flags = get_pve_cpu_flags
($conf, $kvm, $cputype, $arch, $machine_version);
552 ? get_hyperv_enlightenments
(
561 my $builtin_cputype_flags = parse_cpuflag_list
(
562 $cpu_flag_any_re, "set by builtin CPU model", $builtin_cpu->{flags
});
564 my $custom_cputype_flags = parse_cpuflag_list
(
565 $cpu_flag_any_re, "set by custom CPU model", $custom_cpu->{flags
});
567 my $vm_flags = parse_cpuflag_list
(
568 $cpu_flag_supported_re, "manually set for VM", $cpu->{flags
});
570 my $pve_forced_flags = {};
571 $pve_forced_flags->{'enforce'} = {
572 reason
=> "error if requested CPU settings not available",
573 } if $cputype ne 'host' && $kvm && $arch eq 'x86_64';
574 $pve_forced_flags->{'kvm'} = {
576 reason
=> "hide KVM virtualization from guest",
579 # $cputype is the "reported-model" for custom types, so we can just look up
580 # the vendor in the default list
581 my $cpu_vendor = $cpu_vendor_list->{$cputype};
583 $pve_forced_flags->{'vendor'} = {
584 value
=> $cpu_vendor,
585 } if $cpu_vendor ne 'default';
586 } elsif ($arch ne 'aarch64') {
587 die "internal error"; # should not happen
590 my $cpu_str = $cputype;
592 # will be resolved in parameter order
593 $cpu_str .= resolve_cpu_flags
(
594 $pve_flags, $hv_flags, $builtin_cputype_flags, $custom_cputype_flags, $vm_flags, $pve_forced_flags);
597 foreach my $conf ($custom_cpu, $cpu) {
598 next if !defined($conf);
599 my $conf_val = $conf->{'phys-bits'};
601 if ($conf_val eq 'host') {
602 $phys_bits = ",host-phys-bits=true";
604 $phys_bits = ",phys-bits=$conf_val";
607 $cpu_str .= $phys_bits;
609 return ('-cpu', $cpu_str);
612 # Some hardcoded flags required by certain configurations
613 sub get_pve_cpu_flags
{
614 my ($conf, $kvm, $cputype, $arch, $machine_version) = @_;
617 my $pve_msg = "set by PVE;";
619 $pve_flags->{'lahf_lm'} = {
621 reason
=> "$pve_msg to support Windows 8.1+",
622 } if $cputype eq 'kvm64' && $arch eq 'x86_64';
624 $pve_flags->{'x2apic'} = {
626 reason
=> "$pve_msg incompatible with Solaris",
627 } if $conf->{ostype
} && $conf->{ostype
} eq 'solaris';
629 $pve_flags->{'sep'} = {
631 reason
=> "$pve_msg to support Windows 8+ and improve Windows XP+",
632 } if $cputype eq 'kvm64' || $cputype eq 'kvm32';
634 $pve_flags->{'rdtscp'} = {
636 reason
=> "$pve_msg broken on AMD Opteron",
637 } if $cputype =~ m/^Opteron/;
639 if (min_version
($machine_version, 2, 3) && $kvm && $arch eq 'x86_64') {
640 $pve_flags->{'kvm_pv_unhalt'} = {
642 reason
=> "$pve_msg to improve Linux guest spinlock performance",
644 $pve_flags->{'kvm_pv_eoi'} = {
646 reason
=> "$pve_msg to improve Linux guest interrupt performance",
653 sub get_hyperv_enlightenments
{
654 my ($winversion, $machine_version, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
656 return if $winversion < 6;
657 return if $bios && $bios eq 'ovmf' && $winversion < 8;
660 my $default_reason = "automatic Hyper-V enlightenment for Windows";
662 my ($flag, $value, $reason) = @_;
664 reason
=> $reason // $default_reason,
669 my $hv_vendor_set = defined($hv_vendor_id);
670 if ($gpu_passthrough || $hv_vendor_set) {
671 $hv_vendor_id //= 'proxmox';
672 $flagfn->('hv_vendor_id', $hv_vendor_id, $hv_vendor_set ?
673 "custom hv_vendor_id set" : "NVIDIA workaround for GPU passthrough");
676 if (min_version
($machine_version, 2, 3)) {
677 $flagfn->('hv_spinlocks', '0x1fff');
678 $flagfn->('hv_vapic');
679 $flagfn->('hv_time');
681 $flagfn->('hv_spinlocks', '0xffff');
684 if (min_version
($machine_version, 2, 6)) {
685 $flagfn->('hv_reset');
686 $flagfn->('hv_vpindex');
687 $flagfn->('hv_runtime');
690 if ($winversion >= 7) {
691 my $win7_reason = $default_reason . " 7 and higher";
692 $flagfn->('hv_relaxed', undef, $win7_reason);
694 if (min_version
($machine_version, 2, 12)) {
695 $flagfn->('hv_synic', undef, $win7_reason);
696 $flagfn->('hv_stimer', undef, $win7_reason);
699 if (min_version
($machine_version, 3, 1)) {
700 $flagfn->('hv_ipi', undef, $win7_reason);
707 sub get_cpu_from_running_vm
{
710 my $cmdline = PVE
::QemuServer
::Helpers
::parse_cmdline
($pid);
711 die "could not read commandline of running machine\n"
712 if !$cmdline->{cpu
}->{value
};
714 # sanitize and untaint value
715 $cmdline->{cpu
}->{value
} =~ $qemu_cmdline_cpu_re;
719 sub get_default_cpu_type
{
720 my ($arch, $kvm) = @_;
722 my $cputype = $kvm ?
'kvm64' : 'qemu64';
723 $cputype = 'cortex-a57' if $arch eq 'aarch64';
728 __PACKAGE__-
>register();