PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. This option is generally not required and"
 371         ." is often better handled from within the guest OS.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.',
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041     return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046     my ($volid, $noerr) = @_;
1047
1048     return $volid if $volid =~ m|^/|;
1049
1050     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051     if ($@) {
1052         return if $noerr;
1053         die $@;
1054     }
1055     return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059     host => {
1060         default_key => 1,
1061         type => 'string', format => 'pve-qm-usb-device',
1062         format_description => 'HOSTUSBDEVICE|spice',
1063         description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066  'bus-port(.port)*' (decimal numbers) or
1067  'vendor_id:product_id' (hexadeciaml numbers) or
1068  'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077     },
1078     usb3 => {
1079         optional => 1,
1080         type => 'boolean',
1081         description => "Specifies whether if given host option is a USB3 device or port.",
1082         default => 0,
1083     },
1084 };
1085
1086 my $usbdesc = {
1087     optional => 1,
1088     type => 'string', format => $usb_fmt,
1089     description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '(/dev/.+|socket)',
1097         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1098         verbose_description =>  <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114         description =>  "Map host parallel devices (n is 0 to 2).",
1115         verbose_description =>  <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1126     $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1130     $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1134     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1142     $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146     legacy => {
1147         optional => 1,
1148         default_key => 1,
1149         type => 'string',
1150         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151                      . " Deprecated, use 'order=' instead.",
1152         pattern => '[acdn]{1,4}',
1153         format_description => "[acdn]{1,4}",
1154
1155         # note: this is also the fallback if boot: is not given at all
1156         default => 'cdn',
1157     },
1158     order => {
1159         optional => 1,
1160         type => 'string',
1161         format => 'pve-qm-bootdev-list',
1162         format_description => "device[;device...]",
1163         description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176     },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182     my ($dev, $noerr) = @_;
1183
1184     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187     my $check = sub {
1188         my ($base) = @_;
1189         return 0 if $dev !~ m/^$base\d+$/;
1190         return 0 if !$confdesc->{$dev};
1191         return 1;
1192     };
1193
1194     return $dev if $check->("net");
1195     return $dev if $check->("usb");
1196     return $dev if $check->("hostpci");
1197
1198     return if $noerr;
1199     die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203     my ($devs) = @_;
1204     return "" if !@$devs;
1205     my $data = { order => join(';', @$devs) };
1206     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212     return $kvm_api_version if $kvm_api_version;
1213
1214     open my $fh, '<', '/dev/kvm' or return;
1215
1216     # 0xae00 => KVM_GET_API_VERSION
1217     $kvm_api_version = ioctl($fh, 0xae00, 0);
1218     close($fh);
1219
1220     return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227     my ($binary) = @_;
1228
1229     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230     my $st = stat($binary);
1231
1232     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234         $cachedmtime == $st->mtime;
1235
1236     $kvm_user_version->{$binary} = 'unknown';
1237     $kvm_mtime->{$binary} = $st->mtime;
1238
1239     my $code = sub {
1240         my $line = shift;
1241         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242             $kvm_user_version->{$binary} = $2;
1243         }
1244     };
1245
1246     eval { run_command([$binary, '--version'], outfunc => $code); };
1247     warn $@ if $@;
1248
1249     return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253     my ($machine_type, $version) = @_;
1254     $version = kvm_user_version() if !defined($version);
1255     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259     return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263     my $key = shift;
1264     return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270     return  $cdrom_path if $cdrom_path;
1271
1272     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278     my ($storecfg, $vmid, $cdrom) = @_;
1279
1280     if ($cdrom eq 'cdrom') {
1281         return get_cdrom_path();
1282     } elsif ($cdrom eq 'none') {
1283         return '';
1284     } elsif ($cdrom =~ m|^/|) {
1285         return $cdrom;
1286     } else {
1287         return PVE::Storage::path($storecfg, $cdrom);
1288     }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293     my ($vmid, $file, $media) = @_;
1294
1295      if (!($file eq 'none' || $file eq 'cdrom' ||
1296           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298         return if $file =~ m|/|;
1299
1300         if ($media && $media eq 'cdrom') {
1301             $file = "local:iso/$file";
1302         } else {
1303             $file = "local:$vmid/$file";
1304         }
1305     }
1306
1307     return $file;
1308 }
1309
1310 sub verify_media_type {
1311     my ($opt, $vtype, $media) = @_;
1312
1313     return if !$media;
1314
1315     my $etype;
1316     if ($media eq 'disk') {
1317         $etype = 'images';
1318     } elsif ($media eq 'cdrom') {
1319         $etype = 'iso';
1320     } else {
1321         die "internal error";
1322     }
1323
1324     return if ($vtype eq $etype);
1325
1326     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330     my ($opt, $storecfg, $drive) = @_;
1331
1332     # try to convert filesystem paths to volume IDs
1333
1334     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335         ($drive->{file} !~ m|^/dev/.+|) &&
1336         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337         ($drive->{file} !~ m/^\d+$/)) {
1338         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340             if !$vtype;
1341         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342         verify_media_type($opt, $vtype, $drive->{media});
1343         $drive->{file} = $volid;
1344     }
1345
1346     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350     my ($data) = @_;
1351
1352     my $res = {};
1353
1354     return $res if $data eq '0';
1355
1356     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358     foreach my $feature (PVE::Tools::split_list($data)) {
1359         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360             $res->{$1} = 1;
1361         } else {
1362             die "invalid hotplug feature '$feature'\n";
1363         }
1364     }
1365     return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370     my ($value, $noerr) = @_;
1371
1372     return $value if parse_hotplug_features($value);
1373
1374     return if $noerr;
1375
1376     die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380     my($fh, $noerr) = @_;
1381
1382     my $SG_IO = 0x2285;
1383     my $SG_GET_VERSION_NUM = 0x2282;
1384
1385     my $versionbuf = "\x00" x 8;
1386     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387     if (!$ret) {
1388         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389         return;
1390     }
1391     my $version = unpack("I", $versionbuf);
1392     if ($version < 30000) {
1393         die "scsi generic interface too old\n"  if !$noerr;
1394         return;
1395     }
1396
1397     my $buf = "\x00" x 36;
1398     my $sensebuf = "\x00" x 8;
1399     my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401     # see /usr/include/scsi/sg.h
1402     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404     my $packet = pack(
1405         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406     );
1407
1408     $ret = ioctl($fh, $SG_IO, $packet);
1409     if (!$ret) {
1410         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my @res = unpack($sg_io_hdr_t, $packet);
1415     if ($res[17] || $res[18]) {
1416         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417         return;
1418     }
1419
1420     my $res = {};
1421     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424     $res->{type} &= 0x1F;
1425
1426     return $res;
1427 }
1428
1429 sub path_is_scsi {
1430     my ($path) = @_;
1431
1432     my $fh = IO::File->new("+<$path") || return;
1433     my $res = scsi_inquiry($fh, 1);
1434     close($fh);
1435
1436     return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440     my ($conf, $arch) = @_;
1441
1442     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444     # we use uhci for old VMs because tablet driver was buggy in older qemu
1445     my $usbbus;
1446     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447         $usbbus = 'ehci';
1448     } else {
1449         $usbbus = 'uhci';
1450     }
1451
1452     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     return if $arch ne 'aarch64';
1459
1460     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464     my ($drive) = @_;
1465     return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     my $drive_id = get_drive_id($drive);
1475     if ($drive->{interface} eq 'virtio') {
1476         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479     } elsif ($drive->{interface} eq 'scsi') {
1480
1481         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482         my $unit = $drive->{index} % $maxdev;
1483         my $devicetype = 'hd';
1484         my $path = '';
1485         if (drive_is_cdrom($drive)) {
1486             $devicetype = 'cd';
1487         } else {
1488             if ($drive->{file} =~ m|^/|) {
1489                 $path = $drive->{file};
1490                 if (my $info = path_is_scsi($path)) {
1491                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1492                         $devicetype = 'block';
1493                     } elsif ($info->{type} == 1) { # tape
1494                         $devicetype = 'generic';
1495                     }
1496                 }
1497             } else {
1498                  $path = PVE::Storage::path($storecfg, $drive->{file});
1499             }
1500
1501             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502             my $version = extract_version($machine_type, kvm_user_version());
1503             if ($path =~ m/^iscsi\:\/\// &&
1504                !min_version($version, 4, 1)) {
1505                 $devicetype = 'generic';
1506             }
1507         }
1508
1509         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511         } else {
1512             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513                 .",lun=$drive->{index}";
1514         }
1515         $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518             $device .= ",rotation_rate=1";
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524         my $controller = int($drive->{index} / $maxdev);
1525         my $unit = $drive->{index} % $maxdev;
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format = $drive->{format};
1584     my $drive_id = get_drive_id($drive);
1585
1586     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589     if (drive_is_cdrom($drive)) {
1590         $path = get_iso_path($storecfg, $vmid, $volid);
1591         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592     } else {
1593         if ($storeid) {
1594             $path = PVE::Storage::path($storecfg, $volid);
1595             $format //= qemu_img_format($scfg, $volname);
1596         } else {
1597             $path = $volid;
1598             $format //= "raw";
1599         }
1600    }
1601
1602    my $is_rbd = $path =~ m/^rbd:/;
1603
1604     my $opts = '';
1605     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606     foreach my $o (@qemu_drive_options) {
1607         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608     }
1609
1610     # snapshot only accepts on|off
1611     if (defined($drive->{snapshot})) {
1612         my $v = $drive->{snapshot} ? 'on' : 'off';
1613         $opts .= ",snapshot=$v";
1614     }
1615
1616     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618     }
1619
1620     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621         my ($dir, $qmpname) = @$type;
1622         if (my $v = $drive->{"mbps$dir"}) {
1623             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624         }
1625         if (my $v = $drive->{"mbps${dir}_max"}) {
1626             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627         }
1628         if (my $v = $drive->{"bps${dir}_max_length"}) {
1629             $opts .= ",throttling.bps$qmpname-max-length=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}"}) {
1632             $opts .= ",throttling.iops$qmpname=$v";
1633         }
1634         if (my $v = $drive->{"iops${dir}_max"}) {
1635             $opts .= ",throttling.iops$qmpname-max=$v";
1636         }
1637         if (my $v = $drive->{"iops${dir}_max_length"}) {
1638             $opts .= ",throttling.iops$qmpname-max-length=$v";
1639         }
1640     }
1641
1642     if ($pbs_name) {
1643         $format = "rbd" if $is_rbd;
1644         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645             if !$format;
1646         $opts .= ",format=alloc-track,file.driver=$format";
1647     } elsif ($format) {
1648         $opts .= ",format=$format";
1649     }
1650
1651     my $cache_direct = 0;
1652
1653     if (my $cache = $drive->{cache}) {
1654         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656         $opts .= ",cache=none";
1657         $cache_direct = 1;
1658     }
1659
1660     # io_uring with cache mode writeback or writethrough on krbd will hang...
1661     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664     # sometimes, just plain disable...
1665     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667     if (!$drive->{aio}) {
1668         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669             # io_uring supports all cache modes
1670             $opts .= ",aio=io_uring";
1671         } else {
1672             # aio native works only with O_DIRECT
1673             if($cache_direct) {
1674                 $opts .= ",aio=native";
1675             } else {
1676                 $opts .= ",aio=threads";
1677             }
1678         }
1679     }
1680
1681     if (!drive_is_cdrom($drive)) {
1682         my $detectzeroes;
1683         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684             $detectzeroes = 'off';
1685         } elsif ($drive->{discard}) {
1686             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687         } else {
1688             # This used to be our default with discard not being specified:
1689             $detectzeroes = 'on';
1690         }
1691
1692         # note: 'detect-zeroes' works per blockdev and we want it to persist
1693         # after the alloc-track is removed, so put it on 'file' directly
1694         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     if ($pbs_name) {
1699         $opts .= ",backing=$pbs_name";
1700         $opts .= ",auto-remove=on";
1701     }
1702
1703     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704     my $file_param = "file";
1705     if ($pbs_name) {
1706         # non-rbd drivers require the underlying file to be a seperate block
1707         # node, so add a second .file indirection
1708         $file_param .= ".file" if !$is_rbd;
1709         $file_param .= ".filename";
1710     }
1711     my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717     my ($pbs_conf, $pbs_name) = @_;
1718     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719     $blockdev .= ",repository=$pbs_conf->{repository}";
1720     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1721     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1722     $blockdev .= ",archive=$pbs_conf->{archive}";
1723     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1724     return $blockdev;
1725 }
1726
1727 sub print_netdevice_full {
1728     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1729
1730     my $device = $net->{model};
1731     if ($net->{model} eq 'virtio') {
1732          $device = 'virtio-net-pci';
1733      };
1734
1735     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1736     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1737     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1738         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1739         # and out of each queue plus one config interrupt and control vector queue
1740         my $vectors = $net->{queues} * 2 + 2;
1741         $tmpstr .= ",vectors=$vectors,mq=on";
1742     }
1743     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1744
1745     if (my $mtu = $net->{mtu}) {
1746         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1747             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1748             if ($mtu == 1) {
1749                  $mtu = $bridge_mtu;
1750             } elsif ($mtu < 576) {
1751                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1752             } elsif ($mtu > $bridge_mtu) {
1753                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1754             }
1755             $tmpstr .= ",host_mtu=$mtu";
1756         } else {
1757             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1758         }
1759     }
1760
1761     if ($use_old_bios_files) {
1762         my $romfile;
1763         if ($device eq 'virtio-net-pci') {
1764             $romfile = 'pxe-virtio.rom';
1765         } elsif ($device eq 'e1000') {
1766             $romfile = 'pxe-e1000.rom';
1767         } elsif ($device eq 'e1000e') {
1768             $romfile = 'pxe-e1000e.rom';
1769         } elsif ($device eq 'ne2k') {
1770             $romfile = 'pxe-ne2k_pci.rom';
1771         } elsif ($device eq 'pcnet') {
1772             $romfile = 'pxe-pcnet.rom';
1773         } elsif ($device eq 'rtl8139') {
1774             $romfile = 'pxe-rtl8139.rom';
1775         }
1776         $tmpstr .= ",romfile=$romfile" if $romfile;
1777     }
1778
1779     return $tmpstr;
1780 }
1781
1782 sub print_netdev_full {
1783     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1784
1785     my $i = '';
1786     if ($netid =~ m/^net(\d+)$/) {
1787         $i = int($1);
1788     }
1789
1790     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1791
1792     my $ifname = "tap${vmid}i$i";
1793
1794     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1795     die "interface name '$ifname' is too long (max 15 character)\n"
1796         if length($ifname) >= 16;
1797
1798     my $vhostparam = '';
1799     if (is_native($arch)) {
1800         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1801     }
1802
1803     my $vmname = $conf->{name} || "vm$vmid";
1804
1805     my $netdev = "";
1806     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1807
1808     if ($net->{bridge}) {
1809         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1810             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1811     } else {
1812         $netdev = "type=user,id=$netid,hostname=$vmname";
1813     }
1814
1815     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1816
1817     return $netdev;
1818 }
1819
1820 my $vga_map = {
1821     'cirrus' => 'cirrus-vga',
1822     'std' => 'VGA',
1823     'vmware' => 'vmware-svga',
1824     'virtio' => 'virtio-vga',
1825     'virtio-gl' => 'virtio-vga-gl',
1826 };
1827
1828 sub print_vga_device {
1829     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1830
1831     my $type = $vga_map->{$vga->{type}};
1832     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1833         $type = 'virtio-gpu';
1834     }
1835     my $vgamem_mb = $vga->{memory};
1836
1837     my $max_outputs = '';
1838     if ($qxlnum) {
1839         $type = $id ? 'qxl' : 'qxl-vga';
1840
1841         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1842             # set max outputs so linux can have up to 4 qxl displays with one device
1843             if (min_version($machine_version, 4, 1)) {
1844                 $max_outputs = ",max_outputs=4";
1845             }
1846         }
1847     }
1848
1849     die "no devicetype for $vga->{type}\n" if !$type;
1850
1851     my $memory = "";
1852     if ($vgamem_mb) {
1853         if ($vga->{type} =~ /^virtio/) {
1854             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1855             $memory = ",max_hostmem=$bytes";
1856         } elsif ($qxlnum) {
1857             # from https://www.spice-space.org/multiple-monitors.html
1858             $memory = ",vgamem_mb=$vga->{memory}";
1859             my $ram = $vgamem_mb * 4;
1860             my $vram = $vgamem_mb * 2;
1861             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1862         } else {
1863             $memory = ",vgamem_mb=$vga->{memory}";
1864         }
1865     } elsif ($qxlnum && $id) {
1866         $memory = ",ram_size=67108864,vram_size=33554432";
1867     }
1868
1869     my $edidoff = "";
1870     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1871         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1872     }
1873
1874     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1875     my $vgaid = "vga" . ($id // '');
1876     my $pciaddr;
1877     if ($q35 && $vgaid eq 'vga') {
1878         # the first display uses pcie.0 bus on q35 machines
1879         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1880     } else {
1881         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1882     }
1883
1884     if ($vga->{type} eq 'virtio-gl') {
1885         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1886         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1887             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1888
1889         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1890             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1891     }
1892
1893     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1894 }
1895
1896 sub parse_number_sets {
1897     my ($set) = @_;
1898     my $res = [];
1899     foreach my $part (split(/;/, $set)) {
1900         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1901             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1902             push @$res, [ $1, $2 ];
1903         } else {
1904             die "invalid range: $part\n";
1905         }
1906     }
1907     return $res;
1908 }
1909
1910 sub parse_numa {
1911     my ($data) = @_;
1912
1913     my $res = parse_property_string($numa_fmt, $data);
1914     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1915     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1916     return $res;
1917 }
1918
1919 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1920 sub parse_net {
1921     my ($data) = @_;
1922
1923     my $res = eval { parse_property_string($net_fmt, $data) };
1924     if ($@) {
1925         warn $@;
1926         return;
1927     }
1928     if (!defined($res->{macaddr})) {
1929         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1930         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1931     }
1932     return $res;
1933 }
1934
1935 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1936 sub parse_ipconfig {
1937     my ($data) = @_;
1938
1939     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1940     if ($@) {
1941         warn $@;
1942         return;
1943     }
1944
1945     if ($res->{gw} && !$res->{ip}) {
1946         warn 'gateway specified without specifying an IP address';
1947         return;
1948     }
1949     if ($res->{gw6} && !$res->{ip6}) {
1950         warn 'IPv6 gateway specified without specifying an IPv6 address';
1951         return;
1952     }
1953     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1954         warn 'gateway specified together with DHCP';
1955         return;
1956     }
1957     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1958         # gw6 + auto/dhcp
1959         warn "IPv6 gateway specified together with $res->{ip6} address";
1960         return;
1961     }
1962
1963     if (!$res->{ip} && !$res->{ip6}) {
1964         return { ip => 'dhcp', ip6 => 'dhcp' };
1965     }
1966
1967     return $res;
1968 }
1969
1970 sub print_net {
1971     my $net = shift;
1972
1973     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1974 }
1975
1976 sub add_random_macs {
1977     my ($settings) = @_;
1978
1979     foreach my $opt (keys %$settings) {
1980         next if $opt !~ m/^net(\d+)$/;
1981         my $net = parse_net($settings->{$opt});
1982         next if !$net;
1983         $settings->{$opt} = print_net($net);
1984     }
1985 }
1986
1987 sub vm_is_volid_owner {
1988     my ($storecfg, $vmid, $volid) = @_;
1989
1990     if ($volid !~  m|^/|) {
1991         my ($path, $owner);
1992         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1993         if ($owner && ($owner == $vmid)) {
1994             return 1;
1995         }
1996     }
1997
1998     return;
1999 }
2000
2001 sub vmconfig_register_unused_drive {
2002     my ($storecfg, $vmid, $conf, $drive) = @_;
2003
2004     if (drive_is_cloudinit($drive)) {
2005         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2006         warn $@ if $@;
2007     } elsif (!drive_is_cdrom($drive)) {
2008         my $volid = $drive->{file};
2009         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2010             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2011         }
2012     }
2013 }
2014
2015 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2016 my $smbios1_fmt = {
2017     uuid => {
2018         type => 'string',
2019         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2020         format_description => 'UUID',
2021         description => "Set SMBIOS1 UUID.",
2022         optional => 1,
2023     },
2024     version => {
2025         type => 'string',
2026         pattern => '[A-Za-z0-9+\/]+={0,2}',
2027         format_description => 'Base64 encoded string',
2028         description => "Set SMBIOS1 version.",
2029         optional => 1,
2030     },
2031     serial => {
2032         type => 'string',
2033         pattern => '[A-Za-z0-9+\/]+={0,2}',
2034         format_description => 'Base64 encoded string',
2035         description => "Set SMBIOS1 serial number.",
2036         optional => 1,
2037     },
2038     manufacturer => {
2039         type => 'string',
2040         pattern => '[A-Za-z0-9+\/]+={0,2}',
2041         format_description => 'Base64 encoded string',
2042         description => "Set SMBIOS1 manufacturer.",
2043         optional => 1,
2044     },
2045     product => {
2046         type => 'string',
2047         pattern => '[A-Za-z0-9+\/]+={0,2}',
2048         format_description => 'Base64 encoded string',
2049         description => "Set SMBIOS1 product ID.",
2050         optional => 1,
2051     },
2052     sku => {
2053         type => 'string',
2054         pattern => '[A-Za-z0-9+\/]+={0,2}',
2055         format_description => 'Base64 encoded string',
2056         description => "Set SMBIOS1 SKU string.",
2057         optional => 1,
2058     },
2059     family => {
2060         type => 'string',
2061         pattern => '[A-Za-z0-9+\/]+={0,2}',
2062         format_description => 'Base64 encoded string',
2063         description => "Set SMBIOS1 family string.",
2064         optional => 1,
2065     },
2066     base64 => {
2067         type => 'boolean',
2068         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2069         optional => 1,
2070     },
2071 };
2072
2073 sub parse_smbios1 {
2074     my ($data) = @_;
2075
2076     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2077     warn $@ if $@;
2078     return $res;
2079 }
2080
2081 sub print_smbios1 {
2082     my ($smbios1) = @_;
2083     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2084 }
2085
2086 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2087
2088 sub parse_watchdog {
2089     my ($value) = @_;
2090
2091     return if !$value;
2092
2093     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2094     warn $@ if $@;
2095     return $res;
2096 }
2097
2098 sub parse_guest_agent {
2099     my ($conf) = @_;
2100
2101     return {} if !defined($conf->{agent});
2102
2103     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2104     warn $@ if $@;
2105
2106     # if the agent is disabled ignore the other potentially set properties
2107     return {} if !$res->{enabled};
2108     return $res;
2109 }
2110
2111 sub get_qga_key {
2112     my ($conf, $key) = @_;
2113     return undef if !defined($conf->{agent});
2114
2115     my $agent = parse_guest_agent($conf);
2116     return $agent->{$key};
2117 }
2118
2119 sub parse_vga {
2120     my ($value) = @_;
2121
2122     return {} if !$value;
2123     my $res = eval { parse_property_string($vga_fmt, $value) };
2124     warn $@ if $@;
2125     return $res;
2126 }
2127
2128 sub parse_rng {
2129     my ($value) = @_;
2130
2131     return if !$value;
2132
2133     my $res = eval { parse_property_string($rng_fmt, $value) };
2134     warn $@ if $@;
2135     return $res;
2136 }
2137
2138 sub parse_meta_info {
2139     my ($value) = @_;
2140
2141     return if !$value;
2142
2143     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2144     warn $@ if $@;
2145     return $res;
2146 }
2147
2148 sub new_meta_info_string {
2149     my () = @_; # for now do not allow to override any value
2150
2151     return PVE::JSONSchema::print_property_string(
2152         {
2153             'creation-qemu' => kvm_user_version(),
2154             ctime => "". int(time()),
2155         },
2156         $meta_info_fmt
2157     );
2158 }
2159
2160 sub qemu_created_version_fixups {
2161     my ($conf, $forcemachine, $kvmver) = @_;
2162
2163     my $meta = parse_meta_info($conf->{meta}) // {};
2164     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2165
2166     # check if we need to apply some handling for VMs that always use the latest machine version but
2167     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2168     # would be required (we do not want to pin machine version for non-windows OS type)
2169     if (
2170         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2171         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2172         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2173         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2174     ) {
2175         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2176         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2177             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2178             # and thus with the predictable interface naming of systemd
2179             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2180         }
2181     }
2182     return;
2183 }
2184
2185 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2186 sub verify_usb_device {
2187     my ($value, $noerr) = @_;
2188
2189     return $value if parse_usb_device($value);
2190
2191     return if $noerr;
2192
2193     die "unable to parse usb device\n";
2194 }
2195
2196 # add JSON properties for create and set function
2197 sub json_config_properties {
2198     my ($prop, $with_disk_alloc) = @_;
2199
2200     my $skip_json_config_opts = {
2201         parent => 1,
2202         snaptime => 1,
2203         vmstate => 1,
2204         runningmachine => 1,
2205         runningcpu => 1,
2206         meta => 1,
2207     };
2208
2209     foreach my $opt (keys %$confdesc) {
2210         next if $skip_json_config_opts->{$opt};
2211
2212         if ($with_disk_alloc && is_valid_drivename($opt)) {
2213             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2214         } else {
2215             $prop->{$opt} = $confdesc->{$opt};
2216         }
2217     }
2218
2219     return $prop;
2220 }
2221
2222 # Properties that we can read from an OVF file
2223 sub json_ovf_properties {
2224     my $prop = {};
2225
2226     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2227         $prop->{$device} = {
2228             type => 'string',
2229             format => 'pve-volume-id-or-absolute-path',
2230             description => "Disk image that gets imported to $device",
2231             optional => 1,
2232         };
2233     }
2234
2235     $prop->{cores} = {
2236         type => 'integer',
2237         description => "The number of CPU cores.",
2238         optional => 1,
2239     };
2240     $prop->{memory} = {
2241         type => 'integer',
2242         description => "Amount of RAM for the VM in MB.",
2243         optional => 1,
2244     };
2245     $prop->{name} = {
2246         type => 'string',
2247         description => "Name of the VM.",
2248         optional => 1,
2249     };
2250
2251     return $prop;
2252 }
2253
2254 # return copy of $confdesc_cloudinit to generate documentation
2255 sub cloudinit_config_properties {
2256
2257     return dclone($confdesc_cloudinit);
2258 }
2259
2260 sub check_type {
2261     my ($key, $value) = @_;
2262
2263     die "unknown setting '$key'\n" if !$confdesc->{$key};
2264
2265     my $type = $confdesc->{$key}->{type};
2266
2267     if (!defined($value)) {
2268         die "got undefined value\n";
2269     }
2270
2271     if ($value =~ m/[\n\r]/) {
2272         die "property contains a line feed\n";
2273     }
2274
2275     if ($type eq 'boolean') {
2276         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2277         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2278         die "type check ('boolean') failed - got '$value'\n";
2279     } elsif ($type eq 'integer') {
2280         return int($1) if $value =~ m/^(\d+)$/;
2281         die "type check ('integer') failed - got '$value'\n";
2282     } elsif ($type eq 'number') {
2283         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2284         die "type check ('number') failed - got '$value'\n";
2285     } elsif ($type eq 'string') {
2286         if (my $fmt = $confdesc->{$key}->{format}) {
2287             PVE::JSONSchema::check_format($fmt, $value);
2288             return $value;
2289         }
2290         $value =~ s/^\"(.*)\"$/$1/;
2291         return $value;
2292     } else {
2293         die "internal error"
2294     }
2295 }
2296
2297 sub destroy_vm {
2298     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2299
2300     my $conf = PVE::QemuConfig->load_config($vmid);
2301
2302     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2303
2304     if ($conf->{template}) {
2305         # check if any base image is still used by a linked clone
2306         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2307                 my ($ds, $drive) = @_;
2308                 return if drive_is_cdrom($drive);
2309
2310                 my $volid = $drive->{file};
2311                 return if !$volid || $volid =~ m|^/|;
2312
2313                 die "base volume '$volid' is still in use by linked cloned\n"
2314                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2315
2316         });
2317     }
2318
2319     my $volids = {};
2320     my $remove_owned_drive = sub {
2321         my ($ds, $drive) = @_;
2322         return if drive_is_cdrom($drive, 1);
2323
2324         my $volid = $drive->{file};
2325         return if !$volid || $volid =~ m|^/|;
2326         return if $volids->{$volid};
2327
2328         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2329         return if !$path || !$owner || ($owner != $vmid);
2330
2331         $volids->{$volid} = 1;
2332         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2333         warn "Could not remove disk '$volid', check manually: $@" if $@;
2334     };
2335
2336     # only remove disks owned by this VM (referenced in the config)
2337     my $include_opts = {
2338         include_unused => 1,
2339         extra_keys => ['vmstate'],
2340     };
2341     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2342
2343     for my $snap (values %{$conf->{snapshots}}) {
2344         next if !defined($snap->{vmstate});
2345         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2346         next if !defined($drive);
2347         $remove_owned_drive->('vmstate', $drive);
2348     }
2349
2350     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2351
2352     if ($purge_unreferenced) { # also remove unreferenced disk
2353         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2354         PVE::Storage::foreach_volid($vmdisks, sub {
2355             my ($volid, $sid, $volname, $d) = @_;
2356             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2357             warn $@ if $@;
2358         });
2359     }
2360
2361     if (defined $replacement_conf) {
2362         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2363     } else {
2364         PVE::QemuConfig->destroy_config($vmid);
2365     }
2366 }
2367
2368 sub parse_vm_config {
2369     my ($filename, $raw, $strict) = @_;
2370
2371     return if !defined($raw);
2372
2373     my $res = {
2374         digest => Digest::SHA::sha1_hex($raw),
2375         snapshots => {},
2376         pending => {},
2377     };
2378
2379     my $handle_error = sub {
2380         my ($msg) = @_;
2381
2382         if ($strict) {
2383             die $msg;
2384         } else {
2385             warn $msg;
2386         }
2387     };
2388
2389     $filename =~ m|/qemu-server/(\d+)\.conf$|
2390         || die "got strange filename '$filename'";
2391
2392     my $vmid = $1;
2393
2394     my $conf = $res;
2395     my $descr;
2396     my $section = '';
2397
2398     my @lines = split(/\n/, $raw);
2399     foreach my $line (@lines) {
2400         next if $line =~ m/^\s*$/;
2401
2402         if ($line =~ m/^\[PENDING\]\s*$/i) {
2403             $section = 'pending';
2404             if (defined($descr)) {
2405                 $descr =~ s/\s+$//;
2406                 $conf->{description} = $descr;
2407             }
2408             $descr = undef;
2409             $conf = $res->{$section} = {};
2410             next;
2411
2412         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2413             $section = $1;
2414             if (defined($descr)) {
2415                 $descr =~ s/\s+$//;
2416                 $conf->{description} = $descr;
2417             }
2418             $descr = undef;
2419             $conf = $res->{snapshots}->{$section} = {};
2420             next;
2421         }
2422
2423         if ($line =~ m/^\#(.*)$/) {
2424             $descr = '' if !defined($descr);
2425             $descr .= PVE::Tools::decode_text($1) . "\n";
2426             next;
2427         }
2428
2429         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2430             $descr = '' if !defined($descr);
2431             $descr .= PVE::Tools::decode_text($2);
2432         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2433             $conf->{snapstate} = $1;
2434         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2435             my $key = $1;
2436             my $value = $2;
2437             $conf->{$key} = $value;
2438         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2439             my $value = $1;
2440             if ($section eq 'pending') {
2441                 $conf->{delete} = $value; # we parse this later
2442             } else {
2443                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2444             }
2445         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2446             my $key = $1;
2447             my $value = $2;
2448             eval { $value = check_type($key, $value); };
2449             if ($@) {
2450                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2451             } else {
2452                 $key = 'ide2' if $key eq 'cdrom';
2453                 my $fmt = $confdesc->{$key}->{format};
2454                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2455                     my $v = parse_drive($key, $value);
2456                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2457                         $v->{file} = $volid;
2458                         $value = print_drive($v);
2459                     } else {
2460                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2461                         next;
2462                     }
2463                 }
2464
2465                 $conf->{$key} = $value;
2466             }
2467         } else {
2468             $handle_error->("vm $vmid - unable to parse config: $line\n");
2469         }
2470     }
2471
2472     if (defined($descr)) {
2473         $descr =~ s/\s+$//;
2474         $conf->{description} = $descr;
2475     }
2476     delete $res->{snapstate}; # just to be sure
2477
2478     return $res;
2479 }
2480
2481 sub write_vm_config {
2482     my ($filename, $conf) = @_;
2483
2484     delete $conf->{snapstate}; # just to be sure
2485
2486     if ($conf->{cdrom}) {
2487         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2488         $conf->{ide2} = $conf->{cdrom};
2489         delete $conf->{cdrom};
2490     }
2491
2492     # we do not use 'smp' any longer
2493     if ($conf->{sockets}) {
2494         delete $conf->{smp};
2495     } elsif ($conf->{smp}) {
2496         $conf->{sockets} = $conf->{smp};
2497         delete $conf->{cores};
2498         delete $conf->{smp};
2499     }
2500
2501     my $used_volids = {};
2502
2503     my $cleanup_config = sub {
2504         my ($cref, $pending, $snapname) = @_;
2505
2506         foreach my $key (keys %$cref) {
2507             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2508                 $key eq 'snapstate' || $key eq 'pending';
2509             my $value = $cref->{$key};
2510             if ($key eq 'delete') {
2511                 die "propertry 'delete' is only allowed in [PENDING]\n"
2512                     if !$pending;
2513                 # fixme: check syntax?
2514                 next;
2515             }
2516             eval { $value = check_type($key, $value); };
2517             die "unable to parse value of '$key' - $@" if $@;
2518
2519             $cref->{$key} = $value;
2520
2521             if (!$snapname && is_valid_drivename($key)) {
2522                 my $drive = parse_drive($key, $value);
2523                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2524             }
2525         }
2526     };
2527
2528     &$cleanup_config($conf);
2529
2530     &$cleanup_config($conf->{pending}, 1);
2531
2532     foreach my $snapname (keys %{$conf->{snapshots}}) {
2533         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2534         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2535     }
2536
2537     # remove 'unusedX' settings if we re-add a volume
2538     foreach my $key (keys %$conf) {
2539         my $value = $conf->{$key};
2540         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2541             delete $conf->{$key};
2542         }
2543     }
2544
2545     my $generate_raw_config = sub {
2546         my ($conf, $pending) = @_;
2547
2548         my $raw = '';
2549
2550         # add description as comment to top of file
2551         if (defined(my $descr = $conf->{description})) {
2552             if ($descr) {
2553                 foreach my $cl (split(/\n/, $descr)) {
2554                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2555                 }
2556             } else {
2557                 $raw .= "#\n" if $pending;
2558             }
2559         }
2560
2561         foreach my $key (sort keys %$conf) {
2562             next if $key =~ /^(digest|description|pending|snapshots)$/;
2563             $raw .= "$key: $conf->{$key}\n";
2564         }
2565         return $raw;
2566     };
2567
2568     my $raw = &$generate_raw_config($conf);
2569
2570     if (scalar(keys %{$conf->{pending}})){
2571         $raw .= "\n[PENDING]\n";
2572         $raw .= &$generate_raw_config($conf->{pending}, 1);
2573     }
2574
2575     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2576         $raw .= "\n[$snapname]\n";
2577         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2578     }
2579
2580     return $raw;
2581 }
2582
2583 sub load_defaults {
2584
2585     my $res = {};
2586
2587     # we use static defaults from our JSON schema configuration
2588     foreach my $key (keys %$confdesc) {
2589         if (defined(my $default = $confdesc->{$key}->{default})) {
2590             $res->{$key} = $default;
2591         }
2592     }
2593
2594     return $res;
2595 }
2596
2597 sub config_list {
2598     my $vmlist = PVE::Cluster::get_vmlist();
2599     my $res = {};
2600     return $res if !$vmlist || !$vmlist->{ids};
2601     my $ids = $vmlist->{ids};
2602     my $nodename = nodename();
2603
2604     foreach my $vmid (keys %$ids) {
2605         my $d = $ids->{$vmid};
2606         next if !$d->{node} || $d->{node} ne $nodename;
2607         next if !$d->{type} || $d->{type} ne 'qemu';
2608         $res->{$vmid}->{exists} = 1;
2609     }
2610     return $res;
2611 }
2612
2613 # test if VM uses local resources (to prevent migration)
2614 sub check_local_resources {
2615     my ($conf, $noerr) = @_;
2616
2617     my @loc_res = ();
2618
2619     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2620     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2621
2622     push @loc_res, "ivshmem" if $conf->{ivshmem};
2623
2624     foreach my $k (keys %$conf) {
2625         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2626         # sockets are safe: they will recreated be on the target side post-migrate
2627         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2628         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2629     }
2630
2631     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2632
2633     return \@loc_res;
2634 }
2635
2636 # check if used storages are available on all nodes (use by migrate)
2637 sub check_storage_availability {
2638     my ($storecfg, $conf, $node) = @_;
2639
2640     PVE::QemuConfig->foreach_volume($conf, sub {
2641         my ($ds, $drive) = @_;
2642
2643         my $volid = $drive->{file};
2644         return if !$volid;
2645
2646         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2647         return if !$sid;
2648
2649         # check if storage is available on both nodes
2650         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2651         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2652
2653         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2654
2655         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2656             if !$scfg->{content}->{$vtype};
2657    });
2658 }
2659
2660 # list nodes where all VM images are available (used by has_feature API)
2661 sub shared_nodes {
2662     my ($conf, $storecfg) = @_;
2663
2664     my $nodelist = PVE::Cluster::get_nodelist();
2665     my $nodehash = { map { $_ => 1 } @$nodelist };
2666     my $nodename = nodename();
2667
2668     PVE::QemuConfig->foreach_volume($conf, sub {
2669         my ($ds, $drive) = @_;
2670
2671         my $volid = $drive->{file};
2672         return if !$volid;
2673
2674         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2675         if ($storeid) {
2676             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2677             if ($scfg->{disable}) {
2678                 $nodehash = {};
2679             } elsif (my $avail = $scfg->{nodes}) {
2680                 foreach my $node (keys %$nodehash) {
2681                     delete $nodehash->{$node} if !$avail->{$node};
2682                 }
2683             } elsif (!$scfg->{shared}) {
2684                 foreach my $node (keys %$nodehash) {
2685                     delete $nodehash->{$node} if $node ne $nodename
2686                 }
2687             }
2688         }
2689     });
2690
2691     return $nodehash
2692 }
2693
2694 sub check_local_storage_availability {
2695     my ($conf, $storecfg) = @_;
2696
2697     my $nodelist = PVE::Cluster::get_nodelist();
2698     my $nodehash = { map { $_ => {} } @$nodelist };
2699
2700     PVE::QemuConfig->foreach_volume($conf, sub {
2701         my ($ds, $drive) = @_;
2702
2703         my $volid = $drive->{file};
2704         return if !$volid;
2705
2706         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2707         if ($storeid) {
2708             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2709
2710             if ($scfg->{disable}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2713                 }
2714             } elsif (my $avail = $scfg->{nodes}) {
2715                 foreach my $node (keys %$nodehash) {
2716                     if (!$avail->{$node}) {
2717                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2718                     }
2719                 }
2720             }
2721         }
2722     });
2723
2724     foreach my $node (values %$nodehash) {
2725         if (my $unavail = $node->{unavailable_storages}) {
2726             $node->{unavailable_storages} = [ sort keys %$unavail ];
2727         }
2728     }
2729
2730     return $nodehash
2731 }
2732
2733 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2734 sub check_running {
2735     my ($vmid, $nocheck, $node) = @_;
2736
2737     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2738     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2739 }
2740
2741 sub vzlist {
2742
2743     my $vzlist = config_list();
2744
2745     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2746
2747     while (defined(my $de = $fd->read)) {
2748         next if $de !~ m/^(\d+)\.pid$/;
2749         my $vmid = $1;
2750         next if !defined($vzlist->{$vmid});
2751         if (my $pid = check_running($vmid)) {
2752             $vzlist->{$vmid}->{pid} = $pid;
2753         }
2754     }
2755
2756     return $vzlist;
2757 }
2758
2759 our $vmstatus_return_properties = {
2760     vmid => get_standard_option('pve-vmid'),
2761     status => {
2762         description => "Qemu process status.",
2763         type => 'string',
2764         enum => ['stopped', 'running'],
2765     },
2766     maxmem => {
2767         description => "Maximum memory in bytes.",
2768         type => 'integer',
2769         optional => 1,
2770         renderer => 'bytes',
2771     },
2772     maxdisk => {
2773         description => "Root disk size in bytes.",
2774         type => 'integer',
2775         optional => 1,
2776         renderer => 'bytes',
2777     },
2778     name => {
2779         description => "VM name.",
2780         type => 'string',
2781         optional => 1,
2782     },
2783     qmpstatus => {
2784         description => "Qemu QMP agent status.",
2785         type => 'string',
2786         optional => 1,
2787     },
2788     pid => {
2789         description => "PID of running qemu process.",
2790         type => 'integer',
2791         optional => 1,
2792     },
2793     uptime => {
2794         description => "Uptime.",
2795         type => 'integer',
2796         optional => 1,
2797         renderer => 'duration',
2798     },
2799     cpus => {
2800         description => "Maximum usable CPUs.",
2801         type => 'number',
2802         optional => 1,
2803     },
2804     lock => {
2805         description => "The current config lock, if any.",
2806         type => 'string',
2807         optional => 1,
2808     },
2809     tags => {
2810         description  => "The current configured tags, if any",
2811         type => 'string',
2812         optional => 1,
2813     },
2814     'running-machine' => {
2815         description => "The currently running machine type (if running).",
2816         type => 'string',
2817         optional => 1,
2818     },
2819     'running-qemu' => {
2820         description => "The currently running QEMU version (if running).",
2821         type => 'string',
2822         optional => 1,
2823     },
2824 };
2825
2826 my $last_proc_pid_stat;
2827
2828 # get VM status information
2829 # This must be fast and should not block ($full == false)
2830 # We only query KVM using QMP if $full == true (this can be slow)
2831 sub vmstatus {
2832     my ($opt_vmid, $full) = @_;
2833
2834     my $res = {};
2835
2836     my $storecfg = PVE::Storage::config();
2837
2838     my $list = vzlist();
2839     my $defaults = load_defaults();
2840
2841     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2842
2843     my $cpucount = $cpuinfo->{cpus} || 1;
2844
2845     foreach my $vmid (keys %$list) {
2846         next if $opt_vmid && ($vmid ne $opt_vmid);
2847
2848         my $conf = PVE::QemuConfig->load_config($vmid);
2849
2850         my $d = { vmid => int($vmid) };
2851         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2852
2853         # fixme: better status?
2854         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2855
2856         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2857         if (defined($size)) {
2858             $d->{disk} = 0; # no info available
2859             $d->{maxdisk} = $size;
2860         } else {
2861             $d->{disk} = 0;
2862             $d->{maxdisk} = 0;
2863         }
2864
2865         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2866             * ($conf->{cores} || $defaults->{cores});
2867         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2868         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2869
2870         $d->{name} = $conf->{name} || "VM $vmid";
2871         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2872             : $defaults->{memory}*(1024*1024);
2873
2874         if ($conf->{balloon}) {
2875             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2876             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2877                 : $defaults->{shares};
2878         }
2879
2880         $d->{uptime} = 0;
2881         $d->{cpu} = 0;
2882         $d->{mem} = 0;
2883
2884         $d->{netout} = 0;
2885         $d->{netin} = 0;
2886
2887         $d->{diskread} = 0;
2888         $d->{diskwrite} = 0;
2889
2890         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2891
2892         $d->{serial} = 1 if conf_has_serial($conf);
2893         $d->{lock} = $conf->{lock} if $conf->{lock};
2894         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2895
2896         $res->{$vmid} = $d;
2897     }
2898
2899     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2900     foreach my $dev (keys %$netdev) {
2901         next if $dev !~ m/^tap([1-9]\d*)i/;
2902         my $vmid = $1;
2903         my $d = $res->{$vmid};
2904         next if !$d;
2905
2906         $d->{netout} += $netdev->{$dev}->{receive};
2907         $d->{netin} += $netdev->{$dev}->{transmit};
2908
2909         if ($full) {
2910             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2911             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2912         }
2913
2914     }
2915
2916     my $ctime = gettimeofday;
2917
2918     foreach my $vmid (keys %$list) {
2919
2920         my $d = $res->{$vmid};
2921         my $pid = $d->{pid};
2922         next if !$pid;
2923
2924         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2925         next if !$pstat; # not running
2926
2927         my $used = $pstat->{utime} + $pstat->{stime};
2928
2929         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2930
2931         if ($pstat->{vsize}) {
2932             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2933         }
2934
2935         my $old = $last_proc_pid_stat->{$pid};
2936         if (!$old) {
2937             $last_proc_pid_stat->{$pid} = {
2938                 time => $ctime,
2939                 used => $used,
2940                 cpu => 0,
2941             };
2942             next;
2943         }
2944
2945         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2946
2947         if ($dtime > 1000) {
2948             my $dutime = $used -  $old->{used};
2949
2950             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2951             $last_proc_pid_stat->{$pid} = {
2952                 time => $ctime,
2953                 used => $used,
2954                 cpu => $d->{cpu},
2955             };
2956         } else {
2957             $d->{cpu} = $old->{cpu};
2958         }
2959     }
2960
2961     return $res if !$full;
2962
2963     my $qmpclient = PVE::QMPClient->new();
2964
2965     my $ballooncb = sub {
2966         my ($vmid, $resp) = @_;
2967
2968         my $info = $resp->{'return'};
2969         return if !$info->{max_mem};
2970
2971         my $d = $res->{$vmid};
2972
2973         # use memory assigned to VM
2974         $d->{maxmem} = $info->{max_mem};
2975         $d->{balloon} = $info->{actual};
2976
2977         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2978             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2979             $d->{freemem} = $info->{free_mem};
2980         }
2981
2982         $d->{ballooninfo} = $info;
2983     };
2984
2985     my $blockstatscb = sub {
2986         my ($vmid, $resp) = @_;
2987         my $data = $resp->{'return'} || [];
2988         my $totalrdbytes = 0;
2989         my $totalwrbytes = 0;
2990
2991         for my $blockstat (@$data) {
2992             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2993             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2994
2995             $blockstat->{device} =~ s/drive-//;
2996             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2997         }
2998         $res->{$vmid}->{diskread} = $totalrdbytes;
2999         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3000     };
3001
3002     my $machinecb = sub {
3003         my ($vmid, $resp) = @_;
3004         my $data = $resp->{'return'} || [];
3005
3006         $res->{$vmid}->{'running-machine'} =
3007             PVE::QemuServer::Machine::current_from_query_machines($data);
3008     };
3009
3010     my $versioncb = sub {
3011         my ($vmid, $resp) = @_;
3012         my $data = $resp->{'return'} // {};
3013         my $version = 'unknown';
3014
3015         if (my $v = $data->{qemu}) {
3016             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3017         }
3018
3019         $res->{$vmid}->{'running-qemu'} = $version;
3020     };
3021
3022     my $statuscb = sub {
3023         my ($vmid, $resp) = @_;
3024
3025         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3026         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3027         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3028         # this fails if ballon driver is not loaded, so this must be
3029         # the last commnand (following command are aborted if this fails).
3030         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3031
3032         my $status = 'unknown';
3033         if (!defined($status = $resp->{'return'}->{status})) {
3034             warn "unable to get VM status\n";
3035             return;
3036         }
3037
3038         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3039     };
3040
3041     foreach my $vmid (keys %$list) {
3042         next if $opt_vmid && ($vmid ne $opt_vmid);
3043         next if !$res->{$vmid}->{pid}; # not running
3044         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3045     }
3046
3047     $qmpclient->queue_execute(undef, 2);
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         next if !$res->{$vmid}->{pid}; #not running
3052
3053         # we can't use the $qmpclient since it might have already aborted on
3054         # 'query-balloon', but this might also fail for older versions...
3055         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3056         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3057     }
3058
3059     foreach my $vmid (keys %$list) {
3060         next if $opt_vmid && ($vmid ne $opt_vmid);
3061         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3062     }
3063
3064     return $res;
3065 }
3066
3067 sub conf_has_serial {
3068     my ($conf) = @_;
3069
3070     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3071         if ($conf->{"serial$i"}) {
3072             return 1;
3073         }
3074     }
3075
3076     return 0;
3077 }
3078
3079 sub conf_has_audio {
3080     my ($conf, $id) = @_;
3081
3082     $id //= 0;
3083     my $audio = $conf->{"audio$id"};
3084     return if !defined($audio);
3085
3086     my $audioproperties = parse_property_string($audio_fmt, $audio);
3087     my $audiodriver = $audioproperties->{driver} // 'spice';
3088
3089     return {
3090         dev => $audioproperties->{device},
3091         dev_id => "audiodev$id",
3092         backend => $audiodriver,
3093         backend_id => "$audiodriver-backend${id}",
3094     };
3095 }
3096
3097 sub audio_devs {
3098     my ($audio, $audiopciaddr, $machine_version) = @_;
3099
3100     my $devs = [];
3101
3102     my $id = $audio->{dev_id};
3103     my $audiodev = "";
3104     if (min_version($machine_version, 4, 2)) {
3105         $audiodev = ",audiodev=$audio->{backend_id}";
3106     }
3107
3108     if ($audio->{dev} eq 'AC97') {
3109         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3110     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3111         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3112         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3113         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3114     } else {
3115         die "unkown audio device '$audio->{dev}', implement me!";
3116     }
3117
3118     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3119
3120     return $devs;
3121 }
3122
3123 sub get_tpm_paths {
3124     my ($vmid) = @_;
3125     return {
3126         socket => "/var/run/qemu-server/$vmid.swtpm",
3127         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3128     };
3129 }
3130
3131 sub add_tpm_device {
3132     my ($vmid, $devices, $conf) = @_;
3133
3134     return if !$conf->{tpmstate0};
3135
3136     my $paths = get_tpm_paths($vmid);
3137
3138     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3139     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3140     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3141 }
3142
3143 sub start_swtpm {
3144     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3145
3146     return if !$tpmdrive;
3147
3148     my $state;
3149     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3150     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3151     if ($storeid) {
3152         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3153     } else {
3154         $state = $tpm->{file};
3155     }
3156
3157     my $paths = get_tpm_paths($vmid);
3158
3159     # during migration, we will get state from remote
3160     #
3161     if (!$migration) {
3162         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3163         my $setup_cmd = [
3164             "swtpm_setup",
3165             "--tpmstate",
3166             "file://$state",
3167             "--createek",
3168             "--create-ek-cert",
3169             "--create-platform-cert",
3170             "--lock-nvram",
3171             "--config",
3172             "/etc/swtpm_setup.conf", # do not use XDG configs
3173             "--runas",
3174             "0", # force creation as root, error if not possible
3175             "--not-overwrite", # ignore existing state, do not modify
3176         ];
3177
3178         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3179         # TPM 2.0 supports ECC crypto, use if possible
3180         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3181
3182         run_command($setup_cmd, outfunc => sub {
3183             print "swtpm_setup: $1\n";
3184         });
3185     }
3186
3187     my $emulator_cmd = [
3188         "swtpm",
3189         "socket",
3190         "--tpmstate",
3191         "backend-uri=file://$state,mode=0600",
3192         "--ctrl",
3193         "type=unixio,path=$paths->{socket},mode=0600",
3194         "--pid",
3195         "file=$paths->{pid}",
3196         "--terminate", # terminate on QEMU disconnect
3197         "--daemon",
3198     ];
3199     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3200     run_command($emulator_cmd, outfunc => sub { print $1; });
3201
3202     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3203     while (! -e $paths->{pid}) {
3204         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3205         usleep(50_000);
3206     }
3207
3208     # return untainted PID of swtpm daemon so it can be killed on error
3209     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3210     return $1;
3211 }
3212
3213 sub vga_conf_has_spice {
3214     my ($vga) = @_;
3215
3216     my $vgaconf = parse_vga($vga);
3217     my $vgatype = $vgaconf->{type};
3218     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3219
3220     return $1 || 1;
3221 }
3222
3223 sub is_native($) {
3224     my ($arch) = @_;
3225     return get_host_arch() eq $arch;
3226 }
3227
3228 sub get_vm_arch {
3229     my ($conf) = @_;
3230     return $conf->{arch} // get_host_arch();
3231 }
3232
3233 my $default_machines = {
3234     x86_64 => 'pc',
3235     aarch64 => 'virt',
3236 };
3237
3238 sub get_installed_machine_version {
3239     my ($kvmversion) = @_;
3240     $kvmversion = kvm_user_version() if !defined($kvmversion);
3241     $kvmversion =~ m/^(\d+\.\d+)/;
3242     return $1;
3243 }
3244
3245 sub windows_get_pinned_machine_version {
3246     my ($machine, $base_version, $kvmversion) = @_;
3247
3248     my $pin_version = $base_version;
3249     if (!defined($base_version) ||
3250         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3251     ) {
3252         $pin_version = get_installed_machine_version($kvmversion);
3253     }
3254     if (!$machine || $machine eq 'pc') {
3255         $machine = "pc-i440fx-$pin_version";
3256     } elsif ($machine eq 'q35') {
3257         $machine = "pc-q35-$pin_version";
3258     } elsif ($machine eq 'virt') {
3259         $machine = "virt-$pin_version";
3260     } else {
3261         warn "unknown machine type '$machine', not touching that!\n";
3262     }
3263
3264     return $machine;
3265 }
3266
3267 sub get_vm_machine {
3268     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3269
3270     my $machine = $forcemachine || $conf->{machine};
3271
3272     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3273         $kvmversion //= kvm_user_version();
3274         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3275         # layout which confuses windows quite a bit and may result in various regressions..
3276         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3277         if (windows_version($conf->{ostype})) {
3278             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3279         }
3280         $arch //= 'x86_64';
3281         $machine ||= $default_machines->{$arch};
3282         if ($add_pve_version) {
3283             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3284             $machine .= "+pve$pvever";
3285         }
3286     }
3287
3288     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3289         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3290         $machine = $1 if $is_pxe;
3291
3292         # for version-pinned machines that do not include a pve-version (e.g.
3293         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3294         $machine .= '+pve0';
3295
3296         $machine .= '.pxe' if $is_pxe;
3297     }
3298
3299     return $machine;
3300 }
3301
3302 sub get_ovmf_files($$$) {
3303     my ($arch, $efidisk, $smm) = @_;
3304
3305     my $types = $OVMF->{$arch}
3306         or die "no OVMF images known for architecture '$arch'\n";
3307
3308     my $type = 'default';
3309     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3310         $type = $smm ? "4m" : "4m-no-smm";
3311         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3312     }
3313
3314     return $types->{$type}->@*;
3315 }
3316
3317 my $Arch2Qemu = {
3318     aarch64 => '/usr/bin/qemu-system-aarch64',
3319     x86_64 => '/usr/bin/qemu-system-x86_64',
3320 };
3321 sub get_command_for_arch($) {
3322     my ($arch) = @_;
3323     return '/usr/