PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::HA::Config;
  38 use PVE::Mapping::PCI;
  39 use PVE::Mapping::USB;
  40 use PVE::INotify;
  41 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  42 use PVE::ProcFSTools;
  43 use PVE::PBSClient;
  44 use PVE::RESTEnvironment qw(log_warn);
  45 use PVE::RPCEnvironment;
  46 use PVE::Storage;
  47 use PVE::SysFSTools;
  48 use PVE::Systemd;
  49 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  50
  51 use PVE::QMPClient;
  52 use PVE::QemuConfig;
  53 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  54 use PVE::QemuServer::Cloudinit;
  55 use PVE::QemuServer::CGroup;
  56 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
  57 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  58 use PVE::QemuServer::Machine;
  59 use PVE::QemuServer::Memory qw(get_current_memory);
  60 use PVE::QemuServer::Monitor qw(mon_cmd);
  61 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  62 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  63 use PVE::QemuServer::USB;
  64
  65 my $have_sdn;
  66 eval {
  67     require PVE::Network::SDN::Zones;
  68     require PVE::Network::SDN::Vnets;
  69     $have_sdn = 1;
  70 };
  71
  72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  73 my $OVMF = {
  74     x86_64 => {
  75         '4m-no-smm' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-no-smm-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         '4m' => [
  84             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  86         ],
  87         '4m-ms' => [
  88             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  89             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  90         ],
  91         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  92         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  93         # older backups and snapshot?
  94         default => [
  95             "$EDK2_FW_BASE/OVMF_CODE.fd",
  96             "$EDK2_FW_BASE/OVMF_VARS.fd",
  97         ],
  98     },
  99     aarch64 => {
 100         default => [
 101             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 102             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 103         ],
 104     },
 105 };
 106
 107 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 108
 109 # Note about locking: we use flock on the config file protect against concurent actions.
 110 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 112 # But you can ignore this kind of lock with the --skiplock flag.
 113
 114 cfs_register_file(
 115     '/qemu-server/',
 116     \&parse_vm_config,
 117     \&write_vm_config
 118 );
 119
 120 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 121     description => "Some command save/restore state from this location.",
 122     type => 'string',
 123     maxLength => 128,
 124     optional => 1,
 125 });
 126
 127 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 128         description => "Specifies the QEMU machine type.",
 129         type => 'string',
 130         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 131         maxLength => 40,
 132         optional => 1,
 133 });
 134
 135 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 136 my $nodename_cache;
 137 sub nodename {
 138     $nodename_cache //= PVE::INotify::nodename();
 139     return $nodename_cache;
 140 }
 141
 142 my $watchdog_fmt = {
 143     model => {
 144         default_key => 1,
 145         type => 'string',
 146         enum => [qw(i6300esb ib700)],
 147         description => "Watchdog type to emulate.",
 148         default => 'i6300esb',
 149         optional => 1,
 150     },
 151     action => {
 152         type => 'string',
 153         enum => [qw(reset shutdown poweroff pause debug none)],
 154         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 155         optional => 1,
 156     },
 157 };
 158 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 159
 160 my $agent_fmt = {
 161     enabled => {
 162         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 163         type => 'boolean',
 164         default => 0,
 165         default_key => 1,
 166     },
 167     fstrim_cloned_disks => {
 168         description => "Run fstrim after moving a disk or migrating the VM.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0,
 172     },
 173     'freeze-fs-on-backup' => {
 174         description => "Freeze/thaw guest filesystems on backup for consistency.",
 175         type => 'boolean',
 176         optional => 1,
 177         default => 1,
 178     },
 179     type => {
 180         description => "Select the agent type",
 181         type => 'string',
 182         default => 'virtio',
 183         optional => 1,
 184         enum => [qw(virtio isa)],
 185     },
 186 };
 187
 188 my $vga_fmt = {
 189     type => {
 190         description => "Select the VGA type.",
 191         type => 'string',
 192         default => 'std',
 193         optional => 1,
 194         default_key => 1,
 195         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 196     },
 197     memory => {
 198         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 199         type => 'integer',
 200         optional => 1,
 201         minimum => 4,
 202         maximum => 512,
 203     },
 204     clipboard => {
 205         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 206             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 207         type => 'string',
 208         enum => ['vnc'],
 209         optional => 1,
 210     },
 211 };
 212
 213 my $ivshmem_fmt = {
 214     size => {
 215         type => 'integer',
 216         minimum => 1,
 217         description => "The size of the file in MB.",
 218     },
 219     name => {
 220         type => 'string',
 221         pattern => '[a-zA-Z0-9\-]+',
 222         optional => 1,
 223         format_description => 'string',
 224         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 225     },
 226 };
 227
 228 my $audio_fmt = {
 229     device => {
 230         type => 'string',
 231         enum => [qw(ich9-intel-hda intel-hda AC97)],
 232         description =>  "Configure an audio device."
 233     },
 234     driver =>  {
 235         type => 'string',
 236         enum => ['spice', 'none'],
 237         default => 'spice',
 238         optional => 1,
 239         description => "Driver backend for the audio device."
 240     },
 241 };
 242
 243 my $spice_enhancements_fmt = {
 244     foldersharing => {
 245         type => 'boolean',
 246         optional => 1,
 247         default => '0',
 248         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 249     },
 250     videostreaming =>  {
 251         type => 'string',
 252         enum => ['off', 'all', 'filter'],
 253         default => 'off',
 254         optional => 1,
 255         description => "Enable video streaming. Uses compression for detected video streams."
 256     },
 257 };
 258
 259 my $rng_fmt = {
 260     source => {
 261         type => 'string',
 262         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 263         default_key => 1,
 264         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 265             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 266             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 267             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 268             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 269             ." a hardware RNG from the host.",
 270     },
 271     max_bytes => {
 272         type => 'integer',
 273         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 274             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 275             ." `0` to disable limiting (potentially dangerous!).",
 276         optional => 1,
 277
 278         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 279         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 280         # reading from /dev/urandom
 281         default => 1024,
 282     },
 283     period => {
 284         type => 'integer',
 285         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 286             ." the guest to retrieve another 'max_bytes' of entropy.",
 287         optional => 1,
 288         default => 1000,
 289     },
 290 };
 291
 292 my $meta_info_fmt = {
 293     'ctime' => {
 294         type => 'integer',
 295         description => "The guest creation timestamp as UNIX epoch time",
 296         minimum => 0,
 297         optional => 1,
 298     },
 299     'creation-qemu' => {
 300         type => 'string',
 301         description => "The QEMU (machine) version from the time this VM was created.",
 302         pattern => '\d+(\.\d+)+',
 303         optional => 1,
 304     },
 305 };
 306
 307 my $confdesc = {
 308     onboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Specifies whether a VM will be started during system bootup.",
 312         default => 0,
 313     },
 314     autostart => {
 315         optional => 1,
 316         type => 'boolean',
 317         description => "Automatic restart after crash (currently ignored).",
 318         default => 0,
 319     },
 320     hotplug => {
 321         optional => 1,
 322         type => 'string', format => 'pve-hotplug-features',
 323         description => "Selectively enable hotplug features. This is a comma separated list of"
 324             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 325             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 326             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 327             ." windows > 7.",
 328         default => 'network,disk,usb',
 329     },
 330     reboot => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 334         default => 1,
 335     },
 336     lock => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Lock/unlock the VM.",
 340         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 341     },
 342     cpulimit => {
 343         optional => 1,
 344         type => 'number',
 345         description => "Limit of CPU usage.",
 346         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 347             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 348         minimum => 0,
 349         maximum => 128,
 350         default => 0,
 351     },
 352     cpuunits => {
 353         optional => 1,
 354         type => 'integer',
 355         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 356         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 357             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 358             ." weights of all the other running VMs.",
 359         minimum => 1,
 360         maximum => 262144,
 361         default => 'cgroup v1: 1024, cgroup v2: 100',
 362     },
 363     memory => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Memory properties.",
 367         format => $PVE::QemuServer::Memory::memory_fmt
 368     },
 369     balloon => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 373         minimum => 0,
 374     },
 375     shares => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 379             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 380             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 381         minimum => 0,
 382         maximum => 50000,
 383         default => 1000,
 384     },
 385     keyboard => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Keyboard layout for VNC server. This option is generally not required and"
 389         ." is often better handled from within the guest OS.",
 390         enum => PVE::Tools::kvmkeymaplist(),
 391         default => undef,
 392     },
 393     name => {
 394         optional => 1,
 395         type => 'string', format => 'dns-name',
 396         description => "Set a name for the VM. Only used on the configuration web interface.",
 397     },
 398     scsihw => {
 399         optional => 1,
 400         type => 'string',
 401         description => "SCSI controller model",
 402         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 403         default => 'lsi',
 404     },
 405     description => {
 406         optional => 1,
 407         type => 'string',
 408         description => "Description for the VM. Shown in the web-interface VM's summary."
 409             ." This is saved as comment inside the configuration file.",
 410         maxLength => 1024 * 8,
 411     },
 412     ostype => {
 413         optional => 1,
 414         type => 'string',
 415         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 416         description => "Specify guest operating system.",
 417         verbose_description => <<EODESC,
 418 Specify guest operating system. This is used to enable special
 419 optimization/features for specific operating systems:
 420
 421 [horizontal]
 422 other;; unspecified OS
 423 wxp;; Microsoft Windows XP
 424 w2k;; Microsoft Windows 2000
 425 w2k3;; Microsoft Windows 2003
 426 w2k8;; Microsoft Windows 2008
 427 wvista;; Microsoft Windows Vista
 428 win7;; Microsoft Windows 7
 429 win8;; Microsoft Windows 8/2012/2012r2
 430 win10;; Microsoft Windows 10/2016/2019
 431 win11;; Microsoft Windows 11/2022
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6 - 6.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string', format => 'pve-qm-boot',
 440         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 441             ." key or 'legacy=' is deprecated.",
 442     },
 443     bootdisk => {
 444         optional => 1,
 445         type => 'string', format => 'pve-qm-bootdisk',
 446         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 447         pattern => '(ide|sata|scsi|virtio)\d+',
 448     },
 449     smp => {
 450         optional => 1,
 451         type => 'integer',
 452         description => "The number of CPUs. Please use option -sockets instead.",
 453         minimum => 1,
 454         default => 1,
 455     },
 456     sockets => {
 457         optional => 1,
 458         type => 'integer',
 459         description => "The number of CPU sockets.",
 460         minimum => 1,
 461         default => 1,
 462     },
 463     cores => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "The number of cores per socket.",
 467         minimum => 1,
 468         default => 1,
 469     },
 470     numa => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable NUMA.",
 474         default => 0,
 475     },
 476     hugepages => {
 477         optional => 1,
 478         type => 'string',
 479         description => "Enable/disable hugepages memory.",
 480         enum => [qw(any 2 1024)],
 481     },
 482     keephugepages => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 0,
 486         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 487             ." after VM shutdown and can be used for subsequent starts.",
 488     },
 489     vcpus => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Number of hotplugged vcpus.",
 493         minimum => 1,
 494         default => 0,
 495     },
 496     acpi => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable ACPI.",
 500         default => 1,
 501     },
 502     agent => {
 503         optional => 1,
 504         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 505         type => 'string',
 506         format => $agent_fmt,
 507     },
 508     kvm => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable KVM hardware virtualization.",
 512         default => 1,
 513     },
 514     tdf => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Enable/disable time drift fix.",
 518         default => 0,
 519     },
 520     localtime => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 524             ." the `ostype` indicates a Microsoft Windows OS.",
 525     },
 526     freeze => {
 527         optional => 1,
 528         type => 'boolean',
 529         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 530     },
 531     vga => {
 532         optional => 1,
 533         type => 'string', format => $vga_fmt,
 534         description => "Configure the VGA hardware.",
 535         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 536             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 537             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 538             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 539             ." display server. For win* OS you can select how many independent displays you want,"
 540             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 541             ." using a serial device as terminal.",
 542     },
 543     watchdog => {
 544         optional => 1,
 545         type => 'string', format => 'pve-qm-watchdog',
 546         description => "Create a virtual hardware watchdog device.",
 547         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 548             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 549             ." else the watchdog will reset the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are:"
 556             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 557         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 558         default => 'now',
 559     },
 560     startup =>  get_standard_option('pve-startup-order'),
 561     template => {
 562         optional => 1,
 563         type => 'boolean',
 564         description => "Enable/disable Template.",
 565         default => 0,
 566     },
 567     args => {
 568         optional => 1,
 569         type => 'string',
 570         description => "Arbitrary arguments passed to kvm.",
 571         verbose_description => <<EODESCR,
 572 Arbitrary arguments passed to kvm, for example:
 573
 574 args: -no-reboot -smbios 'type=0,vendor=FOO'
 575
 576 NOTE: this option is for experts only.
 577 EODESCR
 578     },
 579     tablet => {
 580         optional => 1,
 581         type => 'boolean',
 582         default => 1,
 583         description => "Enable/disable the USB tablet device.",
 584         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 585             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 586             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 587             ." may consider disabling this to save some context switches. This is turned off by"
 588             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => 'pve-vm-cpu-conf',
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally"
 630             ." for snapshots.",
 631     },
 632     vmstatestorage => get_standard_option('pve-storage-id', {
 633         description => "Default storage for VM state volumes/files.",
 634         optional => 1,
 635     }),
 636     runningmachine => get_standard_option('pve-qemu-machine', {
 637         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 638             ." for snapshots.",
 639     }),
 640     runningcpu => {
 641         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 642             ." internally for snapshots.",
 643         optional => 1,
 644         type => 'string',
 645         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 646         format_description => 'QEMU -cpu parameter'
 647     },
 648     machine => get_standard_option('pve-qemu-machine'),
 649     arch => {
 650         description => "Virtual processor architecture. Defaults to the host.",
 651         optional => 1,
 652         type => 'string',
 653         enum => [qw(x86_64 aarch64)],
 654     },
 655     smbios1 => {
 656         description => "Specify SMBIOS type 1 fields.",
 657         type => 'string', format => 'pve-qm-smbios1',
 658         maxLength => 512,
 659         optional => 1,
 660     },
 661     protection => {
 662         optional => 1,
 663         type => 'boolean',
 664         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 665             ." remove disk operations.",
 666         default => 0,
 667     },
 668     bios => {
 669         optional => 1,
 670         type => 'string',
 671         enum => [ qw(seabios ovmf) ],
 672         description => "Select BIOS implementation.",
 673         default => 'seabios',
 674     },
 675     vmgenid => {
 676         type => 'string',
 677         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 678         format_description => 'UUID',
 679         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 680             ." to disable explicitly.",
 681         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 682             ." value identifier to the guest OS. This allows to notify the guest operating system"
 683             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 684             ." execution or creation from a template). The guest operating system notices the"
 685             ." change, and is then able to react as appropriate by marking its copies of"
 686             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 687             ."Note that auto-creation only works when done through API/CLI create or update methods"
 688             .", but not when manually editing the config file.",
 689         default => "1 (autogenerated)",
 690         optional => 1,
 691     },
 692     hookscript => {
 693         type => 'string',
 694         format => 'pve-volume-id',
 695         optional => 1,
 696         description => "Script that will be executed during various steps in the vms lifetime.",
 697     },
 698     ivshmem => {
 699         type => 'string',
 700         format => $ivshmem_fmt,
 701         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 702             ." the host.",
 703         optional => 1,
 704     },
 705     audio0 => {
 706         type => 'string',
 707         format => $audio_fmt,
 708         description => "Configure a audio device, useful in combination with QXL/Spice.",
 709         optional => 1
 710     },
 711     spice_enhancements => {
 712         type => 'string',
 713         format => $spice_enhancements_fmt,
 714         description => "Configure additional enhancements for SPICE.",
 715         optional => 1
 716     },
 717     tags => {
 718         type => 'string', format => 'pve-tag-list',
 719         description => 'Tags of the VM. This is only meta information.',
 720         optional => 1,
 721     },
 722     rng0 => {
 723         type => 'string',
 724         format => $rng_fmt,
 725         description => "Configure a VirtIO-based Random Number Generator.",
 726         optional => 1,
 727     },
 728     meta => {
 729         type => 'string',
 730         format => $meta_info_fmt,
 731         description => "Some (read-only) meta-information about this guest.",
 732         optional => 1,
 733     },
 734     affinity => {
 735         type => 'string', format => 'pve-cpuset',
 736         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 737         optional => 1,
 738     },
 739 };
 740
 741 my $cicustom_fmt = {
 742     meta => {
 743         type => 'string',
 744         optional => 1,
 745         description => 'Specify a custom file containing all meta data passed to the VM via"
 746             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 747         format => 'pve-volume-id',
 748         format_description => 'volume',
 749     },
 750     network => {
 751         type => 'string',
 752         optional => 1,
 753         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 754         format => 'pve-volume-id',
 755         format_description => 'volume',
 756     },
 757     user => {
 758         type => 'string',
 759         optional => 1,
 760         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 761         format => 'pve-volume-id',
 762         format_description => 'volume',
 763     },
 764     vendor => {
 765         type => 'string',
 766         optional => 1,
 767         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 768         format => 'pve-volume-id',
 769         format_description => 'volume',
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 773
 774 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 775 my $confdesc_cloudinit = {
 776     citype => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'Specifies the cloud-init configuration format. The default depends on the'
 780             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 781             .' and `configdrive2` for windows.',
 782         enum => ['configdrive2', 'nocloud', 'opennebula'],
 783     },
 784     ciuser => {
 785         optional => 1,
 786         type => 'string',
 787         description => "cloud-init: User name to change ssh keys and password for instead of the"
 788             ." image's configured default user.",
 789     },
 790     cipassword => {
 791         optional => 1,
 792         type => 'string',
 793         description => 'cloud-init: Password to assign the user. Using this is generally not'
 794             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 795             .' support hashed passwords.',
 796     },
 797     ciupgrade => {
 798         optional => 1,
 799         type => 'boolean',
 800         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 801         default => 1,
 802     },
 803     cicustom => {
 804         optional => 1,
 805         type => 'string',
 806         description => 'cloud-init: Specify custom files to replace the automatically generated'
 807             .' ones at start.',
 808         format => 'pve-qm-cicustom',
 809     },
 810     searchdomain => {
 811         optional => 1,
 812         type => 'string',
 813         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 814             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 815             .' are set.',
 816     },
 817     nameserver => {
 818         optional => 1,
 819         type => 'string', format => 'address-list',
 820         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 821             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 822             .' are set.',
 823     },
 824     sshkeys => {
 825         optional => 1,
 826         type => 'string',
 827         format => 'urlencoded',
 828         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 829     },
 830 };
 831
 832 # what about other qemu settings ?
 833 #cpu => 'string',
 834 #machine => 'string',
 835 #fda => 'file',
 836 #fdb => 'file',
 837 #mtdblock => 'file',
 838 #sd => 'file',
 839 #pflash => 'file',
 840 #snapshot => 'bool',
 841 #bootp => 'file',
 842 ##tftp => 'dir',
 843 ##smb => 'dir',
 844 #kernel => 'file',
 845 #append => 'string',
 846 #initrd => 'file',
 847 ##soundhw => 'string',
 848
 849 while (my ($k, $v) = each %$confdesc) {
 850     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 851 }
 852
 853 my $MAX_NETS = 32;
 854 my $MAX_SERIAL_PORTS = 4;
 855 my $MAX_PARALLEL_PORTS = 3;
 856
 857 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 858     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 859 }
 860
 861 my $nic_model_list = [
 862     'e1000',
 863     'e1000-82540em',
 864     'e1000-82544gc',
 865     'e1000-82545em',
 866     'e1000e',
 867     'i82551',
 868     'i82557b',
 869     'i82559er',
 870     'ne2k_isa',
 871     'ne2k_pci',
 872     'pcnet',
 873     'rtl8139',
 874     'virtio',
 875     'vmxnet3',
 876 ];
 877 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 878
 879 my $net_fmt_bridge_descr = <<__EOD__;
 880 Bridge to attach the network device to. The Proxmox VE standard bridge
 881 is called 'vmbr0'.
 882
 883 If you do not specify a bridge, we create a kvm user (NATed) network
 884 device, which provides DHCP and DNS services. The following addresses
 885 are used:
 886
 887  10.0.2.2   Gateway
 888  10.0.2.3   DNS Server
 889  10.0.2.4   SMB Server
 890
 891 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 892 __EOD__
 893
 894 my $net_fmt = {
 895     macaddr  => get_standard_option('mac-addr', {
 896         description => "MAC address. That address must be unique withing your network. This is"
 897             ." automatically generated if not specified.",
 898     }),
 899     model => {
 900         type => 'string',
 901         description => "Network Card Model. The 'virtio' model provides the best performance with"
 902             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 903             ." best to use 'e1000'.",
 904         enum => $nic_model_list,
 905         default_key => 1,
 906     },
 907     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 908     bridge => get_standard_option('pve-bridge-id', {
 909         description => $net_fmt_bridge_descr,
 910         optional => 1,
 911     }),
 912     queues => {
 913         type => 'integer',
 914         minimum => 0, maximum => 64,
 915         description => 'Number of packet queues to be used on the device.',
 916         optional => 1,
 917     },
 918     rate => {
 919         type => 'number',
 920         minimum => 0,
 921         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 922         optional => 1,
 923     },
 924     tag => {
 925         type => 'integer',
 926         minimum => 1, maximum => 4094,
 927         description => 'VLAN tag to apply to packets on this interface.',
 928         optional => 1,
 929     },
 930     trunks => {
 931         type => 'string',
 932         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 933         description => 'VLAN trunks to pass through this interface.',
 934         format_description => 'vlanid[;vlanid...]',
 935         optional => 1,
 936     },
 937     firewall => {
 938         type => 'boolean',
 939         description => 'Whether this interface should be protected by the firewall.',
 940         optional => 1,
 941     },
 942     link_down => {
 943         type => 'boolean',
 944         description => 'Whether this interface should be disconnected (like pulling the plug).',
 945         optional => 1,
 946     },
 947     mtu => {
 948         type => 'integer',
 949         minimum => 1, maximum => 65520,
 950         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 951         optional => 1,
 952     },
 953 };
 954
 955 my $netdesc = {
 956     optional => 1,
 957     type => 'string', format => $net_fmt,
 958     description => "Specify network devices.",
 959 };
 960
 961 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 962
 963 my $ipconfig_fmt = {
 964     ip => {
 965         type => 'string',
 966         format => 'pve-ipv4-config',
 967         format_description => 'IPv4Format/CIDR',
 968         description => 'IPv4 address in CIDR format.',
 969         optional => 1,
 970         default => 'dhcp',
 971     },
 972     gw => {
 973         type => 'string',
 974         format => 'ipv4',
 975         format_description => 'GatewayIPv4',
 976         description => 'Default gateway for IPv4 traffic.',
 977         optional => 1,
 978         requires => 'ip',
 979     },
 980     ip6 => {
 981         type => 'string',
 982         format => 'pve-ipv6-config',
 983         format_description => 'IPv6Format/CIDR',
 984         description => 'IPv6 address in CIDR format.',
 985         optional => 1,
 986         default => 'dhcp',
 987     },
 988     gw6 => {
 989         type => 'string',
 990         format => 'ipv6',
 991         format_description => 'GatewayIPv6',
 992         description => 'Default gateway for IPv6 traffic.',
 993         optional => 1,
 994         requires => 'ip6',
 995     },
 996 };
 997 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 998 my $ipconfigdesc = {
 999     optional => 1,
1000     type => 'string', format => 'pve-qm-ipconfig',
1001     description => <<'EODESCR',
1002 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1003
1004 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1005
1006 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1007 gateway should be provided.
1008 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1009 cloud-init 19.4 or newer.
1010
1011 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1012 dhcp on IPv4.
1013 EODESCR
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1016
1017 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1018     $confdesc->{"net$i"} = $netdesc;
1019     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1020 }
1021
1022 foreach my $key (keys %$confdesc_cloudinit) {
1023     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1024 }
1025
1026 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1027 sub pve_verify_cpuset {
1028     my ($set_text, $noerr) = @_;
1029
1030     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1031
1032     if ($@) {
1033         return if $noerr;
1034         die "unable to parse cpuset option\n";
1035     }
1036
1037     return PVE::CpuSet->new($members)->short_string();
1038 }
1039
1040 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1041 sub verify_volume_id_or_qm_path {
1042     my ($volid, $noerr) = @_;
1043
1044     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1045
1046     return verify_volume_id_or_absolute_path($volid, $noerr);
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1050 sub verify_volume_id_or_absolute_path {
1051     my ($volid, $noerr) = @_;
1052
1053     return $volid if $volid =~ m|^/|;
1054
1055     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1056     if ($@) {
1057         return if $noerr;
1058         die $@;
1059     }
1060     return $volid;
1061 }
1062
1063 my $serialdesc = {
1064         optional => 1,
1065         type => 'string',
1066         pattern => '(/dev/.+|socket)',
1067         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1068         verbose_description =>  <<EODESCR,
1069 Create a serial device inside the VM (n is 0 to 3), and pass through a
1070 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1071 host side (use 'qm terminal' to open a terminal connection).
1072
1073 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1074 use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 my $paralleldesc= {
1081         optional => 1,
1082         type => 'string',
1083         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1084         description =>  "Map host parallel devices (n is 0 to 2).",
1085         verbose_description =>  <<EODESCR,
1086 Map host parallel devices (n is 0 to 2).
1087
1088 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1089 machines - use with special care.
1090
1091 CAUTION: Experimental! User reported problems with this option.
1092 EODESCR
1093 };
1094
1095 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1096     $confdesc->{"parallel$i"} = $paralleldesc;
1097 }
1098
1099 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1100     $confdesc->{"serial$i"} = $serialdesc;
1101 }
1102
1103 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1104     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1105 }
1106
1107 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1108     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1109 }
1110
1111 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1112     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1113 }
1114
1115 my $boot_fmt = {
1116     legacy => {
1117         optional => 1,
1118         default_key => 1,
1119         type => 'string',
1120         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1121                      . " Deprecated, use 'order=' instead.",
1122         pattern => '[acdn]{1,4}',
1123         format_description => "[acdn]{1,4}",
1124
1125         # note: this is also the fallback if boot: is not given at all
1126         default => 'cdn',
1127     },
1128     order => {
1129         optional => 1,
1130         type => 'string',
1131         format => 'pve-qm-bootdev-list',
1132         format_description => "device[;device...]",
1133         description => <<EODESC,
1134 The guest will attempt to boot from devices in the order they appear here.
1135
1136 Disks, optical drives and passed-through storage USB devices will be directly
1137 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1138 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1139
1140 Note that only devices in this list will be marked as bootable and thus loaded
1141 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1142 (e.g. software-raid), you need to specify all of them here.
1143
1144 Overrides the deprecated 'legacy=[acdn]*' value when given.
1145 EODESC
1146     },
1147 };
1148 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1149
1150 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1151 sub verify_bootdev {
1152     my ($dev, $noerr) = @_;
1153
1154     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1155     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1156
1157     my $check = sub {
1158         my ($base) = @_;
1159         return 0 if $dev !~ m/^$base\d+$/;
1160         return 0 if !$confdesc->{$dev};
1161         return 1;
1162     };
1163
1164     return $dev if $check->("net");
1165     return $dev if $check->("usb");
1166     return $dev if $check->("hostpci");
1167
1168     return if $noerr;
1169     die "invalid boot device '$dev'\n";
1170 }
1171
1172 sub print_bootorder {
1173     my ($devs) = @_;
1174     return "" if !@$devs;
1175     my $data = { order => join(';', @$devs) };
1176     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1177 }
1178
1179 my $kvm_api_version = 0;
1180
1181 sub kvm_version {
1182     return $kvm_api_version if $kvm_api_version;
1183
1184     open my $fh, '<', '/dev/kvm' or return;
1185
1186     # 0xae00 => KVM_GET_API_VERSION
1187     $kvm_api_version = ioctl($fh, 0xae00, 0);
1188     close($fh);
1189
1190     return $kvm_api_version;
1191 }
1192
1193 my $kvm_user_version = {};
1194 my $kvm_mtime = {};
1195
1196 sub kvm_user_version {
1197     my ($binary) = @_;
1198
1199     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1200     my $st = stat($binary);
1201
1202     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1203     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1204         $cachedmtime == $st->mtime;
1205
1206     $kvm_user_version->{$binary} = 'unknown';
1207     $kvm_mtime->{$binary} = $st->mtime;
1208
1209     my $code = sub {
1210         my $line = shift;
1211         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1212             $kvm_user_version->{$binary} = $2;
1213         }
1214     };
1215
1216     eval { run_command([$binary, '--version'], outfunc => $code); };
1217     warn $@ if $@;
1218
1219     return $kvm_user_version->{$binary};
1220
1221 }
1222 my sub extract_version {
1223     my ($machine_type, $version) = @_;
1224     $version = kvm_user_version() if !defined($version);
1225     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1226 }
1227
1228 sub kernel_has_vhost_net {
1229     return -c '/dev/vhost-net';
1230 }
1231
1232 sub option_exists {
1233     my $key = shift;
1234     return defined($confdesc->{$key});
1235 }
1236
1237 my $cdrom_path;
1238 sub get_cdrom_path {
1239
1240     return $cdrom_path if defined($cdrom_path);
1241
1242     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1243
1244     if (!defined($cdrom_path)) {
1245         log_warn("no physical CD-ROM available, ignoring");
1246         $cdrom_path = '';
1247     }
1248
1249     return $cdrom_path;
1250 }
1251
1252 sub get_iso_path {
1253     my ($storecfg, $vmid, $cdrom) = @_;
1254
1255     if ($cdrom eq 'cdrom') {
1256         return get_cdrom_path();
1257     } elsif ($cdrom eq 'none') {
1258         return '';
1259     } elsif ($cdrom =~ m|^/|) {
1260         return $cdrom;
1261     } else {
1262         return PVE::Storage::path($storecfg, $cdrom);
1263     }
1264 }
1265
1266 # try to convert old style file names to volume IDs
1267 sub filename_to_volume_id {
1268     my ($vmid, $file, $media) = @_;
1269
1270      if (!($file eq 'none' || $file eq 'cdrom' ||
1271           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272
1273         return if $file =~ m|/|;
1274
1275         if ($media && $media eq 'cdrom') {
1276             $file = "local:iso/$file";
1277         } else {
1278             $file = "local:$vmid/$file";
1279         }
1280     }
1281
1282     return $file;
1283 }
1284
1285 sub verify_media_type {
1286     my ($opt, $vtype, $media) = @_;
1287
1288     return if !$media;
1289
1290     my $etype;
1291     if ($media eq 'disk') {
1292         $etype = 'images';
1293     } elsif ($media eq 'cdrom') {
1294         $etype = 'iso';
1295     } else {
1296         die "internal error";
1297     }
1298
1299     return if ($vtype eq $etype);
1300
1301     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1302 }
1303
1304 sub cleanup_drive_path {
1305     my ($opt, $storecfg, $drive) = @_;
1306
1307     # try to convert filesystem paths to volume IDs
1308
1309     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1310         ($drive->{file} !~ m|^/dev/.+|) &&
1311         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1312         ($drive->{file} !~ m/^\d+$/)) {
1313         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1314         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315             if !$vtype;
1316         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1317         verify_media_type($opt, $vtype, $drive->{media});
1318         $drive->{file} = $volid;
1319     }
1320
1321     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1322 }
1323
1324 sub parse_hotplug_features {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     return $res if $data eq '0';
1330
1331     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1332
1333     foreach my $feature (PVE::Tools::split_list($data)) {
1334         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1335             $res->{$1} = 1;
1336         } else {
1337             die "invalid hotplug feature '$feature'\n";
1338         }
1339     }
1340     return $res;
1341 }
1342
1343 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1344 sub pve_verify_hotplug_features {
1345     my ($value, $noerr) = @_;
1346
1347     return $value if parse_hotplug_features($value);
1348
1349     return if $noerr;
1350
1351     die "unable to parse hotplug option\n";
1352 }
1353
1354 sub assert_clipboard_config {
1355     my ($vga) = @_;
1356
1357     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1358
1359     if (
1360         $vga->{'clipboard'}
1361         && $vga->{'clipboard'} eq 'vnc'
1362         && $vga->{type}
1363         && $vga->{type} !~ $clipboard_regex
1364     ) {
1365         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1366     }
1367 }
1368
1369 sub print_tabletdevice_full {
1370     my ($conf, $arch) = @_;
1371
1372     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1373
1374     # we use uhci for old VMs because tablet driver was buggy in older qemu
1375     my $usbbus;
1376     if ($q35 || $arch eq 'aarch64') {
1377         $usbbus = 'ehci';
1378     } else {
1379         $usbbus = 'uhci';
1380     }
1381
1382     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1383 }
1384
1385 sub print_keyboarddevice_full {
1386     my ($conf, $arch) = @_;
1387
1388     return if $arch ne 'aarch64';
1389
1390     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1391 }
1392
1393 my sub get_drive_id {
1394     my ($drive) = @_;
1395     return "$drive->{interface}$drive->{index}";
1396 }
1397
1398 sub print_drivedevice_full {
1399     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1400
1401     my $device = '';
1402     my $maxdev = 0;
1403
1404     my $drive_id = get_drive_id($drive);
1405     if ($drive->{interface} eq 'virtio') {
1406         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1407         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1408         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1409     } elsif ($drive->{interface} eq 'scsi') {
1410
1411         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1412         my $unit = $drive->{index} % $maxdev;
1413
1414         my $machine_version = extract_version($machine_type, kvm_user_version());
1415         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1416             $drive, $storecfg, $machine_version);
1417
1418         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1419             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1420         } else {
1421             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1422                 .",lun=$drive->{index}";
1423         }
1424         $device .= ",drive=drive-$drive_id,id=$drive_id";
1425
1426         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1427             $device .= ",rotation_rate=1";
1428         }
1429         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1430
1431         # only scsi-hd and scsi-cd support passing vendor and product information
1432         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1433             if (my $vendor = $drive->{vendor}) {
1434                 $device .= ",vendor=$vendor";
1435             }
1436             if (my $product = $drive->{product}) {
1437                 $device .= ",product=$product";
1438             }
1439         }
1440
1441     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1442         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1443         my $controller = int($drive->{index} / $maxdev);
1444         my $unit = $drive->{index} % $maxdev;
1445
1446         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1447         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1448         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1449         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1450         # were before. Move odd ones up by 2 where they don't clash.
1451         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1452             $controller += 2 * ($unit % 2);
1453             $unit = 0;
1454         }
1455
1456         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1457
1458         $device = "ide-$devicetype";
1459         if ($drive->{interface} eq 'ide') {
1460             $device .= ",bus=ide.$controller,unit=$unit";
1461         } else {
1462             $device .= ",bus=ahci$controller.$unit";
1463         }
1464         $device .= ",drive=drive-$drive_id,id=$drive_id";
1465
1466         if ($devicetype eq 'hd') {
1467             if (my $model = $drive->{model}) {
1468                 $model = URI::Escape::uri_unescape($model);
1469                 $device .= ",model=$model";
1470             }
1471             if ($drive->{ssd}) {
1472                 $device .= ",rotation_rate=1";
1473             }
1474         }
1475         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1476     } elsif ($drive->{interface} eq 'usb') {
1477         die "implement me";
1478         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1479     } else {
1480         die "unsupported interface type";
1481     }
1482
1483     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1484
1485     if (my $serial = $drive->{serial}) {
1486         $serial = URI::Escape::uri_unescape($serial);
1487         $device .= ",serial=$serial";
1488     }
1489
1490
1491     return $device;
1492 }
1493
1494 sub get_initiator_name {
1495     my $initiator;
1496
1497     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1498     while (defined(my $line = <$fh>)) {
1499         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1500         $initiator = $1;
1501         last;
1502     }
1503     $fh->close();
1504
1505     return $initiator;
1506 }
1507
1508 my sub storage_allows_io_uring_default {
1509     my ($scfg, $cache_direct) = @_;
1510
1511     # io_uring with cache mode writeback or writethrough on krbd will hang...
1512     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1513
1514     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1515     # sometimes, just plain disable...
1516     return if $scfg && $scfg->{type} eq 'lvm';
1517
1518     # io_uring causes problems when used with CIFS since kernel 5.15
1519     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1520     return if $scfg && $scfg->{type} eq 'cifs';
1521
1522     return 1;
1523 }
1524
1525 my sub drive_uses_cache_direct {
1526     my ($drive, $scfg) = @_;
1527
1528     my $cache_direct = 0;
1529
1530     if (my $cache = $drive->{cache}) {
1531         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1532     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1533         $cache_direct = 1;
1534     }
1535
1536     return $cache_direct;
1537 }
1538
1539 sub print_drive_commandline_full {
1540     my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
1541
1542     my $path;
1543     my $volid = $drive->{file};
1544     my $format = $drive->{format};
1545     my $drive_id = get_drive_id($drive);
1546
1547     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1548     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1549
1550     if (drive_is_cdrom($drive)) {
1551         $path = get_iso_path($storecfg, $vmid, $volid);
1552         die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
1553     } else {
1554         if ($storeid) {
1555             $path = PVE::Storage::path($storecfg, $volid);
1556             $format //= qemu_img_format($scfg, $volname);
1557         } else {
1558             $path = $volid;
1559             $format //= "raw";
1560         }
1561    }
1562
1563    my $is_rbd = $path =~ m/^rbd:/;
1564
1565     my $opts = '';
1566     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1567     foreach my $o (@qemu_drive_options) {
1568         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1569     }
1570
1571     # snapshot only accepts on|off
1572     if (defined($drive->{snapshot})) {
1573         my $v = $drive->{snapshot} ? 'on' : 'off';
1574         $opts .= ",snapshot=$v";
1575     }
1576
1577     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1578         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1579     }
1580
1581     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1582         my ($dir, $qmpname) = @$type;
1583         if (my $v = $drive->{"mbps$dir"}) {
1584             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1585         }
1586         if (my $v = $drive->{"mbps${dir}_max"}) {
1587             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1588         }
1589         if (my $v = $drive->{"bps${dir}_max_length"}) {
1590             $opts .= ",throttling.bps$qmpname-max-length=$v";
1591         }
1592         if (my $v = $drive->{"iops${dir}"}) {
1593             $opts .= ",throttling.iops$qmpname=$v";
1594         }
1595         if (my $v = $drive->{"iops${dir}_max"}) {
1596             $opts .= ",throttling.iops$qmpname-max=$v";
1597         }
1598         if (my $v = $drive->{"iops${dir}_max_length"}) {
1599             $opts .= ",throttling.iops$qmpname-max-length=$v";
1600         }
1601     }
1602
1603     if ($live_restore_name) {
1604         $format = "rbd" if $is_rbd;
1605         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1606             if !$format;
1607         $opts .= ",format=alloc-track,file.driver=$format";
1608     } elsif ($format) {
1609         $opts .= ",format=$format";
1610     }
1611
1612     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1613
1614     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1615
1616     if (!$drive->{aio}) {
1617         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1618             # io_uring supports all cache modes
1619             $opts .= ",aio=io_uring";
1620         } else {
1621             # aio native works only with O_DIRECT
1622             if($cache_direct) {
1623                 $opts .= ",aio=native";
1624             } else {
1625                 $opts .= ",aio=threads";
1626             }
1627         }
1628     }
1629
1630     if (!drive_is_cdrom($drive)) {
1631         my $detectzeroes;
1632         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1633             $detectzeroes = 'off';
1634         } elsif ($drive->{discard}) {
1635             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1636         } else {
1637             # This used to be our default with discard not being specified:
1638             $detectzeroes = 'on';
1639         }
1640
1641         # note: 'detect-zeroes' works per blockdev and we want it to persist
1642         # after the alloc-track is removed, so put it on 'file' directly
1643         my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes";
1644         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1645     }
1646
1647     if ($live_restore_name) {
1648         $opts .= ",backing=$live_restore_name";
1649         $opts .= ",auto-remove=on";
1650     }
1651
1652     # my $file_param = $live_restore_name ? "file.file.filename" : "file";
1653     my $file_param = "file";
1654     if ($live_restore_name) {
1655         # non-rbd drivers require the underlying file to be a seperate block
1656         # node, so add a second .file indirection
1657         $file_param .= ".file" if !$is_rbd;
1658         $file_param .= ".filename";
1659     }
1660     my $pathinfo = $path ? "$file_param=$path," : '';
1661
1662     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1663 }
1664
1665 sub print_pbs_blockdev {
1666     my ($pbs_conf, $pbs_name) = @_;
1667     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1668     $blockdev .= ",repository=$pbs_conf->{repository}";
1669     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1670     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1671     $blockdev .= ",archive=$pbs_conf->{archive}";
1672     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1673     return $blockdev;
1674 }
1675
1676 sub print_netdevice_full {
1677     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1678
1679     my $device = $net->{model};
1680     if ($net->{model} eq 'virtio') {
1681          $device = 'virtio-net-pci';
1682      };
1683
1684     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1685     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1686     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1687         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1688         # and out of each queue plus one config interrupt and control vector queue
1689         my $vectors = $net->{queues} * 2 + 2;
1690         $tmpstr .= ",vectors=$vectors,mq=on";
1691         if (min_version($machine_version, 7, 1)) {
1692             $tmpstr .= ",packed=on";
1693         }
1694     }
1695
1696     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1697         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1698     }
1699
1700     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1701
1702     if (my $mtu = $net->{mtu}) {
1703         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1704             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1705             if ($mtu == 1) {
1706                  $mtu = $bridge_mtu;
1707             } elsif ($mtu < 576) {
1708                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709             } elsif ($mtu > $bridge_mtu) {
1710                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1711             }
1712             $tmpstr .= ",host_mtu=$mtu";
1713         } else {
1714             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1715         }
1716     }
1717
1718     if ($use_old_bios_files) {
1719         my $romfile;
1720         if ($device eq 'virtio-net-pci') {
1721             $romfile = 'pxe-virtio.rom';
1722         } elsif ($device eq 'e1000') {
1723             $romfile = 'pxe-e1000.rom';
1724         } elsif ($device eq 'e1000e') {
1725             $romfile = 'pxe-e1000e.rom';
1726         } elsif ($device eq 'ne2k') {
1727             $romfile = 'pxe-ne2k_pci.rom';
1728         } elsif ($device eq 'pcnet') {
1729             $romfile = 'pxe-pcnet.rom';
1730         } elsif ($device eq 'rtl8139') {
1731             $romfile = 'pxe-rtl8139.rom';
1732         }
1733         $tmpstr .= ",romfile=$romfile" if $romfile;
1734     }
1735
1736     return $tmpstr;
1737 }
1738
1739 sub print_netdev_full {
1740     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1741
1742     my $i = '';
1743     if ($netid =~ m/^net(\d+)$/) {
1744         $i = int($1);
1745     }
1746
1747     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749     my $ifname = "tap${vmid}i$i";
1750
1751     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752     die "interface name '$ifname' is too long (max 15 character)\n"
1753         if length($ifname) >= 16;
1754
1755     my $vhostparam = '';
1756     if (is_native_arch($arch)) {
1757         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1758     }
1759
1760     my $vmname = $conf->{name} || "vm$vmid";
1761
1762     my $netdev = "";
1763     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765     if ($net->{bridge}) {
1766         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1768     } else {
1769         $netdev = "type=user,id=$netid,hostname=$vmname";
1770     }
1771
1772     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1773
1774     return $netdev;
1775 }
1776
1777 my $vga_map = {
1778     'cirrus' => 'cirrus-vga',
1779     'std' => 'VGA',
1780     'vmware' => 'vmware-svga',
1781     'virtio' => 'virtio-vga',
1782     'virtio-gl' => 'virtio-vga-gl',
1783 };
1784
1785 sub print_vga_device {
1786     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1787
1788     my $type = $vga_map->{$vga->{type}};
1789     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1790         $type = 'virtio-gpu';
1791     }
1792     my $vgamem_mb = $vga->{memory};
1793
1794     my $max_outputs = '';
1795     if ($qxlnum) {
1796         $type = $id ? 'qxl' : 'qxl-vga';
1797
1798         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1799             # set max outputs so linux can have up to 4 qxl displays with one device
1800             if (min_version($machine_version, 4, 1)) {
1801                 $max_outputs = ",max_outputs=4";
1802             }
1803         }
1804     }
1805
1806     die "no devicetype for $vga->{type}\n" if !$type;
1807
1808     my $memory = "";
1809     if ($vgamem_mb) {
1810         if ($vga->{type} =~ /^virtio/) {
1811             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1812             $memory = ",max_hostmem=$bytes";
1813         } elsif ($qxlnum) {
1814             # from https://www.spice-space.org/multiple-monitors.html
1815             $memory = ",vgamem_mb=$vga->{memory}";
1816             my $ram = $vgamem_mb * 4;
1817             my $vram = $vgamem_mb * 2;
1818             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1819         } else {
1820             $memory = ",vgamem_mb=$vga->{memory}";
1821         }
1822     } elsif ($qxlnum && $id) {
1823         $memory = ",ram_size=67108864,vram_size=33554432";
1824     }
1825
1826     my $edidoff = "";
1827     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1828         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1829     }
1830
1831     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1832     my $vgaid = "vga" . ($id // '');
1833     my $pciaddr;
1834     if ($q35 && $vgaid eq 'vga') {
1835         # the first display uses pcie.0 bus on q35 machines
1836         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1837     } else {
1838         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1839     }
1840
1841     if ($vga->{type} eq 'virtio-gl') {
1842         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1843         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1844             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1845
1846         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1847             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1848     }
1849
1850     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1851 }
1852
1853 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1854 sub parse_net {
1855     my ($data, $disable_mac_autogen) = @_;
1856
1857     my $res = eval { parse_property_string($net_fmt, $data) };
1858     if ($@) {
1859         warn $@;
1860         return;
1861     }
1862     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1863         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1864         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1865     }
1866     return $res;
1867 }
1868
1869 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1870 sub parse_ipconfig {
1871     my ($data) = @_;
1872
1873     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1874     if ($@) {
1875         warn $@;
1876         return;
1877     }
1878
1879     if ($res->{gw} && !$res->{ip}) {
1880         warn 'gateway specified without specifying an IP address';
1881         return;
1882     }
1883     if ($res->{gw6} && !$res->{ip6}) {
1884         warn 'IPv6 gateway specified without specifying an IPv6 address';
1885         return;
1886     }
1887     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1888         warn 'gateway specified together with DHCP';
1889         return;
1890     }
1891     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1892         # gw6 + auto/dhcp
1893         warn "IPv6 gateway specified together with $res->{ip6} address";
1894         return;
1895     }
1896
1897     if (!$res->{ip} && !$res->{ip6}) {
1898         return { ip => 'dhcp', ip6 => 'dhcp' };
1899     }
1900
1901     return $res;
1902 }
1903
1904 sub print_net {
1905     my $net = shift;
1906
1907     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1908 }
1909
1910 sub add_random_macs {
1911     my ($settings) = @_;
1912
1913     foreach my $opt (keys %$settings) {
1914         next if $opt !~ m/^net(\d+)$/;
1915         my $net = parse_net($settings->{$opt});
1916         next if !$net;
1917         $settings->{$opt} = print_net($net);
1918     }
1919 }
1920
1921 sub vm_is_volid_owner {
1922     my ($storecfg, $vmid, $volid) = @_;
1923
1924     if ($volid !~  m|^/|) {
1925         my ($path, $owner);
1926         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1927         if ($owner && ($owner == $vmid)) {
1928             return 1;
1929         }
1930     }
1931
1932     return;
1933 }
1934
1935 sub vmconfig_register_unused_drive {
1936     my ($storecfg, $vmid, $conf, $drive) = @_;
1937
1938     if (drive_is_cloudinit($drive)) {
1939         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1940         warn $@ if $@;
1941         delete $conf->{cloudinit};
1942     } elsif (!drive_is_cdrom($drive)) {
1943         my $volid = $drive->{file};
1944         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1945             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1946         }
1947     }
1948 }
1949
1950 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1951 my $smbios1_fmt = {
1952     uuid => {
1953         type => 'string',
1954         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1955         format_description => 'UUID',
1956         description => "Set SMBIOS1 UUID.",
1957         optional => 1,
1958     },
1959     version => {
1960         type => 'string',
1961         pattern => '[A-Za-z0-9+\/]+={0,2}',
1962         format_description => 'Base64 encoded string',
1963         description => "Set SMBIOS1 version.",
1964         optional => 1,
1965     },
1966     serial => {
1967         type => 'string',
1968         pattern => '[A-Za-z0-9+\/]+={0,2}',
1969         format_description => 'Base64 encoded string',
1970         description => "Set SMBIOS1 serial number.",
1971         optional => 1,
1972     },
1973     manufacturer => {
1974         type => 'string',
1975         pattern => '[A-Za-z0-9+\/]+={0,2}',
1976         format_description => 'Base64 encoded string',
1977         description => "Set SMBIOS1 manufacturer.",
1978         optional => 1,
1979     },
1980     product => {
1981         type => 'string',
1982         pattern => '[A-Za-z0-9+\/]+={0,2}',
1983         format_description => 'Base64 encoded string',
1984         description => "Set SMBIOS1 product ID.",
1985         optional => 1,
1986     },
1987     sku => {
1988         type => 'string',
1989         pattern => '[A-Za-z0-9+\/]+={0,2}',
1990         format_description => 'Base64 encoded string',
1991         description => "Set SMBIOS1 SKU string.",
1992         optional => 1,
1993     },
1994     family => {
1995         type => 'string',
1996         pattern => '[A-Za-z0-9+\/]+={0,2}',
1997         format_description => 'Base64 encoded string',
1998         description => "Set SMBIOS1 family string.",
1999         optional => 1,
2000     },
2001     base64 => {
2002         type => 'boolean',
2003         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2004         optional => 1,
2005     },
2006 };
2007
2008 sub parse_smbios1 {
2009     my ($data) = @_;
2010
2011     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2012     warn $@ if $@;
2013     return $res;
2014 }
2015
2016 sub print_smbios1 {
2017     my ($smbios1) = @_;
2018     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2019 }
2020
2021 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2022
2023 sub parse_watchdog {
2024     my ($value) = @_;
2025
2026     return if !$value;
2027
2028     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2029     warn $@ if $@;
2030     return $res;
2031 }
2032
2033 sub parse_guest_agent {
2034     my ($conf) = @_;
2035
2036     return {} if !defined($conf->{agent});
2037
2038     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2039     warn $@ if $@;
2040
2041     # if the agent is disabled ignore the other potentially set properties
2042     return {} if !$res->{enabled};
2043     return $res;
2044 }
2045
2046 sub get_qga_key {
2047     my ($conf, $key) = @_;
2048     return undef if !defined($conf->{agent});
2049
2050     my $agent = parse_guest_agent($conf);
2051     return $agent->{$key};
2052 }
2053
2054 sub parse_vga {
2055     my ($value) = @_;
2056
2057     return {} if !$value;
2058     my $res = eval { parse_property_string($vga_fmt, $value) };
2059     warn $@ if $@;
2060     return $res;
2061 }
2062
2063 sub parse_rng {
2064     my ($value) = @_;
2065
2066     return if !$value;
2067
2068     my $res = eval { parse_property_string($rng_fmt, $value) };
2069     warn $@ if $@;
2070     return $res;
2071 }
2072
2073 sub parse_meta_info {
2074     my ($value) = @_;
2075
2076     return if !$value;
2077
2078     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2079     warn $@ if $@;
2080     return $res;
2081 }
2082
2083 sub new_meta_info_string {
2084     my () = @_; # for now do not allow to override any value
2085
2086     return PVE::JSONSchema::print_property_string(
2087         {
2088             'creation-qemu' => kvm_user_version(),
2089             ctime => "". int(time()),
2090         },
2091         $meta_info_fmt
2092     );
2093 }
2094
2095 sub qemu_created_version_fixups {
2096     my ($conf, $forcemachine, $kvmver) = @_;
2097
2098     my $meta = parse_meta_info($conf->{meta}) // {};
2099     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2100
2101     # check if we need to apply some handling for VMs that always use the latest machine version but
2102     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2103     # would be required (we do not want to pin machine version for non-windows OS type)
2104     if (
2105         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2106         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2107         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2108         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2109     ) {
2110         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2111         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2112             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2113             # and thus with the predictable interface naming of systemd
2114             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2115         }
2116     }
2117     return;
2118 }
2119
2120 # add JSON properties for create and set function
2121 sub json_config_properties {
2122     my ($prop, $with_disk_alloc) = @_;
2123
2124     my $skip_json_config_opts = {
2125         parent => 1,
2126         snaptime => 1,
2127         vmstate => 1,
2128         runningmachine => 1,
2129         runningcpu => 1,
2130         meta => 1,
2131     };
2132
2133     foreach my $opt (keys %$confdesc) {
2134         next if $skip_json_config_opts->{$opt};
2135
2136         if ($with_disk_alloc && is_valid_drivename($opt)) {
2137             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2138         } else {
2139             $prop->{$opt} = $confdesc->{$opt};
2140         }
2141     }
2142
2143     return $prop;
2144 }
2145
2146 # Properties that we can read from an OVF file
2147 sub json_ovf_properties {
2148     my $prop = {};
2149
2150     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2151         $prop->{$device} = {
2152             type => 'string',
2153             format => 'pve-volume-id-or-absolute-path',
2154             description => "Disk image that gets imported to $device",
2155             optional => 1,
2156         };
2157     }
2158
2159     $prop->{cores} = {
2160         type => 'integer',
2161         description => "The number of CPU cores.",
2162         optional => 1,
2163     };
2164     $prop->{memory} = {
2165         type => 'integer',
2166         description => "Amount of RAM for the VM in MB.",
2167         optional => 1,
2168     };
2169     $prop->{name} = {
2170         type => 'string',
2171         description => "Name of the VM.",
2172         optional => 1,
2173     };
2174
2175     return $prop;
2176 }
2177
2178 # return copy of $confdesc_cloudinit to generate documentation
2179 sub cloudinit_config_properties {
2180
2181     return dclone($confdesc_cloudinit);
2182 }
2183
2184 sub cloudinit_pending_properties {
2185     my $p = {
2186         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2187         name => 1,
2188     };
2189     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2190     return $p;
2191 }
2192
2193 sub check_type {
2194     my ($key, $value) = @_;
2195
2196     die "unknown setting '$key'\n" if !$confdesc->{$key};
2197
2198     my $type = $confdesc->{$key}->{type};
2199
2200     if (!defined($value)) {
2201         die "got undefined value\n";
2202     }
2203
2204     if ($value =~ m/[\n\r]/) {
2205         die "property contains a line feed\n";
2206     }
2207
2208     if ($type eq 'boolean') {
2209         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2210         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2211         die "type check ('boolean') failed - got '$value'\n";
2212     } elsif ($type eq 'integer') {
2213         return int($1) if $value =~ m/^(\d+)$/;
2214         die "type check ('integer') failed - got '$value'\n";
2215     } elsif ($type eq 'number') {
2216         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2217         die "type check ('number') failed - got '$value'\n";
2218     } elsif ($type eq 'string') {
2219         if (my $fmt = $confdesc->{$key}->{format}) {
2220             PVE::JSONSchema::check_format($fmt, $value);
2221             return $value;
2222         }
2223         $value =~ s/^\"(.*)\"$/$1/;
2224         return $value;
2225     } else {
2226         die "internal error"
2227     }
2228 }
2229
2230 sub destroy_vm {
2231     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2232
2233     my $conf = PVE::QemuConfig->load_config($vmid);
2234
2235     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2236         PVE::QemuConfig->check_lock($conf);
2237     }
2238
2239     if ($conf->{template}) {
2240         # check if any base image is still used by a linked clone
2241         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2242                 my ($ds, $drive) = @_;
2243                 return if drive_is_cdrom($drive);
2244
2245                 my $volid = $drive->{file};
2246                 return if !$volid || $volid =~ m|^/|;
2247
2248                 die "base volume '$volid' is still in use by linked cloned\n"
2249                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2250
2251         });
2252     }
2253
2254     my $volids = {};
2255     my $remove_owned_drive = sub {
2256         my ($ds, $drive) = @_;
2257         return if drive_is_cdrom($drive, 1);
2258
2259         my $volid = $drive->{file};
2260         return if !$volid || $volid =~ m|^/|;
2261         return if $volids->{$volid};
2262
2263         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2264         return if !$path || !$owner || ($owner != $vmid);
2265
2266         $volids->{$volid} = 1;
2267         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2268         warn "Could not remove disk '$volid', check manually: $@" if $@;
2269     };
2270
2271     # only remove disks owned by this VM (referenced in the config)
2272     my $include_opts = {
2273         include_unused => 1,
2274         extra_keys => ['vmstate'],
2275     };
2276     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2277
2278     for my $snap (values %{$conf->{snapshots}}) {
2279         next if !defined($snap->{vmstate});
2280         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2281         next if !defined($drive);
2282         $remove_owned_drive->('vmstate', $drive);
2283     }
2284
2285     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2286
2287     if ($purge_unreferenced) { # also remove unreferenced disk
2288         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2289         PVE::Storage::foreach_volid($vmdisks, sub {
2290             my ($volid, $sid, $volname, $d) = @_;
2291             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2292             warn $@ if $@;
2293         });
2294     }
2295
2296     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2297     warn $@ if $@;
2298
2299     if (defined $replacement_conf) {
2300         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2301     } else {
2302         PVE::QemuConfig->destroy_config($vmid);
2303     }
2304 }
2305
2306 sub parse_vm_config {
2307     my ($filename, $raw, $strict) = @_;
2308
2309     return if !defined($raw);
2310
2311     my $res = {
2312         digest => Digest::SHA::sha1_hex($raw),
2313         snapshots => {},
2314         pending => {},
2315         cloudinit => {},
2316     };
2317
2318     my $handle_error = sub {
2319         my ($msg) = @_;
2320
2321         if ($strict) {
2322             die $msg;
2323         } else {
2324             warn $msg;
2325         }
2326     };
2327
2328     $filename =~ m|/qemu-server/(\d+)\.conf$|
2329         || die "got strange filename '$filename'";
2330
2331     my $vmid = $1;
2332
2333     my $conf = $res;
2334     my $descr;
2335     my $finish_description = sub {
2336         if (defined($descr)) {
2337             $descr =~ s/\s+$//;
2338             $conf->{description} = $descr;
2339         }
2340         $descr = undef;
2341     };
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             $finish_description->();
2351             $conf = $res->{$section} = {};
2352             next;
2353         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2354             $section = 'cloudinit';
2355             $finish_description->();
2356             $conf = $res->{$section} = {};
2357             next;
2358
2359         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2360             $section = $1;
2361             $finish_description->();
2362             $conf = $res->{snapshots}->{$section} = {};
2363             next;
2364         }
2365
2366         if ($line =~ m/^\#(.*)$/) {
2367             $descr = '' if !defined($descr);
2368             $descr .= PVE::Tools::decode_text($1) . "\n";
2369             next;
2370         }
2371
2372         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2373             $descr = '' if !defined($descr);
2374             $descr .= PVE::Tools::decode_text($2);
2375         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2376             $conf->{snapstate} = $1;
2377         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2378             my $key = $1;
2379             my $value = $2;
2380             $conf->{$key} = $value;
2381         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2382             my $value = $1;
2383             if ($section eq 'pending') {
2384                 $conf->{delete} = $value; # we parse this later
2385             } else {
2386                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2387             }
2388         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             if ($section eq 'cloudinit') {
2392                 # ignore validation only used for informative purpose
2393                 $conf->{$key} = $value;
2394                 next;
2395             }
2396             eval { $value = check_type($key, $value); };
2397             if ($@) {
2398                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2399             } else {
2400                 $key = 'ide2' if $key eq 'cdrom';
2401                 my $fmt = $confdesc->{$key}->{format};
2402                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2403                     my $v = parse_drive($key, $value);
2404                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2405                         $v->{file} = $volid;
2406                         $value = print_drive($v);
2407                     } else {
2408                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2409                         next;
2410                     }
2411                 }
2412
2413                 $conf->{$key} = $value;
2414             }
2415         } else {
2416             $handle_error->("vm $vmid - unable to parse config: $line\n");
2417         }
2418     }
2419
2420     $finish_description->();
2421     delete $res->{snapstate}; # just to be sure
2422
2423     return $res;
2424 }
2425
2426 sub write_vm_config {
2427     my ($filename, $conf) = @_;
2428
2429     delete $conf->{snapstate}; # just to be sure
2430
2431     if ($conf->{cdrom}) {
2432         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2433         $conf->{ide2} = $conf->{cdrom};
2434         delete $conf->{cdrom};
2435     }
2436
2437     # we do not use 'smp' any longer
2438     if ($conf->{sockets}) {
2439         delete $conf->{smp};
2440     } elsif ($conf->{smp}) {
2441         $conf->{sockets} = $conf->{smp};
2442         delete $conf->{cores};
2443         delete $conf->{smp};
2444     }
2445
2446     my $used_volids = {};
2447
2448     my $cleanup_config = sub {
2449         my ($cref, $pending, $snapname) = @_;
2450
2451         foreach my $key (keys %$cref) {
2452             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2453                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2454             my $value = $cref->{$key};
2455             if ($key eq 'delete') {
2456                 die "propertry 'delete' is only allowed in [PENDING]\n"
2457                     if !$pending;
2458                 # fixme: check syntax?
2459                 next;
2460             }
2461             eval { $value = check_type($key, $value); };
2462             die "unable to parse value of '$key' - $@" if $@;
2463
2464             $cref->{$key} = $value;
2465
2466             if (!$snapname && is_valid_drivename($key)) {
2467                 my $drive = parse_drive($key, $value);
2468                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2469             }
2470         }
2471     };
2472
2473     &$cleanup_config($conf);
2474
2475     &$cleanup_config($conf->{pending}, 1);
2476
2477     foreach my $snapname (keys %{$conf->{snapshots}}) {
2478         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2479         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2480     }
2481
2482     # remove 'unusedX' settings if we re-add a volume
2483     foreach my $key (keys %$conf) {
2484         my $value = $conf->{$key};
2485         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2486             delete $conf->{$key};
2487         }
2488     }
2489
2490     my $generate_raw_config = sub {
2491         my ($conf, $pending) = @_;
2492
2493         my $raw = '';
2494
2495         # add description as comment to top of file
2496         if (defined(my $descr = $conf->{description})) {
2497             if ($descr) {
2498                 foreach my $cl (split(/\n/, $descr)) {
2499                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2500                 }
2501             } else {
2502                 $raw .= "#\n" if $pending;
2503             }
2504         }
2505
2506         foreach my $key (sort keys %$conf) {
2507             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2508             $raw .= "$key: $conf->{$key}\n";
2509         }
2510         return $raw;
2511     };
2512
2513     my $raw = &$generate_raw_config($conf);
2514
2515     if (scalar(keys %{$conf->{pending}})){
2516         $raw .= "\n[PENDING]\n";
2517         $raw .= &$generate_raw_config($conf->{pending}, 1);
2518     }
2519
2520     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2521         $raw .= "\n[special:cloudinit]\n";
2522         $raw .= &$generate_raw_config($conf->{cloudinit});
2523     }
2524
2525     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2526         $raw .= "\n[$snapname]\n";
2527         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2528     }
2529
2530     return $raw;
2531 }
2532
2533 sub load_defaults {
2534
2535     my $res = {};
2536
2537     # we use static defaults from our JSON schema configuration
2538     foreach my $key (keys %$confdesc) {
2539         if (defined(my $default = $confdesc->{$key}->{default})) {
2540             $res->{$key} = $default;
2541         }
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub config_list {
2548     my $vmlist = PVE::Cluster::get_vmlist();
2549     my $res = {};
2550     return $res if !$vmlist || !$vmlist->{ids};
2551     my $ids = $vmlist->{ids};
2552     my $nodename = nodename();
2553
2554     foreach my $vmid (keys %$ids) {
2555         my $d = $ids->{$vmid};
2556         next if !$d->{node} || $d->{node} ne $nodename;
2557         next if !$d->{type} || $d->{type} ne 'qemu';
2558         $res->{$vmid}->{exists} = 1;
2559     }
2560     return $res;
2561 }
2562
2563 # test if VM uses local resources (to prevent migration)
2564 sub check_local_resources {
2565     my ($conf, $noerr) = @_;
2566
2567     my @loc_res = ();
2568     my $mapped_res = [];
2569
2570     my $nodelist = PVE::Cluster::get_nodelist();
2571     my $pci_map = PVE::Mapping::PCI::config();
2572     my $usb_map = PVE::Mapping::USB::config();
2573
2574     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2575
2576     my $add_missing_mapping = sub {
2577         my ($type, $key, $id) = @_;
2578         for my $node (@$nodelist) {
2579             my $entry;
2580             if ($type eq 'pci') {
2581                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2582             } elsif ($type eq 'usb') {
2583                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2584             }
2585             if (!scalar($entry->@*)) {
2586                 push @{$missing_mappings_by_node->{$node}}, $key;
2587             }
2588         }
2589     };
2590
2591     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2592     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2593
2594     push @loc_res, "ivshmem" if $conf->{ivshmem};
2595
2596     foreach my $k (keys %$conf) {
2597         if ($k =~ m/^usb/) {
2598             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2599             next if $entry->{host} =~ m/^spice$/i;
2600             if ($entry->{mapping}) {
2601                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2602                 push @$mapped_res, $k;
2603             }
2604         }
2605         if ($k =~ m/^hostpci/) {
2606             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2607             if ($entry->{mapping}) {
2608                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2609                 push @$mapped_res, $k;
2610             }
2611         }
2612         # sockets are safe: they will recreated be on the target side post-migrate
2613         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2614         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2615     }
2616
2617     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2618
2619     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2620 }
2621
2622 # check if used storages are available on all nodes (use by migrate)
2623 sub check_storage_availability {
2624     my ($storecfg, $conf, $node) = @_;
2625
2626     PVE::QemuConfig->foreach_volume($conf, sub {
2627         my ($ds, $drive) = @_;
2628
2629         my $volid = $drive->{file};
2630         return if !$volid;
2631
2632         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2633         return if !$sid;
2634
2635         # check if storage is available on both nodes
2636         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2637         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2638
2639         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2640
2641         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2642             if !$scfg->{content}->{$vtype};
2643    });
2644 }
2645
2646 # list nodes where all VM images are available (used by has_feature API)
2647 sub shared_nodes {
2648     my ($conf, $storecfg) = @_;
2649
2650     my $nodelist = PVE::Cluster::get_nodelist();
2651     my $nodehash = { map { $_ => 1 } @$nodelist };
2652     my $nodename = nodename();
2653
2654     PVE::QemuConfig->foreach_volume($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         if ($storeid) {
2662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2663             if ($scfg->{disable}) {
2664                 $nodehash = {};
2665             } elsif (my $avail = $scfg->{nodes}) {
2666                 foreach my $node (keys %$nodehash) {
2667                     delete $nodehash->{$node} if !$avail->{$node};
2668                 }
2669             } elsif (!$scfg->{shared}) {
2670                 foreach my $node (keys %$nodehash) {
2671                     delete $nodehash->{$node} if $node ne $nodename
2672                 }
2673             }
2674         }
2675     });
2676
2677     return $nodehash
2678 }
2679
2680 sub check_local_storage_availability {
2681     my ($conf, $storecfg) = @_;
2682
2683     my $nodelist = PVE::Cluster::get_nodelist();
2684     my $nodehash = { map { $_ => {} } @$nodelist };
2685
2686     PVE::QemuConfig->foreach_volume($conf, sub {
2687         my ($ds, $drive) = @_;
2688
2689         my $volid = $drive->{file};
2690         return if !$volid;
2691
2692         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2693         if ($storeid) {
2694             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2695
2696             if ($scfg->{disable}) {
2697                 foreach my $node (keys %$nodehash) {
2698                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2699                 }
2700             } elsif (my $avail = $scfg->{nodes}) {
2701                 foreach my $node (keys %$nodehash) {
2702                     if (!$avail->{$node}) {
2703                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2704                     }
2705                 }
2706             }
2707         }
2708     });
2709
2710     foreach my $node (values %$nodehash) {
2711         if (my $unavail = $node->{unavailable_storages}) {
2712             $node->{unavailable_storages} = [ sort keys %$unavail ];
2713         }
2714     }
2715
2716     return $nodehash
2717 }
2718
2719 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2720 sub check_running {
2721     my ($vmid, $nocheck, $node) = @_;
2722
2723     # $nocheck is set when called during a migration, in which case the config
2724     # file might still or already reside on the *other* node
2725     # - because rename has already happened, and current node is source
2726     # - because rename hasn't happened yet, and current node is target
2727     # - because rename has happened, current node is target, but hasn't yet
2728     # processed it yet
2729     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2730     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2731 }
2732
2733 sub vzlist {
2734
2735     my $vzlist = config_list();
2736
2737     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2738
2739     while (defined(my $de = $fd->read)) {
2740         next if $de !~ m/^(\d+)\.pid$/;
2741         my $vmid = $1;
2742         next if !defined($vzlist->{$vmid});
2743         if (my $pid = check_running($vmid)) {
2744             $vzlist->{$vmid}->{pid} = $pid;
2745         }
2746     }
2747
2748     return $vzlist;
2749 }
2750
2751 our $vmstatus_return_properties = {
2752     vmid => get_standard_option('pve-vmid'),
2753     status => {
2754         description => "QEMU process status.",
2755         type => 'string',
2756         enum => ['stopped', 'running'],
2757     },
2758     maxmem => {
2759         description => "Maximum memory in bytes.",
2760         type => 'integer',
2761         optional => 1,
2762         renderer => 'bytes',
2763     },
2764     maxdisk => {
2765         description => "Root disk size in bytes.",
2766         type => 'integer',
2767         optional => 1,
2768         renderer => 'bytes',
2769     },
2770     name => {
2771         description => "VM name.",
2772         type => 'string',
2773         optional => 1,
2774     },
2775     qmpstatus => {
2776         description => "VM run state from the 'query-status' QMP monitor command.",
2777         type => 'string',
2778         optional => 1,
2779     },
2780     pid => {
2781         description => "PID of running qemu process.",
2782         type => 'integer',
2783         optional => 1,
2784     },
2785     uptime => {
2786         description => "Uptime.",
2787         type => 'integer',
2788         optional => 1,
2789         renderer => 'duration',
2790     },
2791     cpus => {
2792         description => "Maximum usable CPUs.",
2793         type => 'number',
2794         optional => 1,
2795     },
2796     lock => {
2797         description => "The current config lock, if any.",
2798         type => 'string',
2799         optional => 1,
2800     },
2801     tags => {
2802         description  => "The current configured tags, if any",
2803         type => 'string',
2804         optional => 1,
2805     },
2806     'running-machine' => {
2807         description => "The currently running machine type (if running).",
2808         type => 'string',
2809         optional => 1,
2810     },
2811     'running-qemu' => {
2812         description => "The currently running QEMU version (if running).",
2813         type => 'string',
2814         optional => 1,
2815     },
2816 };
2817
2818 my $last_proc_pid_stat;
2819
2820 # get VM status information
2821 # This must be fast and should not block ($full == false)
2822 # We only query KVM using QMP if $full == true (this can be slow)
2823 sub vmstatus {
2824     my ($opt_vmid, $full) = @_;
2825
2826     my $res = {};
2827
2828     my $storecfg = PVE::Storage::config();
2829
2830     my $list = vzlist();
2831     my $defaults = load_defaults();
2832
2833     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2834
2835     my $cpucount = $cpuinfo->{cpus} || 1;
2836
2837     foreach my $vmid (keys %$list) {
2838         next if $opt_vmid && ($vmid ne $opt_vmid);
2839
2840         my $conf = PVE::QemuConfig->load_config($vmid);
2841
2842         my $d = { vmid => int($vmid) };
2843         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2844
2845         # fixme: better status?
2846         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2847
2848         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2849         if (defined($size)) {
2850             $d->{disk} = 0; # no info available
2851             $d->{maxdisk} = $size;
2852         } else {
2853             $d->{disk} = 0;
2854             $d->{maxdisk} = 0;
2855         }
2856
2857         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2858             * ($conf->{cores} || $defaults->{cores});
2859         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2860         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2861
2862         $d->{name} = $conf->{name} || "VM $vmid";
2863         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2864
2865         if ($conf->{balloon}) {
2866             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2867             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2868                 : $defaults->{shares};
2869         }
2870
2871         $d->{uptime} = 0;
2872         $d->{cpu} = 0;
2873         $d->{mem} = 0;
2874
2875         $d->{netout} = 0;
2876         $d->{netin} = 0;
2877
2878         $d->{diskread} = 0;
2879         $d->{diskwrite} = 0;
2880
2881         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2882
2883         $d->{serial} = 1 if conf_has_serial($conf);
2884         $d->{lock} = $conf->{lock} if $conf->{lock};
2885         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2886
2887         $res->{$vmid} = $d;
2888     }
2889
2890     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2891     foreach my $dev (keys %$netdev) {
2892         next if $dev !~ m/^tap([1-9]\d*)i/;
2893         my $vmid = $1;
2894         my $d = $res->{$vmid};
2895         next if !$d;
2896
2897         $d->{netout} += $netdev->{$dev}->{receive};
2898         $d->{netin} += $netdev->{$dev}->{transmit};
2899
2900         if ($full) {
2901             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2902             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2903         }
2904
2905     }
2906
2907     my $ctime = gettimeofday;
2908
2909     foreach my $vmid (keys %$list) {
2910
2911         my $d = $res->{$vmid};
2912         my $pid = $d->{pid};
2913         next if !$pid;
2914
2915         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2916         next if !$pstat; # not running
2917
2918         my $used = $pstat->{utime} + $pstat->{stime};
2919
2920         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2921
2922         if ($pstat->{vsize}) {
2923             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2924         }
2925
2926         my $old = $last_proc_pid_stat->{$pid};
2927         if (!$old) {
2928             $last_proc_pid_stat->{$pid} = {
2929                 time => $ctime,
2930                 used => $used,
2931                 cpu => 0,
2932             };
2933             next;
2934         }
2935
2936         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2937
2938         if ($dtime > 1000) {
2939             my $dutime = $used -  $old->{used};
2940
2941             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2942             $last_proc_pid_stat->{$pid} = {
2943                 time => $ctime,
2944                 used => $used,
2945                 cpu => $d->{cpu},
2946             };
2947         } else {
2948             $d->{cpu} = $old->{cpu};
2949         }
2950     }
2951
2952     return $res if !$full;
2953
2954     my $qmpclient = PVE::QMPClient->new();
2955
2956     my $ballooncb = sub {
2957         my ($vmid, $resp) = @_;
2958
2959         my $info = $resp->{'return'};
2960         return if !$info->{max_mem};
2961
2962         my $d = $res->{$vmid};
2963
2964         # use memory assigned to VM
2965         $d->{maxmem} = $info->{max_mem};
2966         $d->{balloon} = $info->{actual};
2967
2968         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2969             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2970             $d->{freemem} = $info->{free_mem};
2971         }
2972
2973         $d->{ballooninfo} = $info;
2974     };
2975
2976     my $blockstatscb = sub {
2977         my ($vmid, $resp) = @_;
2978         my $data = $resp->{'return'} || [];
2979         my $totalrdbytes = 0;
2980         my $totalwrbytes = 0;
2981
2982         for my $blockstat (@$data) {
2983             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2984             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2985
2986             $blockstat->{device} =~ s/drive-//;
2987             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2988         }
2989         $res->{$vmid}->{diskread} = $totalrdbytes;
2990         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2991     };
2992
2993     my $machinecb = sub {
2994         my ($vmid, $resp) = @_;
2995         my $data = $resp->{'return'} || [];
2996
2997         $res->{$vmid}->{'running-machine'} =
2998             PVE::QemuServer::Machine::current_from_query_machines($data);
2999     };
3000
3001     my $versioncb = sub {
3002         my ($vmid, $resp) = @_;
3003         my $data = $resp->{'return'} // {};
3004         my $version = 'unknown';
3005
3006         if (my $v = $data->{qemu}) {
3007             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3008         }
3009
3010         $res->{$vmid}->{'running-qemu'} = $version;
3011     };
3012
3013     my $statuscb = sub {
3014         my ($vmid, $resp) = @_;
3015
3016         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3017         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3018         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3019         # this fails if ballon driver is not loaded, so this must be
3020         # the last commnand (following command are aborted if this fails).
3021         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3022
3023         my $status = 'unknown';
3024         if (!defined($status = $resp->{'return'}->{status})) {
3025             warn "unable to get VM status\n";
3026             return;
3027         }
3028
3029         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3030     };
3031
3032     foreach my $vmid (keys %$list) {
3033         next if $opt_vmid && ($vmid ne $opt_vmid);
3034         next if !$res->{$vmid}->{pid}; # not running
3035         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3036     }
3037
3038     $qmpclient->queue_execute(undef, 2);
3039
3040     foreach my $vmid (keys %$list) {
3041         next if $opt_vmid && ($vmid ne $opt_vmid);
3042         next if !$res->{$vmid}->{pid}; #not running
3043
3044         # we can't use the $qmpclient since it might have already aborted on
3045         # 'query-balloon', but this might also fail for older versions...
3046         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3047         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3048     }
3049
3050     foreach my $vmid (keys %$list) {
3051         next if $opt_vmid && ($vmid ne $opt_vmid);
3052         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3053     }
3054
3055     return $res;
3056 }
3057
3058 sub conf_has_serial {
3059     my ($conf) = @_;
3060
3061     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3062         if ($conf->{"serial$i"}) {
3063             return 1;
3064         }
3065     }
3066
3067     return 0;
3068 }
3069
3070 sub conf_has_audio {
3071     my ($conf, $id) = @_;
3072
3073     $id //= 0;
3074     my $audio = $conf->{"audio$id"};
3075     return if !defined($audio);
3076
3077     my $audioproperties = parse_property_string($audio_fmt, $audio);
3078     my $audiodriver = $audioproperties->{driver} // 'spice';
3079
3080     return {
3081         dev => $audioproperties->{device},
3082         dev_id => "audiodev$id",
3083         backend => $audiodriver,
3084         backend_id => "$audiodriver-backend${id}",
3085     };
3086 }
3087
3088 sub audio_devs {
3089     my ($audio, $audiopciaddr, $machine_version) = @_;
3090
3091     my $devs = [];
3092
3093     my $id = $audio->{dev_id};
3094     my $audiodev = "";
3095     if (min_version($machine_version, 4, 2)) {
3096         $audiodev = ",audiodev=$audio->{backend_id}";
3097     }
3098
3099     if ($audio->{dev} eq 'AC97') {
3100         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3101     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3102         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3103         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3104         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3105     } else {
3106         die "unkown audio device '$audio->{dev}', implement me!";
3107     }
3108
3109     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3110
3111     return $devs;
3112 }
3113
3114 sub get_tpm_paths {
3115     my ($vmid) = @_;
3116     return {
3117         socket => "/var/run/qemu-server/$vmid.swtpm",
3118         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3119     };
3120 }
3121
3122 sub add_tpm_device {
3123     my ($vmid, $devices, $conf) = @_;
3124
3125     return if !$conf->{tpmstate0};
3126
3127     my $paths = get_tpm_paths($vmid);
3128
3129     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3130     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3131     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3132 }
3133
3134 sub start_swtpm {
3135     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3136
3137     return if !$tpmdrive;
3138
3139     my $state;
3140     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3141     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3142     if ($storeid) {
3143         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3144     } else {
3145         $state = $tpm->{file};
3146     }
3147
3148     my $paths = get_tpm_paths($vmid);
3149
3150     # during migration, we will get state from remote
3151     #
3152     if (!$migration) {
3153         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3154         my $setup_cmd = [
3155             "swtpm_setup",
3156             "--tpmstate",
3157             "file://$state",
3158             "--createek",
3159             "--create-ek-cert",
3160             "--create-platform-cert",
3161             "--lock-nvram",
3162             "--config",
3163             "/etc/swtpm_setup.conf", # do not use XDG configs
3164             "--runas",
3165             "0", # force creation as root, error if not possible
3166             "--not-overwrite", # ignore existing state, do not modify
3167         ];
3168
3169         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3170         # TPM 2.0 supports ECC crypto, use if possible
3171         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3172
3173         run_command($setup_cmd, outfunc => sub {
3174             print "swtpm_setup: $1\n";
3175         });
3176     }
3177
3178     # Used to distinguish different invocations in the log.
3179     my $log_prefix = "[id=" . int(time()) . "] ";
3180
3181     my $emulator_cmd = [
3182         "swtpm",
3183         "socket",
3184         "--tpmstate",
3185         "backend-uri=file://$state,mode=0600",
3186         "--ctrl",
3187         "type=unixio,path=$paths->{socket},mode=0600",
3188         "--pid",
3189         "file=$paths->{pid}",
3190         "--terminate", # terminate on QEMU disconnect
3191         "--daemon",
3192         "--log",
3193         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3194     ];
3195     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3196     run_command($emulator_cmd, outfunc => sub { print $1; });
3197
3198     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3199     while (! -e $paths->{pid}) {
3200         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3201         usleep(50_000);
3202     }
3203
3204     # return untainted PID of swtpm daemon so it can be killed on error
3205     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3206     return $1;
3207 }
3208
3209 sub vga_conf_has_spice {
3210     my ($vga) = @_;
3211
3212     my $vgaconf = parse_vga($vga);
3213     my $vgatype = $vgaconf->{type};
3214     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3215
3216     return $1 || 1;
3217 }
3218
3219 sub get_vm_arch {
3220     my ($conf) = @_;
3221     return $conf->{arch} // get_host_arch();
3222 }
3223
3224 my $default_machines = {
3225     x86_64 => 'pc',
3226     aarch64 => 'virt',
3227 };
3228
3229 sub get_installed_machine_version {
3230     my ($kvmversion) = @_;
3231     $kvmversion = kvm_user_version() if !defined($kvmversion);
3232     $kvmversion =~ m/^(\d+\.\d+)/;
3233     return $1;
3234 }
3235
3236 sub windows_get_pinned_machine_version {
3237     my ($machine, $base_version, $kvmversion) = @_;
3238
3239     my $pin_version = $base_version;
3240     if (!defined($base_version) ||
3241         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3242     ) {
3243         $pin_version = get_installed_machine_version($kvmversion);
3244     }
3245     if (!$machine || $machine eq 'pc') {
3246         $machine = "pc-i440fx-$pin_version";
3247     } elsif ($machine eq 'q35') {
3248         $machine = "pc-q35-$pin_version";
3249     } elsif ($machine eq 'virt') {
3250         $machine = "virt-$pin_version";
3251     } else {
3252         warn "unknown machine type '$machine', not touching that!\n";
3253     }
3254
3255     return $machine;
3256 }
3257
3258 sub get_vm_machine {
3259     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3260
3261     my $machine = $forcemachine || $conf->{machine};
3262
3263     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3264         $kvmversion //= kvm_user_version();
3265         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3266         # layout which confuses windows quite a bit and may result in various regressions..
3267         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3268         if (windows_version($conf->{ostype})) {
3269             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3270         }
3271         $arch //= 'x86_64';
3272         $machine ||= $default_machines->{$arch};
3273         if ($add_pve_version) {
3274             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3275             $machine .= "+pve$pvever";
3276         }
3277     }
3278
3279     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3280         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3281         $machine = $1 if $is_pxe;
3282
3283         # for version-pinned machines that do not include a pve-version (e.g.
3284         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3285         $machine .= '+pve0';
3286
3287         $machine .= '.pxe' if $is_pxe;
3288     }
3289
3290     return $machine;
3291 }
3292
3293 sub get_ovmf_files($$$) {
3294     my ($arch, $efidisk, $smm) = @_;
3295
3296     my $types = $OVMF->{$arch}
3297         or die "no OVMF images known for architecture '$arch'\n";
3298
3299     my $type = 'default';
3300     if ($arch eq 'x86_64') {
3301         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3302             $type = $smm ? "4m" : "4m-no-smm";
3303             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3304         } else {
3305             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3306         }
3307     }
3308
3309     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3310     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3311     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3312
3313     return ($ovmf_code, $ovmf_vars);
3314 }
3315
3316 my $Arch2Qemu = {
3317     aarch64 => '/usr/bin/qemu-system-aarch64',
3318     x86_64 => '/usr/bin/qemu-system-x86_64',
3319 };
3320 sub get_command_for_arch($) {
3321     my ($arch) = @_;
3322     return '/usr/bin/kvm' if is_native_arch($arch);
3323
3324     my $cmd = $Arch2Qemu->{$arch}
3325         or die "don't know how to emulate architecture '$arch'\n";
3326     return $cmd;
3327 }
3328
3329 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3330 # to use in a QEMU command line (-cpu element), first array_intersect the result
3331 # of query_supported_ with query_understood_. This is necessary because:
3332 #
3333 # a) query_understood_ returns flags the host cannot use and
3334 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3335 #    flags, but CPU settings - with most of them being flags. Those settings
3336 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3337 #
3338 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3339 # expensive. If you need the value returned from this, you can get it much
3340 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3341 # $accel being 'kvm' or 'tcg'.
3342 #
3343 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3344 # changes, automatically populating pmxcfs.
3345 #
3346 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3347 # since kvm and tcg machines support different flags
3348 #
3349 sub query_supported_cpu_flags {
3350     my ($arch) = @_;
3351
3352     $arch //= get_host_arch();
3353     my $default_machine = $default_machines->{$arch};
3354
3355     my $flags = {};
3356
3357     # FIXME: Once this is merged, the code below should work for ARM as well:
3358     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3359     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3360         $arch eq "aarch64";
3361
3362     my $kvm_supported = defined(kvm_version());
3363     my $qemu_cmd = get_command_for_arch($arch);
3364     my $fakevmid = -1;
3365     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3366
3367     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3368     my $query_supported_run_qemu = sub {
3369         my ($kvm) = @_;
3370
3371         my $flags = {};
3372         my $cmd = [
3373             $qemu_cmd,
3374             '-machine', $default_machine,
3375             '-display', 'none',
3376             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3377             '-mon', 'chardev=qmp,mode=control',
3378             '-pidfile', $pidfile,
3379             '-S', '-daemonize'
3380         ];
3381
3382         if (!$kvm) {
3383             push @$cmd, '-accel', 'tcg';
3384         }
3385
3386         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3387         die "QEMU flag querying VM exited with code " . $rc if $rc;
3388
3389         eval {
3390             my $cmd_result = mon_cmd(
3391                 $fakevmid,
3392                 'query-cpu-model-expansion',
3393                 type => 'full',
3394                 model => { name => 'host' }
3395             );
3396
3397             my $props = $cmd_result->{model}->{props};
3398             foreach my $prop (keys %$props) {
3399                 next if $props->{$prop} ne '1';
3400                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3401                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3402                 # We only keep those with underscores, to match /proc/cpuinfo
3403                 $prop =~ s/\.|-/_/g;
3404                 $flags->{$prop} = 1;
3405             }
3406         };
3407         my $err = $@;
3408
3409         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3410         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3411
3412         die $err if $err;
3413
3414         return [ sort keys %$flags ];
3415     };
3416
3417     # We need to query QEMU twice, since KVM and TCG have different supported flags
3418     PVE::QemuConfig->lock_config($fakevmid, sub {
3419         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3420         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3421
3422         if ($kvm_supported) {
3423             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3424             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3425         }
3426     });
3427
3428     return $flags;
3429 }
3430
3431 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3432 my $understood_cpu_flag_dir = "/usr/share/kvm";
3433 sub query_understood_cpu_flags {
3434     my $arch = get_host_arch();
3435     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3436
3437     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3438         if ! -e $filepath;
3439
3440     my $raw = file_get_contents($filepath);
3441     $raw =~ s/^\s+|\s+$//g;
3442     my @flags = split(/\s+/, $raw);
3443
3444     return \@flags;
3445 }
3446
3447 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3448 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3449 my sub should_disable_smm {
3450     my ($conf, $vga, $machine) = @_;
3451
3452     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3453
3454     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3455         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3456 }
3457
3458 my sub print_ovmf_drive_commandlines {
3459     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3460
3461     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3462
3463     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3464
3465     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3466     if ($d) {
3467         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3468         my ($path, $format) = $d->@{'file', 'format'};
3469         if ($storeid) {
3470             $path = PVE::Storage::path($storecfg, $d->{file});
3471             if (!defined($format)) {
3472                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3473                 $format = qemu_img_format($scfg, $volname);
3474             }
3475         } elsif (!defined($format)) {
3476             die "efidisk format must be specified\n";
3477         }
3478         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3479         if ($path =~ m/^rbd:/) {
3480             $var_drive_str .= ',cache=writeback';
3481             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3482         }
3483         $var_drive_str .= ",format=$format,file=$path";
3484
3485         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3486         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3487     } else {
3488         log_warn("no efidisk configured! Using temporary efivars disk.");
3489         my $path = "/tmp/$vmid-ovmf.fd";
3490         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3491         $var_drive_str .= ",format=raw,file=$path";
3492         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3493     }
3494
3495     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3496 }
3497
3498 sub config_to_command {
3499     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3500         $live_restore_backing) = @_;
3501
3502     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3503     my $devices = [];
3504     my $bridges = {};
3505     my $ostype = $conf->{ostype};
3506     my $winversion = windows_version($ostype);
3507     my $kvm = $conf->{kvm};
3508     my $nodename = nodename();
3509
3510     my $arch = get_vm_arch($conf);
3511     my $kvm_binary = get_command_for_arch($arch);
3512     my $kvmver = kvm_user_version($kvm_binary);
3513
3514     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3515         $kvmver //= "undefined";
3516         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3517     }
3518
3519     my $add_pve_version = min_version($kvmver, 4, 1);
3520
3521     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3522     my $machine_version = extract_version($machine_type, $kvmver);
3523     $kvm //= 1 if is_native_arch($arch);
3524
3525     $machine_version =~ m/(\d+)\.(\d+)/;
3526     my ($machine_major, $machine_minor) = ($1, $2);
3527
3528     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3529         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3530     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3531         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3532             ." please upgrade node '$nodename'\n"
3533     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3534         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3535         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3536             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3537             ." node '$nodename'\n";
3538     }
3539
3540     # if a specific +pve version is required for a feature, use $version_guard
3541     # instead of min_version to allow machines to be run with the minimum
3542     # required version
3543     my $required_pve_version = 0;
3544     my $version_guard = sub {
3545         my ($major, $minor, $pve) = @_;
3546         return 0 if !min_version($machine_version, $major, $minor, $pve);
3547         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3548         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3549         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3550         return 1;
3551     };
3552
3553     if ($kvm && !defined kvm_version()) {
3554         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3555             ." or enable in BIOS.\n";
3556     }
3557
3558     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3559     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3560     my $use_old_bios_files = undef;
3561     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3562
3563     my $cmd = [];
3564     if ($conf->{affinity}) {
3565         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3566     }
3567
3568     push @$cmd, $kvm_binary;
3569
3570     push @$cmd, '-id', $vmid;
3571
3572     my $vmname = $conf->{name} || "vm$vmid";
3573
3574     push @$cmd, '-name', "$vmname,debug-threads=on";
3575
3576     push @$cmd, '-no-shutdown';
3577
3578     my $use_virtio = 0;
3579
3580     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3581     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3582     push @$cmd, '-mon', "chardev=qmp,mode=control";
3583
3584     if (min_version($machine_version, 2, 12)) {
3585         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3586         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3587     }
3588
3589     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3590
3591     push @$cmd, '-daemonize';
3592
3593     if ($conf->{smbios1}) {
3594         my $smbios_conf = parse_smbios1($conf->{smbios1});
3595         if ($smbios_conf->{base64}) {
3596             # Do not pass base64 flag to qemu
3597             delete $smbios_conf->{base64};
3598             my $smbios_string = "";
3599             foreach my $key (keys %$smbios_conf) {
3600                 my $value;
3601                 if ($key eq "uuid") {
3602                     $value = $smbios_conf->{uuid}
3603                 } else {
3604                     $value = decode_base64($smbios_conf->{$key});
3605                 }
3606                 # qemu accepts any binary data, only commas need escaping by double comma
3607                 $value =~ s/,/,,/g;
3608                 $smbios_string .= "," . $key . "=" . $value if $value;
3609             }
3610             push @$cmd, '-smbios', "type=1" . $smbios_string;
3611         } else {
3612             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3613         }
3614     }
3615
3616     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3617         die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3618             if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3619
3620         my ($code_drive_str, $var_drive_str) =
3621             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3622         push $cmd->@*, '-drive', $code_drive_str;
3623         push $cmd->@*, '-drive', $var_drive_str;
3624     }
3625
3626     if ($q35) { # tell QEMU to load q35 config early
3627         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628         if (min_version($machine_version, 4, 0)) {
3629             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3630         } else {
3631             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632         }
3633     }
3634
3635     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3636         push @$cmd, $fixups->@*;
3637     }
3638
3639     if ($conf->{vmgenid}) {
3640         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3641     }
3642
3643     # add usb controllers
3644     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3645         $conf, $bridges, $arch, $machine_type, $machine_version);
3646     push @$devices, @usbcontrollers if @usbcontrollers;
3647     my $vga = parse_vga($conf->{vga});
3648
3649     my $qxlnum = vga_conf_has_spice($conf->{vga});
3650     $vga->{type} = 'qxl' if $qxlnum;
3651
3652     if (!$vga->{type}) {
3653         if ($arch eq 'aarch64') {
3654             $vga->{type} = 'virtio';
3655         } elsif (min_version($machine_version, 2, 9)) {
3656             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657         } else {
3658             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659         }
3660     }
3661
3662     # enable absolute mouse coordinates (needed by vnc)
3663     my $tablet = $conf->{tablet};
3664     if (!defined($tablet)) {
3665         $tablet = $defaults->{tablet};
3666         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668     }
3669
3670     if ($tablet) {
3671         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3672         my $kbd = print_keyboarddevice_full($conf, $arch);
3673         push @$devices, '-device', $kbd if defined($kbd);
3674     }
3675
3676     my $bootorder = device_bootorder($conf);
3677
3678     # host pci device passthrough
3679     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3680         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3681
3682     # usb devices
3683     my $usb_dev_features = {};
3684     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3685
3686     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3687         $conf, $usb_dev_features, $bootorder, $machine_version);
3688     push @$devices, @usbdevices if @usbdevices;
3689
3690     # serial devices
3691     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3692         my $path = $conf->{"serial$i"} or next;
3693         if ($path eq 'socket') {
3694             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696             # On aarch64, serial0 is the UART device. QEMU only allows
3697             # connecting UART devices via the '-serial' command line, as
3698             # the device has a fixed slot on the hardware...
3699             if ($arch eq 'aarch64' && $i == 0) {
3700                 push @$devices, '-serial', "chardev:serial$i";
3701             } else {
3702                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703             }
3704         } else {
3705             die "no such serial device\n" if ! -c $path;
3706             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3707             push @$devices, '-device', "isa-serial,chardev=serial$i";
3708         }
3709     }
3710
3711     # parallel devices
3712     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3713         if (my $path = $conf->{"parallel$i"}) {
3714             die "no such parallel device\n" if ! -c $path;
3715             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3716             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718         }
3719     }
3720
3721     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3722         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3723         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3724         push @$devices, @$audio_devs;
3725     }
3726
3727     # Add a TPM only if the VM is not a template,
3728     # to support backing up template VMs even if the TPM disk is write-protected.
3729     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3730
3731     my $sockets = 1;
3732     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3733     $sockets = $conf->{sockets} if  $conf->{sockets};
3734
3735     my $cores = $conf->{cores} || 1;
3736
3737     my $maxcpus = $sockets * $cores;
3738
3739     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3740
3741     my $allowed_vcpus = $cpuinfo->{cpus};
3742
3743     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3744
3745     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3746         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747         for (my $i = 2; $i <= $vcpus; $i++)  {
3748             my $cpustr = print_cpu_device($conf, $arch, $i);
3749             push @$cmd, '-device', $cpustr;
3750         }
3751
3752     } else {
3753
3754         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755     }
3756     push @$cmd, '-nodefaults';
3757
3758     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3759
3760     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3761
3762     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3763
3764     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3765         push @$devices, '-device', print_vga_device(
3766             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3767
3768         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3769
3770         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3771         push @$cmd,  '-vnc', "unix:$socket,password=on";
3772     } else {
3773         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3774         push @$cmd, '-nographic';
3775     }
3776
3777     # time drift fix
3778     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3779     my $useLocaltime = $conf->{localtime};
3780
3781     if ($winversion >= 5) { # windows
3782         $useLocaltime = 1 if !defined($conf->{localtime});
3783
3784         # use time drift fix when acpi is enabled
3785         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3786             $tdf = 1 if !defined($conf->{tdf});
3787         }
3788     }
3789
3790     if ($winversion >= 6) {
3791         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3792         push @$machineFlags, 'hpet=off';
3793     }
3794
3795     push @$rtcFlags, 'driftfix=slew' if $tdf;
3796
3797     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3798         push @$rtcFlags, "base=$conf->{startdate}";
3799     } elsif ($useLocaltime) {
3800         push @$rtcFlags, 'base=localtime';
3801     }
3802
3803     if ($forcecpu) {
3804         push @$cmd, '-cpu', $forcecpu;
3805     } else {
3806         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3807     }
3808
3809     PVE::QemuServer::Memory::config(
3810         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3811
3812     push @$cmd, '-S' if $conf->{freeze};
3813
3814     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3815
3816     my $guest_agent = parse_guest_agent($conf);
3817
3818     if ($guest_agent->{enabled}) {
3819         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3820         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3821
3822         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3823             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3824             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3825             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3826         } elsif ($guest_agent->{type} eq 'isa') {
3827             push @$devices, '-device', "isa-serial,chardev=qga0";
3828         }
3829     }
3830
3831     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3832     if ($rng && $version_guard->(4, 1, 2)) {
3833         check_rng_source($rng->{source});
3834
3835         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3836         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3837         my $limiter_str = "";
3838         if ($max_bytes) {
3839             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3840         }
3841
3842         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3843         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3844         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3845     }
3846
3847     my $spice_port;
3848
3849     assert_clipboard_config($vga);
3850     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3851
3852     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3853         if ($qxlnum > 1) {
3854             if ($winversion){
3855                 for (my $i = 1; $i < $qxlnum; $i++){
3856                     push @$devices, '-device', print_vga_device(
3857                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3858                 }
3859             } else {
3860                 # assume other OS works like Linux
3861                 my ($ram, $vram) = ("134217728", "67108864");
3862                 if ($vga->{memory}) {
3863                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3864                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3865                 }
3866                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3867                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3868             }
3869         }
3870
3871         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3872
3873         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3874         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3875             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3876         } else {
3877             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3878         }
3879         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880
3881         if ($is_spice) {
3882             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3883             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3884             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3885
3886             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3887             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3888
3889             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3890             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3891             if ($spice_enhancement->{foldersharing}) {
3892                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3894             }
3895
3896             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898                 if $spice_enhancement->{videostreaming};
3899             push @$devices, '-spice', "$spice_opts";
3900         }
3901     }
3902
3903     # enable balloon by default, unless explicitly disabled
3904     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3905         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3906         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3908         push @$devices, '-device', $ballooncmd;
3909     }
3910
3911     if ($conf->{watchdog}) {
3912         my $wdopts = parse_watchdog($conf->{watchdog});
3913         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3914         my $watchdog = $wdopts->{model} || 'i6300esb';
3915         push @$devices, '-device', "$watchdog$pciaddr";
3916         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3917     }
3918
3919     my $vollist = [];
3920     my $scsicontroller = {};
3921     my $ahcicontroller = {};
3922     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3923
3924     # Add iscsi initiator name if available
3925     if (my $initiator = get_initiator_name()) {
3926         push @$devices, '-iscsi', "initiator-name=$initiator";
3927     }
3928
3929     PVE::QemuConfig->foreach_volume($conf, sub {
3930         my ($ds, $drive) = @_;
3931
3932         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3933             check_volume_storage_type($storecfg, $drive->{file});
3934             push @$vollist, $drive->{file};
3935         }
3936
3937         # ignore efidisk here, already added in bios/fw handling code above
3938         return if $drive->{interface} eq 'efidisk';
3939         # similar for TPM
3940         return if $drive->{interface} eq 'tpmstate';
3941
3942         $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3945
3946         if ($drive->{interface} eq 'virtio'){
3947            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3948         }
3949
3950         if ($drive->{interface} eq 'scsi') {
3951
3952             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3953
3954             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3956
3957             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3959
3960             my $iothread = '';
3961             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3962                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964             } elsif ($drive->{iothread}) {
3965                 log_warn(
3966                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3967                 );
3968             }
3969
3970             my $queues = '';
3971             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3972                 $queues = ",num_queues=$drive->{queues}";
3973             }
3974
3975             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3976                 if !$scsicontroller->{$controller};
3977             $scsicontroller->{$controller}=1;
3978         }
3979
3980         if ($drive->{interface} eq 'sata') {
3981             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3982             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3983             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3984                 if !$ahcicontroller->{$controller};
3985             $ahcicontroller->{$controller}=1;
3986         }
3987
3988         my $live_restore = $live_restore_backing->{$ds};
3989         my $live_blockdev_name = undef;
3990         if ($live_restore) {
3991             $live_blockdev_name = $live_restore->{name};
3992             push @$devices, '-blockdev', $live_restore->{blockdev};
3993         }
3994
3995         my $drive_cmd = print_drive_commandline_full(
3996             $storecfg, $vmid, $drive, $live_blockdev_name, min_version($kvmver, 6, 0));
3997
3998         # extra protection for templates, but SATA and IDE don't support it..
3999         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4000
4001         push @$devices, '-drive',$drive_cmd;
4002         push @$devices, '-device', print_drivedevice_full(
4003             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4004     });
4005
4006     for (my $i = 0; $i < $MAX_NETS; $i++) {
4007         my $netname = "net$i";
4008
4009         next if !$conf->{$netname};
4010         my $d = parse_net($conf->{$netname});
4011         next if !$d;
4012         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4013
4014         $use_virtio = 1 if $d->{model} eq 'virtio';
4015
4016         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4017
4018         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4019         push @$devices, '-netdev', $netdevfull;
4020
4021         my $netdevicefull = print_netdevice_full(
4022             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4023
4024         push @$devices, '-device', $netdevicefull;
4025     }
4026
4027     if ($conf->{ivshmem}) {
4028         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4029
4030         my $bus;
4031         if ($q35) {
4032             $bus = print_pcie_addr("ivshmem");
4033         } else {
4034             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4035         }
4036
4037         my $ivshmem_name = $ivshmem->{name} // $vmid;
4038         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4039
4040         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4042             .",size=$ivshmem->{size}M";
4043     }
4044
4045     # pci.4 is nested in pci.1
4046     $bridges->{1} = 1 if $bridges->{4};
4047
4048     if (!$q35) { # add pci bridges
4049         if (min_version($machine_version, 2, 3)) {
4050            $bridges->{1} = 1;
4051            $bridges->{2} = 1;
4052         }
4053         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4054     }
4055
4056     for my $k (sort {$b cmp $a} keys %$bridges) {
4057         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4058
4059         my $k_name = $k;
4060         if ($k == 2 && $legacy_igd) {
4061             $k_name = "$k-igd";
4062         }
4063         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4064         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4065
4066         if ($q35) { # add after -readconfig pve-q35.cfg
4067             splice @$devices, 2, 0, '-device', $devstr;
4068         } else {
4069             unshift @$devices, '-device', $devstr if $k > 0;
4070         }
4071     }
4072
4073     if (!$kvm) {
4074         push @$machineFlags, 'accel=tcg';
4075     }
4076
4077     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4078
4079     my $machine_type_min = $machine_type;
4080     if ($add_pve_version) {
4081         $machine_type_min =~ s/\+pve\d+$//;
4082         $machine_type_min .= "+pve$required_pve_version";
4083     }
4084     push @$machineFlags, "type=${machine_type_min}";
4085
4086     push @$cmd, @$devices;
4087     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4088     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4089     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4090
4091     if (my $vmstate = $conf->{vmstate}) {
4092         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4093         push @$vollist, $vmstate;
4094         push @$cmd, '-loadstate', $statepath;
4095         print "activating and using '$vmstate' as vmstate\n";
4096     }
4097
4098     if (PVE::QemuConfig->is_template($conf)) {
4099         # needed to workaround base volumes being read-only
4100         push @$cmd, '-snapshot';
4101     }
4102
4103     # add custom args
4104     if ($conf->{args}) {
4105         my $aa = PVE::Tools::split_args($conf->{args});
4106         push @$cmd, @$aa;
4107     }
4108
4109     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4110 }
4111
4112 sub check_rng_source {
4113     my ($source) = @_;
4114
4115     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4116     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4117         if ! -e $source;
4118
4119     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4120     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4121         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4122         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4123         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4124             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4125             ." to the host.\n";
4126     }
4127 }
4128
4129 sub spice_port {
4130     my ($vmid) = @_;
4131
4132     my $res = mon_cmd($vmid, 'query-spice');
4133
4134     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4135 }
4136
4137 sub vm_devices_list {
4138     my ($vmid) = @_;
4139
4140     my $res = mon_cmd($vmid, 'query-pci');
4141     my $devices_to_check = [];
4142     my $devices = {};
4143     foreach my $pcibus (@$res) {
4144         push @$devices_to_check, @{$pcibus->{devices}},
4145     }
4146
4147     while (@$devices_to_check) {
4148         my $to_check = [];
4149         for my $d (@$devices_to_check) {
4150             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4151             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4152
4153             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4154             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4155         }
4156         $devices_to_check = $to_check;
4157     }
4158
4159     my $resblock = mon_cmd($vmid, 'query-block');
4160     foreach my $block (@$resblock) {
4161         if($block->{device} =~ m/^drive-(\S+)/){
4162                 $devices->{$1} = 1;
4163         }
4164     }
4165
4166     my $resmice = mon_cmd($vmid, 'query-mice');
4167     foreach my $mice (@$resmice) {
4168         if ($mice->{name} eq 'QEMU HID Tablet') {
4169             $devices->{tablet} = 1;
4170             last;
4171         }
4172     }
4173
4174     # for usb devices there is no query-usb
4175     # but we can iterate over the entries in
4176     # qom-list path=/machine/peripheral
4177     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4178     foreach my $per (@$resperipheral) {
4179         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4180             $devices->{$per->{name}} = 1;
4181         }
4182     }
4183
4184     return $devices;
4185 }
4186
4187 sub vm_deviceplug {
4188     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4189
4190     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4191
4192     my $devices_list = vm_devices_list($vmid);
4193     return 1 if defined($devices_list->{$deviceid});
4194
4195     # add PCI bridge if we need it for the device
4196     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4197
4198     if ($deviceid eq 'tablet') {
4199         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4200     } elsif ($deviceid eq 'keyboard') {
4201         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4202     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4203         my $id = $1;
4204         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4205         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4206     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4207         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4208     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4209         qemu_iothread_add($vmid, $deviceid, $device);
4210
4211         qemu_driveadd($storecfg, $vmid, $device);
4212         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4213
4214         qemu_deviceadd($vmid, $devicefull);
4215         eval { qemu_deviceaddverify($vmid, $deviceid); };
4216         if (my $err = $@) {
4217             eval { qemu_drivedel($vmid, $deviceid); };
4218             warn $@ if $@;
4219             die $err;
4220         }
4221     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4222         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4223         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4224         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4225
4226         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4227
4228         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4229             qemu_iothread_add($vmid, $deviceid, $device);
4230             $devicefull .= ",iothread=iothread-$deviceid";
4231         }
4232
4233         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4234             $devicefull .= ",num_queues=$device->{queues}";
4235         }
4236
4237         qemu_deviceadd($vmid, $devicefull);
4238         qemu_deviceaddverify($vmid, $deviceid);
4239     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4240         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4241         qemu_driveadd($storecfg, $vmid, $device);
4242
4243         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4244         eval { qemu_deviceadd($vmid, $devicefull); };
4245         if (my $err = $@) {
4246             eval { qemu_drivedel($vmid, $deviceid); };
4247             warn $@ if $@;
4248             die $err;
4249         }
4250     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4251         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4252
4253         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4254         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4255         my $use_old_bios_files = undef;
4256         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4257
4258         my $netdevicefull = print_netdevice_full(
4259             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4260         qemu_deviceadd($vmid, $netdevicefull);
4261         eval {
4262             qemu_deviceaddverify($vmid, $deviceid);
4263             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4264         };
4265         if (my $err = $@) {
4266             eval { qemu_netdevdel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4271         my $bridgeid = $2;
4272         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4273         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4274
4275         qemu_deviceadd($vmid, $devicefull);
4276         qemu_deviceaddverify($vmid, $deviceid);
4277     } else {
4278         die "can't hotplug device '$deviceid'\n";
4279     }
4280
4281     return 1;
4282 }
4283
4284 # fixme: this should raise exceptions on error!
4285 sub vm_deviceunplug {
4286     my ($vmid, $conf, $deviceid) = @_;
4287
4288     my $devices_list = vm_devices_list($vmid);
4289     return 1 if !defined($devices_list->{$deviceid});
4290
4291     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4292     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4293
4294     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4295         qemu_devicedel($vmid, $deviceid);
4296     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4297         qemu_devicedel($vmid, $deviceid);
4298         qemu_devicedelverify($vmid, $deviceid);
4299     } elsif ($deviceid =~ m/^usb\d+$/) {
4300         qemu_devicedel($vmid, $deviceid);
4301         qemu_devicedelverify($vmid, $deviceid);
4302     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4303         my $device = parse_drive($deviceid, $conf->{$deviceid});
4304
4305         qemu_devicedel($vmid, $deviceid);
4306         qemu_devicedelverify($vmid, $deviceid);
4307         qemu_drivedel($vmid, $deviceid);
4308         qemu_iothread_del($vmid, $deviceid, $device);
4309     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4310         qemu_devicedel($vmid, $deviceid);
4311         qemu_devicedelverify($vmid, $deviceid);
4312     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4313         my $device = parse_drive($deviceid, $conf->{$deviceid});
4314
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_drivedel($vmid, $deviceid);
4318         qemu_deletescsihw($conf, $vmid, $deviceid);
4319
4320         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4321             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4322     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4323         qemu_devicedel($vmid, $deviceid);
4324         qemu_devicedelverify($vmid, $deviceid);
4325         qemu_netdevdel($vmid, $deviceid);
4326     } else {
4327         die "can't unplug device '$deviceid'\n";
4328     }
4329
4330     return 1;
4331 }
4332
4333 sub qemu_spice_usbredir_chardev_add {
4334     my ($vmid, $id) = @_;
4335
4336     mon_cmd($vmid, "chardev-add" , (
4337         id => $id,
4338         backend => {
4339             type => 'spicevmc',
4340             data => {
4341                 type => "usbredir",
4342             },
4343         },
4344     ));
4345 }
4346
4347 sub qemu_iothread_add {
4348     my ($vmid, $deviceid, $device) = @_;
4349
4350     if ($device->{iothread}) {
4351         my $iothreads = vm_iothreads_list($vmid);
4352         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4353     }
4354 }
4355
4356 sub qemu_iothread_del {
4357     my ($vmid, $deviceid, $device) = @_;
4358
4359     if ($device->{iothread}) {
4360         my $iothreads = vm_iothreads_list($vmid);
4361         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4362     }
4363 }
4364
4365 sub qemu_driveadd {
4366     my ($storecfg, $vmid, $device) = @_;
4367
4368     my $kvmver = get_running_qemu_version($vmid);
4369     my $io_uring = min_version($kvmver, 6, 0);
4370     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4371     $drive =~ s/\\/\\\\/g;
4372     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4373
4374     # If the command succeeds qemu prints: "OK"
4375     return 1 if $ret =~ m/OK/s;
4376
4377     die "adding drive failed: $ret\n";
4378 }
4379
4380 sub qemu_drivedel {
4381     my ($vmid, $deviceid) = @_;
4382
4383     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4384     $ret =~ s/^\s+//;
4385
4386     return 1 if $ret eq "";
4387
4388     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4389     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4390
4391     die "deleting drive $deviceid failed : $ret\n";
4392 }
4393
4394 sub qemu_deviceaddverify {
4395     my ($vmid, $deviceid) = @_;
4396
4397     for (my $i = 0; $i <= 5; $i++) {
4398          my $devices_list = vm_devices_list($vmid);
4399          return 1 if defined($devices_list->{$deviceid});
4400          sleep 1;
4401     }
4402
4403     die "error on hotplug device '$deviceid'\n";
4404 }
4405
4406
4407 sub qemu_devicedelverify {
4408     my ($vmid, $deviceid) = @_;
4409
4410     # need to verify that the device is correctly removed as device_del
4411     # is async and empty return is not reliable
4412
4413     for (my $i = 0; $i <= 5; $i++) {
4414          my $devices_list = vm_devices_list($vmid);
4415          return 1 if !defined($devices_list->{$deviceid});
4416          sleep 1;
4417     }
4418
4419     die "error on hot-unplugging device '$deviceid'\n";
4420 }
4421
4422 sub qemu_findorcreatescsihw {
4423     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4424
4425     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4426
4427     my $scsihwid="$controller_prefix$controller";
4428     my $devices_list = vm_devices_list($vmid);
4429
4430     if (!defined($devices_list->{$scsihwid})) {
4431         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4432     }
4433
4434     return 1;
4435 }
4436
4437 sub qemu_deletescsihw {
4438     my ($conf, $vmid, $opt) = @_;
4439
4440     my $device = parse_drive($opt, $conf->{$opt});
4441
4442     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4443         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4444         return 1;
4445     }
4446
4447     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4448
4449     my $devices_list = vm_devices_list($vmid);
4450     foreach my $opt (keys %{$devices_list}) {
4451         if (is_valid_drivename($opt)) {
4452             my $drive = parse_drive($opt, $conf->{$opt});
4453             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4454                 return 1;
4455             }
4456         }
4457     }
4458
4459     my $scsihwid="scsihw$controller";
4460
4461     vm_deviceunplug($vmid, $conf, $scsihwid);
4462
4463     return 1;
4464 }
4465
4466 sub qemu_add_pci_bridge {
4467     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4468
4469     my $bridges = {};
4470
4471     my $bridgeid;
4472
4473     print_pci_addr($device, $bridges, $arch, $machine_type);
4474
4475     while (my ($k, $v) = each %$bridges) {
4476         $bridgeid = $k;
4477     }
4478     return 1 if !defined($bridgeid) || $bridgeid < 1;
4479
4480     my $bridge = "pci.$bridgeid";
4481     my $devices_list = vm_devices_list($vmid);
4482
4483     if (!defined($devices_list->{$bridge})) {
4484         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4485     }
4486
4487     return 1;
4488 }
4489
4490 sub qemu_set_link_status {
4491     my ($vmid, $device, $up) = @_;
4492
4493     mon_cmd($vmid, "set_link", name => $device,
4494                up => $up ? JSON::true : JSON::false);
4495 }
4496
4497 sub qemu_netdevadd {
4498     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4499
4500     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4501     my %options =  split(/[=,]/, $netdev);
4502
4503     if (defined(my $vhost = $options{vhost})) {
4504         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4505     }
4506
4507     if (defined(my $queues = $options{queues})) {
4508         $options{queues} = $queues + 0;
4509     }
4510
4511     mon_cmd($vmid, "netdev_add",  %options);
4512     return 1;
4513 }
4514
4515 sub qemu_netdevdel {
4516     my ($vmid, $deviceid) = @_;
4517
4518     mon_cmd($vmid, "netdev_del", id => $deviceid);
4519 }
4520
4521 sub qemu_usb_hotplug {
4522     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4523
4524     return if !$device;
4525
4526     # remove the old one first
4527     vm_deviceunplug($vmid, $conf, $deviceid);
4528
4529     # check if xhci controller is necessary and available
4530     my $devicelist = vm_devices_list($vmid);
4531
4532     if (!$devicelist->{xhci}) {
4533         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4534         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4535     }
4536
4537     # add the new one
4538     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4539 }
4540
4541 sub qemu_cpu_hotplug {
4542     my ($vmid, $conf, $vcpus) = @_;
4543
4544     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4545
4546     my $sockets = 1;
4547     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4548     $sockets = $conf->{sockets} if  $conf->{sockets};
4549     my $cores = $conf->{cores} || 1;
4550     my $maxcpus = $sockets * $cores;
4551
4552     $vcpus = $maxcpus if !$vcpus;
4553
4554     die "you can't add more vcpus than maxcpus\n"
4555         if $vcpus > $maxcpus;
4556
4557     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4558
4559     if ($vcpus < $currentvcpus) {
4560
4561         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4562
4563             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4564                 qemu_devicedel($vmid, "cpu$i");
4565                 my $retry = 0;
4566                 my $currentrunningvcpus = undef;
4567                 while (1) {
4568                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4569                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4570                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4571                     $retry++;
4572                     sleep 1;
4573                 }
4574                 #update conf after each succesfull cpu unplug
4575                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4576                 PVE::QemuConfig->write_config($vmid, $conf);
4577             }
4578         } else {
4579             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4580         }
4581
4582         return;
4583     }
4584
4585     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4586     die "vcpus in running vm does not match its configuration\n"
4587         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4588
4589     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4590         my $arch = get_vm_arch($conf);
4591
4592         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593             my $cpustr = print_cpu_device($conf, $arch, $i);
4594             qemu_deviceadd($vmid, $cpustr);
4595
4596             my $retry = 0;
4597             my $currentrunningvcpus = undef;
4598             while (1) {
4599                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                 last if scalar(@{$currentrunningvcpus}) == $i;
4601                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4602                 sleep 1;
4603                 $retry++;
4604             }
4605             #update conf after each succesfull cpu hotplug
4606             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607             PVE::QemuConfig->write_config($vmid, $conf);
4608         }
4609     } else {
4610
4611         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612             mon_cmd($vmid, "cpu-add", id => int($i));
4613         }
4614     }
4615 }
4616
4617 sub qemu_block_set_io_throttle {
4618     my ($vmid, $deviceid,
4619         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4623
4624     return if !check_running($vmid) ;
4625
4626     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4627         bps => int($bps),
4628         bps_rd => int($bps_rd),
4629         bps_wr => int($bps_wr),
4630         iops => int($iops),
4631         iops_rd => int($iops_rd),
4632         iops_wr => int($iops_wr),
4633         bps_max => int($bps_max),
4634         bps_rd_max => int($bps_rd_max),
4635         bps_wr_max => int($bps_wr_max),
4636         iops_max => int($iops_max),
4637         iops_rd_max => int($iops_rd_max),
4638         iops_wr_max => int($iops_wr_max),
4639         bps_max_length => int($bps_max_length),
4640         bps_rd_max_length => int($bps_rd_max_length),
4641         bps_wr_max_length => int($bps_wr_max_length),
4642         iops_max_length => int($iops_max_length),
4643         iops_rd_max_length => int($iops_rd_max_length),
4644         iops_wr_max_length => int($iops_wr_max_length),
4645     );
4646
4647 }
4648
4649 sub qemu_block_resize {
4650     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4655
4656     return if !$running;
4657
4658     my $padding = (1024 - $size % 1024) % 1024;
4659     $size = $size + $padding;
4660
4661     mon_cmd(
4662         $vmid,
4663         "block_resize",
4664         device => $deviceid,
4665         size => int($size),
4666         timeout => 60,
4667     );
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686     my $attached_deviceid;
4687
4688     if ($running) {
4689         my $conf = PVE::QemuConfig->load_config($vmid);
4690         PVE::QemuConfig->foreach_volume($conf, sub {
4691             my ($ds, $drive) = @_;
4692             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4693         });
4694     }
4695
4696     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4697         mon_cmd(
4698             $vmid,
4699             'blockdev-snapshot-delete-internal-sync',
4700             device => $attached_deviceid,
4701             name => $snap,
4702         );
4703     } else {
4704         PVE::Storage::volume_snapshot_delete(
4705             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4706     }
4707 }
4708
4709 sub set_migration_caps {
4710     my ($vmid, $savevm) = @_;
4711
4712     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4713
4714     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4715     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4716
4717     my $cap_ref = [];
4718
4719     my $enabled_cap = {
4720         "auto-converge" => 1,
4721         "xbzrle" => 1,
4722         "x-rdma-pin-all" => 0,
4723         "zero-blocks" => 0,
4724         "compress" => 0,
4725         "dirty-bitmaps" => $dirty_bitmaps,
4726     };
4727
4728     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4729
4730     for my $supported_capability (@$supported_capabilities) {
4731         push @$cap_ref, {
4732             capability => $supported_capability->{capability},
4733             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4734         };
4735     }
4736
4737     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4738 }
4739
4740 sub foreach_volid {
4741     my ($conf, $func, @param) = @_;
4742
4743     my $volhash = {};
4744
4745     my $test_volid = sub {
4746         my ($key, $drive, $snapname, $pending) = @_;
4747
4748         my $volid = $drive->{file};
4749         return if !$volid;
4750
4751         $volhash->{$volid}->{cdrom} //= 1;
4752         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4753
4754         my $replicate = $drive->{replicate} // 1;
4755         $volhash->{$volid}->{replicate} //= 0;
4756         $volhash->{$volid}->{replicate} = 1 if $replicate;
4757
4758         $volhash->{$volid}->{shared} //= 0;
4759         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4760
4761         $volhash->{$volid}->{is_unused} //= 0;
4762         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4763
4764         $volhash->{$volid}->{is_attached} //= 0;
4765         $volhash->{$volid}->{is_attached} = 1
4766             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4767
4768         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4769             if defined($snapname);
4770
4771         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4772
4773         my $size = $drive->{size};
4774         $volhash->{$volid}->{size} //= $size if $size;
4775
4776         $volhash->{$volid}->{is_vmstate} //= 0;
4777         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4778
4779         $volhash->{$volid}->{is_tpmstate} //= 0;
4780         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4781
4782         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4783     };
4784
4785     my $include_opts = {
4786         extra_keys => ['vmstate'],
4787         include_unused => 1,
4788     };
4789
4790     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4791
4792     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4793         if defined($conf->{pending}) && $conf->{pending}->%*;
4794
4795     foreach my $snapname (keys %{$conf->{snapshots}}) {
4796         my $snap = $conf->{snapshots}->{$snapname};
4797         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4798     }
4799
4800     foreach my $volid (keys %$volhash) {
4801         &$func($volid, $volhash->{$volid}, @param);
4802     }
4803 }
4804
4805 my $fast_plug_option = {
4806     'description' => 1,
4807     'hookscript' => 1,
4808     'lock' => 1,
4809     'migrate_downtime' => 1,
4810     'migrate_speed' => 1,
4811     'name' => 1,
4812     'onboot' => 1,
4813     'protection' => 1,
4814     'shares' => 1,
4815     'startup' => 1,
4816     'tags' => 1,
4817     'vmstatestorage' => 1,
4818 };
4819
4820 for my $opt (keys %$confdesc_cloudinit) {
4821     $fast_plug_option->{$opt} = 1;
4822 };
4823
4824 # hotplug changes in [PENDING]
4825 # $selection hash can be used to only apply specified options, for
4826 # example: { cores => 1 } (only apply changed 'cores')
4827 # $errors ref is used to return error messages
4828 sub vmconfig_hotplug_pending {
4829     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4830
4831     my $defaults = load_defaults();
4832     my $arch = get_vm_arch($conf);
4833     my $machine_type = get_vm_machine($conf, undef, $arch);
4834
4835     # commit values which do not have any impact on running VM first
4836     # Note: those option cannot raise errors, we we do not care about
4837     # $selection and always apply them.
4838
4839     my $add_error = sub {
4840         my ($opt, $msg) = @_;
4841         $errors->{$opt} = "hotplug problem - $msg";
4842     };
4843
4844     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4845
4846     my $cloudinit_record_changed = sub {
4847         my ($conf, $opt, $old, $new) = @_;
4848         return if !$cloudinit_pending_properties->{$opt};
4849
4850         my $ci = ($conf->{cloudinit} //= {});
4851
4852         my $recorded = $ci->{$opt};
4853         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4854
4855         if (defined($new)) {
4856             if (defined($old)) {
4857                 # an existing value is being modified
4858                 if (defined($recorded)) {
4859                     # the value was already not in sync
4860                     if ($new eq $recorded) {
4861                         # a value is being reverted to the cloud-init state:
4862                         delete $ci->{$opt};
4863                         delete $added{$opt};
4864                     } else {
4865                         # the value was changed multiple times, do nothing
4866                     }
4867                 } elsif ($added{$opt}) {
4868                     # the value had been marked as added and is being changed, do nothing
4869                 } else {
4870                     # the value is new, record it:
4871                     $ci->{$opt} = $old;
4872                 }
4873             } else {
4874                 # a new value is being added
4875                 if (defined($recorded)) {
4876                     # it was already not in sync
4877                     if ($new eq $recorded) {
4878                         # a value is being reverted to the cloud-init state:
4879                         delete $ci->{$opt};
4880                         delete $added{$opt};
4881                     } else {
4882                         # the value had temporarily been removed, do nothing
4883                     }
4884                 } elsif ($added{$opt}) {
4885                     # the value had been marked as added already, do nothing
4886                 } else {
4887                     # the value is new, add it
4888                     $added{$opt} = 1;
4889                 }
4890             }
4891         } elsif (!defined($old)) {
4892             # a non-existent value is being removed? ignore...
4893         } else {
4894             # a value is being deleted
4895             if (defined($recorded)) {
4896                 # a value was already recorded, just keep it
4897             } elsif ($added{$opt}) {
4898                 # the value was marked as added, remove it
4899                 delete $added{$opt};
4900             } else {
4901                 # a previously unrecorded value is being removed, record the old value:
4902                 $ci->{$opt} = $old;
4903             }
4904         }
4905
4906         my $added = join(',', sort keys %added);
4907         $ci->{added} = $added if length($added);
4908     };
4909
4910     my $changes = 0;
4911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4912         if ($fast_plug_option->{$opt}) {
4913             my $new = delete $conf->{pending}->{$opt};
4914             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4915             $conf->{$opt} = $new;
4916             $changes = 1;
4917         }
4918     }
4919
4920     if ($changes) {
4921         PVE::QemuConfig->write_config($vmid, $conf);
4922     }
4923
4924     my $ostype = $conf->{ostype};
4925     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4926     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4927     my $usb_hotplug = $hotplug_features->{usb}
4928         && min_version($version, 7, 1)
4929         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4930
4931     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4932     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4933
4934     foreach my $opt (sort keys %$pending_delete_hash) {
4935         next if $selection && !$selection->{$opt};
4936         my $force = $pending_delete_hash->{$opt}->{force};
4937         eval {
4938             if ($opt eq 'hotplug') {
4939                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4940             } elsif ($opt eq 'tablet') {
4941                 die "skip\n" if !$hotplug_features->{usb};
4942                 if ($defaults->{tablet}) {
4943                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4944                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4945                         if $arch eq 'aarch64';
4946                 } else {
4947                     vm_deviceunplug($vmid, $conf, 'tablet');
4948                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4949                 }
4950             } elsif ($opt =~ m/^usb(\d+)$/) {
4951                 my $index = $1;
4952                 die "skip\n" if !$usb_hotplug;
4953                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif ($opt eq 'vcpus') {
4956                 die "skip\n" if !$hotplug_features->{cpu};
4957                 qemu_cpu_hotplug($vmid, $conf, undef);
4958             } elsif ($opt eq 'balloon') {
4959                 # enable balloon device is not hotpluggable
4960                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4961                 # here we reset the ballooning value to memory
4962                 my $balloon = get_current_memory($conf->{memory});
4963                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4964             } elsif ($fast_plug_option->{$opt}) {
4965                 # do nothing
4966             } elsif ($opt =~ m/^net(\d+)$/) {
4967                 die "skip\n" if !$hotplug_features->{network};
4968                 vm_deviceunplug($vmid, $conf, $opt);
4969                 if($have_sdn) {
4970                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4971                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4972                 }
4973             } elsif (is_valid_drivename($opt)) {
4974                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4975                 vm_deviceunplug($vmid, $conf, $opt);
4976                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4977             } elsif ($opt =~ m/^memory$/) {
4978                 die "skip\n" if !$hotplug_features->{memory};
4979                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4980             } elsif ($opt eq 'cpuunits') {
4981                 $cgroup->change_cpu_shares(undef);
4982             } elsif ($opt eq 'cpulimit') {
4983                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4984             } else {
4985                 die "skip\n";
4986             }
4987         };
4988         if (my $err = $@) {
4989             &$add_error($opt, $err) if $err ne "skip\n";
4990         } else {
4991             my $old = delete $conf->{$opt};
4992             $cloudinit_record_changed->($conf, $opt, $old, undef);
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994         }
4995     }
4996
4997     my $cloudinit_opt;
4998     foreach my $opt (keys %{$conf->{pending}}) {
4999         next if $selection && !$selection->{$opt};
5000         my $value = $conf->{pending}->{$opt};
5001         eval {
5002             if ($opt eq 'hotplug') {
5003                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5004                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5005             } elsif ($opt eq 'tablet') {
5006                 die "skip\n" if !$hotplug_features->{usb};
5007                 if ($value == 1) {
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5010                         if $arch eq 'aarch64';
5011                 } elsif ($value == 0) {
5012                     vm_deviceunplug($vmid, $conf, 'tablet');
5013                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5014                 }
5015             } elsif ($opt =~ m/^usb(\d+)$/) {
5016                 my $index = $1;
5017                 die "skip\n" if !$usb_hotplug;
5018                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5019                 my $id = $opt;
5020                 if ($d->{host} =~ m/^spice$/i)  {
5021                     $id = "usbredirdev$index";
5022                 }
5023                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, $value);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable/disable balloning device is not hotpluggable
5029                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5030                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5031                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5032
5033                 # allow manual ballooning if shares is set to zero
5034                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5035                     my $memory = get_current_memory($conf->{memory});
5036                     my $balloon = $conf->{pending}->{balloon} || $memory;
5037                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5045                 # some changes can be done without hotplug
5046                 my $drive = parse_drive($opt, $value);
5047                 if (drive_is_cloudinit($drive)) {
5048                     $cloudinit_opt = [$opt, $drive];
5049                     # apply all the other changes first, then generate the cloudinit disk
5050                     die "skip\n";
5051                 }
5052                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5053                                      $vmid, $opt, $value, $arch, $machine_type);
5054             } elsif ($opt =~ m/^memory$/) { #dimms
5055                 die "skip\n" if !$hotplug_features->{memory};
5056                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5057             } elsif ($opt eq 'cpuunits') {
5058                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5059                 $cgroup->change_cpu_shares($new_cpuunits);
5060             } elsif ($opt eq 'cpulimit') {
5061                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5062                 $cgroup->change_cpu_quota($cpulimit, 100000);
5063             } elsif ($opt eq 'agent') {
5064                 vmconfig_update_agent($conf, $opt, $value);
5065             } else {
5066                 die "skip\n";  # skip non-hot-pluggable options
5067             }
5068         };
5069         if (my $err = $@) {
5070             &$add_error($opt, $err) if $err ne "skip\n";
5071         } else {
5072             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5073             $conf->{$opt} = $value;
5074             delete $conf->{pending}->{$opt};
5075         }
5076     }
5077
5078     if (defined($cloudinit_opt)) {
5079         my ($opt, $drive) = @$cloudinit_opt;
5080         my $value = $conf->{pending}->{$opt};
5081         eval {
5082             my $temp = {%$conf, $opt => $value};
5083             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5084             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5085                                  $vmid, $opt, $value, $arch, $machine_type);
5086         };
5087         if (my $err = $@) {
5088             &$add_error($opt, $err) if $err ne "skip\n";
5089         } else {
5090             $conf->{$opt} = $value;
5091             delete $conf->{pending}->{$opt};
5092         }
5093     }
5094
5095     # unplug xhci controller if no usb device is left
5096     if ($usb_hotplug) {
5097         my $has_usb = 0;
5098         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5099             next if !defined($conf->{"usb$i"});
5100             $has_usb = 1;
5101             last;
5102         }
5103         if (!$has_usb) {
5104             vm_deviceunplug($vmid, $conf, 'xhci');
5105         }
5106     }
5107
5108     PVE::QemuConfig->write_config($vmid, $conf);
5109
5110     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5111         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5112     }
5113 }
5114
5115 sub try_deallocate_drive {
5116     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5117
5118     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5119         my $volid = $drive->{file};
5120         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5121             my $sid = PVE::Storage::parse_volume_id($volid);
5122             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5123
5124             # check if the disk is really unused
5125             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5126                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5127             PVE::Storage::vdisk_free($storecfg, $volid);
5128             return 1;
5129         } else {
5130             # If vm is not owner of this disk remove from config
5131             return 1;
5132         }
5133     }
5134
5135     return;
5136 }
5137
5138 sub vmconfig_delete_or_detach_drive {
5139     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5140
5141     my $drive = parse_drive($opt, $conf->{$opt});
5142
5143     my $rpcenv = PVE::RPCEnvironment::get();
5144     my $authuser = $rpcenv->get_user();
5145
5146     if ($force) {
5147         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5148         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5149     } else {
5150         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5151     }
5152 }
5153
5154
5155
5156 sub vmconfig_apply_pending {
5157     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5158
5159     return if !scalar(keys %{$conf->{pending}});
5160
5161     my $add_apply_error = sub {
5162         my ($opt, $msg) = @_;
5163         my $err_msg = "unable to apply pending change $opt : $msg";
5164         $errors->{$opt} = $err_msg;
5165         warn $err_msg;
5166     };
5167
5168     # cold plug
5169
5170     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5171     foreach my $opt (sort keys %$pending_delete_hash) {
5172         my $force = $pending_delete_hash->{$opt}->{force};
5173         eval {
5174             if ($opt =~ m/^unused/) {
5175                 die "internal error";
5176             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5177                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5178             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5179                 if($have_sdn) {
5180                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5181                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5182                     warn if $@;
5183                 }
5184             }
5185         };
5186         if (my $err = $@) {
5187             $add_apply_error->($opt, $err);
5188         } else {
5189             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5190             delete $conf->{$opt};
5191         }
5192     }
5193
5194     PVE::QemuConfig->cleanup_pending($conf);
5195
5196     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5197
5198     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5199         next if $opt eq 'delete'; # just to be sure
5200         eval {
5201             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5202                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5203             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5204                 return if !$have_sdn; # return from eval if SDN is not available
5205
5206                 my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5207                 if ($conf->{$opt}) {
5208                     my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5209
5210                     if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (
5211                         safe_string_ne($old_net->{bridge}, $new_net->{bridge}) ||
5212                         safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})
5213                     )) {
5214                         PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5215                     }
5216                 }
5217                 #fixme: reuse ip if mac change && same bridge
5218                 PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5219             }
5220         };
5221         if (my $err = $@) {
5222             $add_apply_error->($opt, $err);
5223         } else {
5224
5225             if (is_valid_drivename($opt)) {
5226                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5227                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5228             }
5229
5230             $conf->{$opt} = delete $conf->{pending}->{$opt};
5231         }
5232     }
5233
5234     # write all changes at once to avoid unnecessary i/o
5235     PVE::QemuConfig->write_config($vmid, $conf);
5236     if ($generate_cloudinit) {
5237         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5238             # After successful generation and if there were changes to be applied, update the
5239             # config to drop the {cloudinit} entry.
5240             PVE::QemuConfig->write_config($vmid, $conf);
5241         }
5242     }
5243 }
5244
5245 sub vmconfig_update_net {
5246     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5247
5248     my $newnet = parse_net($value);
5249
5250     if ($conf->{$opt}) {
5251         my $oldnet = parse_net($conf->{$opt});
5252
5253         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5254             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5255             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5256             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5257             !($newnet->{bridge} && $oldnet->{bridge})
5258         ) { # bridge/nat mode change
5259
5260             # for non online change, we try to hot-unplug
5261             die "skip\n" if !$hotplug;
5262             vm_deviceunplug($vmid, $conf, $opt);
5263
5264             if ($have_sdn) {
5265                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5266             }
5267
5268         } else {
5269
5270             die "internal error" if $opt !~ m/net(\d+)/;
5271             my $iface = "tap${vmid}i$1";
5272
5273             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5274                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5275                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5276                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})
5277             ) {
5278                 PVE::Network::tap_unplug($iface);
5279
5280                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5281                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5282                     safe_num_ne($oldnet->{tag}, $newnet->{tag})
5283                 ) {
5284                     qemu_set_link_status($vmid, $opt, 0);
5285                 }
5286
5287                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5288                     if ($have_sdn) {
5289                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5290                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5291                     }
5292                 }
5293
5294                 if ($have_sdn) {
5295                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5296                 } else {
5297                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5298                 }
5299
5300                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5301                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5302                     safe_num_ne($oldnet->{tag}, $newnet->{tag})
5303                 ) {
5304                     qemu_set_link_status($vmid, $opt, 1);
5305                 }
5306
5307             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5308                 # Rate can be applied on its own but any change above needs to
5309                 # include the rate in tap_plug since OVS resets everything.
5310                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5311             }
5312
5313             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5314                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5315             }
5316
5317             return 1;
5318         }
5319     }
5320
5321     if ($hotplug) {
5322         if ($have_sdn) {
5323             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5324             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5325         }
5326         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5327     } else {
5328         die "skip\n";
5329     }
5330 }
5331
5332 sub vmconfig_update_agent {
5333     my ($conf, $opt, $value) = @_;
5334
5335     die "skip\n" if !$conf->{$opt};
5336
5337     my $hotplug_options = { fstrim_cloned_disks => 1 };
5338
5339     my $old_agent = parse_guest_agent($conf);
5340     my $agent = parse_guest_agent({$opt => $value});
5341
5342     for my $option (keys %$agent) { # added/changed options
5343         next if defined($hotplug_options->{$option});
5344         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5345     }
5346
5347     for my $option (keys %$old_agent) { # removed options
5348         next if defined($hotplug_options->{$option});
5349         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5350     }
5351
5352     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5353 }
5354
5355 sub vmconfig_update_disk {
5356     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5357
5358     my $drive = parse_drive($opt, $value);
5359
5360     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5361         my $media = $drive->{media} || 'disk';
5362         my $oldmedia = $old_drive->{media} || 'disk';
5363         die "unable to change media type\n" if $media ne $oldmedia;
5364
5365         if (!drive_is_cdrom($old_drive)) {
5366
5367             if ($drive->{file} ne $old_drive->{file}) {
5368
5369                 die "skip\n" if !$hotplug;
5370
5371                 # unplug and register as unused
5372                 vm_deviceunplug($vmid, $conf, $opt);
5373                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5374
5375             } else {
5376                 # update existing disk
5377
5378                 # skip non hotpluggable value
5379                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5380                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5381                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5382                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5383                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5384                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5385                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5386                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5387                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5388                     die "skip\n";
5389                 }
5390
5391                 # apply throttle
5392                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5393                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5394                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5395                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5396                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5397                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5398                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5399                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5400                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5401                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5402                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5403                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5404                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5405                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5406                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5407                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5408                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5409                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5410
5411                     qemu_block_set_io_throttle(
5412                         $vmid,"drive-$opt",
5413                         ($drive->{mbps} || 0)*1024*1024,
5414                         ($drive->{mbps_rd} || 0)*1024*1024,
5415                         ($drive->{mbps_wr} || 0)*1024*1024,
5416                          $drive->{iops} || 0,
5417                          $drive->{iops_rd} || 0,
5418                          $drive->{iops_wr} || 0,
5419                         ($drive->{mbps_max} || 0)*1024*1024,
5420                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5421                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5422                          $drive->{iops_max} || 0,
5423                          $drive->{iops_rd_max} || 0,
5424                          $drive->{iops_wr_max} || 0,
5425                          $drive->{bps_max_length} || 1,
5426                          $drive->{bps_rd_max_length} || 1,
5427                          $drive->{bps_wr_max_length} || 1,
5428                          $drive->{iops_max_length} || 1,
5429                          $drive->{iops_rd_max_length} || 1,
5430                          $drive->{iops_wr_max_length} || 1,
5431                     );
5432
5433                 }
5434
5435                 return 1;
5436             }
5437
5438         } else { # cdrom
5439
5440             if ($drive->{file} eq 'none') {
5441                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5442                 if (drive_is_cloudinit($old_drive)) {
5443                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5444                 }
5445             } else {
5446                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5447
5448                 # force eject if locked
5449                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5450
5451                 if ($path) {
5452                     mon_cmd($vmid, "blockdev-change-medium",
5453                         id => "$opt", filename => "$path");
5454                 }
5455             }
5456
5457             return 1;
5458         }
5459     }
5460
5461     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5462     # hotplug new disks
5463     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5464     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5465 }
5466
5467 sub vmconfig_update_cloudinit_drive {
5468     my ($storecfg, $conf, $vmid) = @_;
5469
5470     my $cloudinit_ds = undef;
5471     my $cloudinit_drive = undef;
5472
5473     PVE::QemuConfig->foreach_volume($conf, sub {
5474         my ($ds, $drive) = @_;
5475         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5476             $cloudinit_ds = $ds;
5477             $cloudinit_drive = $drive;
5478         }
5479     });
5480
5481     return if !$cloudinit_drive;
5482
5483     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5484         PVE::QemuConfig->write_config($vmid, $conf);
5485     }
5486
5487     my $running = PVE::QemuServer::check_running($vmid);
5488
5489     if ($running) {
5490         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5491         if ($path) {
5492             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5493             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5494         }
5495     }
5496 }
5497
5498 # called in locked context by incoming migration
5499 sub vm_migrate_get_nbd_disks {
5500     my ($storecfg, $conf, $replicated_volumes) = @_;
5501
5502     my $local_volumes = {};
5503     PVE::QemuConfig->foreach_volume($conf, sub {
5504         my ($ds, $drive) = @_;
5505
5506         return if drive_is_cdrom($drive);
5507         return if $ds eq 'tpmstate0';
5508
5509         my $volid = $drive->{file};
5510
5511         return if !$volid;
5512
5513         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5514
5515         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5516         return if $scfg->{shared};
5517
5518         my $format = qemu_img_format($scfg, $volname);
5519
5520         # replicated disks re-use existing state via bitmap
5521         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5522         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5523     });
5524     return $local_volumes;
5525 }
5526
5527 # called in locked context by incoming migration
5528 sub vm_migrate_alloc_nbd_disks {
5529     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5530
5531     my $nbd = {};
5532     foreach my $opt (sort keys %$source_volumes) {
5533         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5534
5535         if ($use_existing) {
5536             $nbd->{$opt}->{drivestr} = print_drive($drive);
5537             $nbd->{$opt}->{volid} = $volid;
5538             $nbd->{$opt}->{replicated} = 1;
5539             next;
5540         }
5541
5542         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5543
5544         # order of precedence, filtered by whether storage supports it:
5545         # 1. explicit requested format
5546         # 2. default format of storage
5547         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5548         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5549
5550         my $size = $drive->{size} / 1024;
5551         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5552         my $newdrive = $drive;
5553         $newdrive->{format} = $format;
5554         $newdrive->{file} = $newvolid;
5555         my $drivestr = print_drive($newdrive);
5556         $nbd->{$opt}->{drivestr} = $drivestr;
5557         $nbd->{$opt}->{volid} = $newvolid;
5558     }
5559
5560     return $nbd;
5561 }
5562
5563 # see vm_start_nolock for parameters, additionally:
5564 # migrate_opts:
5565 #   storagemap = parsed storage map for allocating NBD disks
5566 sub vm_start {
5567     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5568
5569     return PVE::QemuConfig->lock_config($vmid, sub {
5570         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5571
5572         die "you can't start a vm if it's a template\n"
5573             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5574
5575         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5576         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5577
5578         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5579
5580         if ($has_backup_lock && $running) {
5581             # a backup is currently running, attempt to start the guest in the
5582             # existing QEMU instance
5583             return vm_resume($vmid);
5584         }
5585
5586         PVE::QemuConfig->check_lock($conf)
5587             if !($params->{skiplock} || $has_suspended_lock);
5588
5589         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5590
5591         die "VM $vmid already running\n" if $running;
5592
5593         if (my $storagemap = $migrate_opts->{storagemap}) {
5594             my $replicated = $migrate_opts->{replicated_volumes};
5595             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5596             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5597
5598             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5599                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5600             }
5601         }
5602
5603         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5604     });
5605 }
5606
5607
5608 # params:
5609 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5610 #   skiplock => 0/1, skip checking for config lock
5611 #   skiptemplate => 0/1, skip checking whether VM is template
5612 #   forcemachine => to force QEMU machine (rollback/migration)
5613 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5614 #   timeout => in seconds
5615 #   paused => start VM in paused state (backup)
5616 #   resume => resume from hibernation
5617 #   live-restore-backing => {
5618 #      sata0 => {
5619 #          name => blockdev-name,
5620 #          blockdev => "arg to the -blockdev command instantiating device named 'name'",
5621 #      },
5622 #      virtio2 => ...
5623 #   }
5624 # migrate_opts:
5625 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5626 #   migratedfrom => source node
5627 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5628 #   network => CIDR of migration network
5629 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5630 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5631 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5632 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5633 #       contained in config
5634 sub vm_start_nolock {
5635     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5636
5637     my $statefile = $params->{statefile};
5638     my $resume = $params->{resume};
5639
5640     my $migratedfrom = $migrate_opts->{migratedfrom};
5641     my $migration_type = $migrate_opts->{type};
5642
5643     my $res = {};
5644
5645     # clean up leftover reboot request files
5646     eval { clear_reboot_request($vmid); };
5647     warn $@ if $@;
5648
5649     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5650         vmconfig_apply_pending($vmid, $conf, $storecfg);
5651         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5652     }
5653
5654     # don't regenerate the ISO if the VM is started as part of a live migration
5655     # this way we can reuse the old ISO with the correct config
5656     if (!$migratedfrom) {
5657         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5658             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5659             # $conf->{cloudinit}, so we could just not do this?
5660             # But we do it above, so for now let's be consistent.
5661             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5662         }
5663     }
5664
5665     # override offline migrated volumes, conf is out of date still
5666     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5667         for my $key (sort keys $offline_volumes->%*) {
5668             my $parsed = parse_drive($key, $conf->{$key});
5669             $parsed->{file} = $offline_volumes->{$key};
5670             $conf->{$key} = print_drive($parsed);
5671         }
5672     }
5673
5674     my $defaults = load_defaults();
5675
5676     # set environment variable useful inside network script
5677     # for remote migration the config is available on the target node!
5678     if (!$migrate_opts->{remote_node}) {
5679         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5680     }
5681
5682     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5683
5684     my $forcemachine = $params->{forcemachine};
5685     my $forcecpu = $params->{forcecpu};
5686     if ($resume) {
5687         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5688         $forcemachine = $conf->{runningmachine};
5689         $forcecpu = $conf->{runningcpu};
5690         print "Resuming suspended VM\n";
5691     }
5692
5693     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5694         $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
5695
5696     my $migration_ip;
5697     my $get_migration_ip = sub {
5698         my ($nodename) = @_;
5699
5700         return $migration_ip if defined($migration_ip);
5701
5702         my $cidr = $migrate_opts->{network};
5703
5704         if (!defined($cidr)) {
5705             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5706             $cidr = $dc_conf->{migration}->{network};
5707         }
5708
5709         if (defined($cidr)) {
5710             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5711
5712             die "could not get IP: no address configured on local " .
5713                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5714
5715             die "could not get IP: multiple addresses configured on local " .
5716                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5717
5718             $migration_ip = @$ips[0];
5719         }
5720
5721         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5722             if !defined($migration_ip);
5723
5724         return $migration_ip;
5725     };
5726
5727     if ($statefile) {
5728         if ($statefile eq 'tcp') {
5729             my $migrate = $res->{migrate} = { proto => 'tcp' };
5730             $migrate->{addr} = "localhost";
5731             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5732             my $nodename = nodename();
5733
5734             if (!defined($migration_type)) {
5735                 if (defined($datacenterconf->{migration}->{type})) {
5736                     $migration_type = $datacenterconf->{migration}->{type};
5737                 } else {
5738                     $migration_type = 'secure';
5739                 }
5740             }
5741
5742             if ($migration_type eq 'insecure') {
5743                 $migrate->{addr} = $get_migration_ip->($nodename);
5744                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5745             }
5746
5747             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5748             # via QMP later
5749             push @$cmd, '-incoming', 'defer';
5750             push @$cmd, '-S';
5751
5752         } elsif ($statefile eq 'unix') {
5753             # should be default for secure migrations as a ssh TCP forward
5754             # tunnel is not deterministic reliable ready and fails regurarly
5755             # to set up in time, so use UNIX socket forwards
5756             my $migrate = $res->{migrate} = { proto => 'unix' };
5757             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5758             unlink $migrate->{addr};
5759
5760             $migrate->{uri} = "unix:$migrate->{addr}";
5761             push @$cmd, '-incoming', $migrate->{uri};
5762             push @$cmd, '-S';
5763
5764         } elsif (-e $statefile) {
5765             push @$cmd, '-loadstate', $statefile;
5766         } else {
5767             my $statepath = PVE::Storage::path($storecfg, $statefile);
5768             push @$vollist, $statefile;
5769             push @$cmd, '-loadstate', $statepath;
5770         }
5771     } elsif ($params->{paused}) {
5772         push @$cmd, '-S';
5773     }
5774
5775     my $memory = get_current_memory($conf->{memory});
5776     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5777
5778     my $pci_reserve_list = [];
5779     for my $device (values $pci_devices->%*) {
5780         next if $device->{mdev}; # we don't reserve for mdev devices
5781         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5782     }
5783
5784     # reserve all PCI IDs before actually doing anything with them
5785     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5786
5787     eval {
5788         my $uuid;
5789         for my $id (sort keys %$pci_devices) {
5790             my $d = $pci_devices->{$id};
5791             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5792
5793             my $chosen_mdev;
5794             for my $dev ($d->{ids}->@*) {
5795                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5796                 if ($d->{mdev}) {
5797                     warn $@ if $@;
5798                     $chosen_mdev = $info;
5799                     last if $chosen_mdev; # if successful, we're done
5800                 } else {
5801                     die $@ if $@;
5802                 }
5803             }
5804
5805             next if !$d->{mdev};
5806             die "could not create mediated device\n" if !defined($chosen_mdev);
5807
5808             # nvidia grid needs the uuid of the mdev as qemu parameter
5809             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5810                 if (defined($conf->{smbios1})) {
5811                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5812                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5813                 }
5814                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5815             }
5816         }
5817         push @$cmd, '-uuid', $uuid if defined($uuid);
5818     };
5819     if (my $err = $@) {
5820         eval { cleanup_pci_devices($vmid, $conf) };
5821         warn $@ if $@;
5822         die $err;
5823     }
5824
5825     PVE::Storage::activate_volumes($storecfg, $vollist);
5826
5827
5828     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5829     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5830     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5831     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5832     # timeout should be more than enough here...
5833     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5834
5835     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5836
5837     my %run_params = (
5838         timeout => $statefile ? undef : $start_timeout,
5839         umask => 0077,
5840         noerr => 1,
5841     );
5842
5843     # when migrating, prefix QEMU output so other side can pick up any
5844     # errors that might occur and show the user
5845     if ($migratedfrom) {
5846         $run_params{quiet} = 1;
5847         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5848     }
5849
5850     my %systemd_properties = (
5851         Slice => 'qemu.slice',
5852         KillMode => 'process',
5853         SendSIGKILL => 0,
5854         TimeoutStopUSec => ULONG_MAX, # infinity
5855     );
5856
5857     if (PVE::CGroup::cgroup_mode() == 2) {
5858         $systemd_properties{CPUWeight} = $cpuunits;
5859     } else {
5860         $systemd_properties{CPUShares} = $cpuunits;
5861     }
5862
5863     if (my $cpulimit = $conf->{cpulimit}) {
5864         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5865     }
5866     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5867
5868     my $run_qemu = sub {
5869         PVE::Tools::run_fork sub {
5870             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5871
5872             my $tpmpid;
5873             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5874                 # start the TPM emulator so QEMU can connect on start
5875                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5876             }
5877
5878             my $exitcode = run_command($cmd, %run_params);
5879             if ($exitcode) {
5880                 if ($tpmpid) {
5881                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5882                     kill 'TERM', $tpmpid;
5883                 }
5884                 die "QEMU exited with code $exitcode\n";
5885             }
5886         };
5887     };
5888
5889     if ($conf->{hugepages}) {
5890
5891         my $code = sub {
5892             my $hotplug_features =
5893                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5894             my $hugepages_topology =
5895                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5896
5897             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5898
5899             PVE::QemuServer::Memory::hugepages_mount();
5900             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5901
5902             eval { $run_qemu->() };
5903             if (my $err = $@) {
5904                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5905                     if !$conf->{keephugepages};
5906                 die $err;
5907             }
5908
5909             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5910                 if !$conf->{keephugepages};
5911         };
5912         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5913
5914     } else {
5915         eval { $run_qemu->() };
5916     }
5917
5918     if (my $err = $@) {
5919         # deactivate volumes if start fails
5920         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5921         warn $@ if $@;
5922         eval { cleanup_pci_devices($vmid, $conf) };
5923         warn $@ if $@;
5924
5925         die "start failed: $err";
5926     }
5927
5928     # re-reserve all PCI IDs now that we can know the actual VM PID
5929     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5930     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5931     warn $@ if $@;
5932
5933     if (defined(my $migrate = $res->{migrate})) {
5934         if ($migrate->{proto} eq 'tcp') {
5935             my $nodename = nodename();
5936             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5937             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5938             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5939             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5940         }
5941         print "migration listens on $migrate->{uri}\n";
5942     } elsif ($statefile) {
5943         eval { mon_cmd($vmid, "cont"); };
5944         warn $@ if $@;
5945     }
5946
5947     #start nbd server for storage migration
5948     if (my $nbd = $migrate_opts->{nbd}) {
5949         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5950
5951         my $migrate_storage_uri;
5952         # nbd_protocol_version > 0 for unix socket support
5953         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5954             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5955             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5956             $migrate_storage_uri = "nbd:unix:$socket_path";
5957             $res->{migrate}->{unix_sockets} = [$socket_path];
5958         } else {
5959             my $nodename = nodename();
5960             my $localip = $get_migration_ip->($nodename);
5961             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5962             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5963
5964             mon_cmd($vmid, "nbd-server-start", addr => {
5965                 type => 'inet',
5966                 data => {
5967                     host => "${localip}",
5968                     port => "${storage_migrate_port}",
5969                 },
5970             });
5971             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5972             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5973         }
5974
5975         my $block_info = mon_cmd($vmid, "query-block");
5976         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5977
5978         foreach my $opt (sort keys %$nbd) {
5979             my $drivestr = $nbd->{$opt}->{drivestr};
5980             my $volid = $nbd->{$opt}->{volid};
5981
5982             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5983
5984             mon_cmd(
5985                 $vmid,
5986                 "block-export-add",
5987                 id => "drive-$opt",
5988                 'node-name' => $block_node,
5989                 writable => JSON::true,
5990                 type => "nbd",
5991                 name => "drive-$opt", # NBD export name
5992             );
5993
5994             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5995             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5996             print "re-using replicated volume: $opt - $volid\n"
5997                 if $nbd->{$opt}->{replicated};
5998
5999             $res->{drives}->{$opt} = $nbd->{$opt};
6000             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6001         }
6002     }
6003
6004     if ($migratedfrom) {
6005         eval {
6006             set_migration_caps($vmid);
6007         };
6008         warn $@ if $@;
6009
6010         if ($spice_port) {
6011             print "spice listens on port $spice_port\n";
6012             $res->{spice_port} = $spice_port;
6013             if ($migrate_opts->{spice_ticket}) {
6014                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6015                     $migrate_opts->{spice_ticket});
6016                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6017             }
6018         }
6019
6020     } else {
6021         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6022             if !$statefile && $conf->{balloon};
6023
6024         foreach my $opt (keys %$conf) {
6025             next if $opt !~  m/^net\d+$/;
6026             my $nicconf = parse_net($conf->{$opt});
6027             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6028         }
6029         add_nets_bridge_fdb($conf, $vmid);
6030     }
6031
6032    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6033         eval {
6034             mon_cmd(
6035                 $vmid,
6036                 'qom-set',
6037                 path => "machine/peripheral/balloon0",
6038                 property => "guest-stats-polling-interval",
6039                 value => 2
6040             );
6041         };
6042         log_warn("could not set polling interval for ballooning - $@") if $@;
6043     }
6044
6045     if ($resume) {
6046         print "Resumed VM, removing state\n";
6047         if (my $vmstate = $conf->{vmstate}) {
6048             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6049             PVE::Storage::vdisk_free($storecfg, $vmstate);
6050         }
6051         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6052         PVE::QemuConfig->write_config($vmid, $conf);
6053     }
6054
6055     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6056
6057     my ($current_machine, $is_deprecated) =
6058         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6059     if ($is_deprecated) {
6060         log_warn(
6061             "current machine version '$current_machine' is deprecated - see the documentation and ".
6062             "change to a newer one",
6063         );
6064     }
6065
6066     return $res;
6067 }
6068
6069 sub vm_commandline {
6070     my ($storecfg, $vmid, $snapname) = @_;
6071
6072     my $conf = PVE::QemuConfig->load_config($vmid);
6073
6074     my ($forcemachine, $forcecpu);
6075     if ($snapname) {
6076         my $snapshot = $conf->{snapshots}->{$snapname};
6077         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6078
6079         # check for machine or CPU overrides in snapshot
6080         $forcemachine = $snapshot->{runningmachine};
6081         $forcecpu = $snapshot->{runningcpu};
6082
6083         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6084
6085         $conf = $snapshot;
6086     }
6087
6088     my $defaults = load_defaults();
6089
6090     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6091
6092     return PVE::Tools::cmd2string($cmd);
6093 }
6094
6095 sub vm_reset {
6096     my ($vmid, $skiplock) = @_;
6097
6098     PVE::QemuConfig->lock_config($vmid, sub {
6099
6100         my $conf = PVE::QemuConfig->load_config($vmid);
6101
6102         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6103
6104         mon_cmd($vmid, "system_reset");
6105     });
6106 }
6107
6108 sub get_vm_volumes {
6109     my ($conf) = @_;
6110
6111     my $vollist = [];
6112     foreach_volid($conf, sub {
6113         my ($volid, $attr) = @_;
6114
6115         return if $volid =~ m|^/|;
6116
6117         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6118         return if !$sid;
6119
6120         push @$vollist, $volid;
6121     });
6122
6123     return $vollist;
6124 }
6125
6126 sub cleanup_pci_devices {
6127     my ($vmid, $conf) = @_;
6128
6129     foreach my $key (keys %$conf) {
6130         next if $key !~ m/^hostpci(\d+)$/;
6131         my $hostpciindex = $1;
6132         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6133         my $d = parse_hostpci($conf->{$key});
6134         if ($d->{mdev}) {
6135             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6136             # don't want to break ABI just for this two liner
6137             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6138
6139             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6140             # out when we do it first. so wait for up to 10 seconds and then try it manually
6141             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6142                 my $count = 0;
6143                 while (-e $dev_sysfs_dir && $count < 10) {
6144                     sleep 1;
6145                     $count++;
6146                 }
6147                 print "waited $count seconds for mediated device driver finishing clean up\n";
6148             }
6149
6150             if (-e $dev_sysfs_dir) {
6151                 print "actively clean up mediated device with UUID $uuid\n";
6152                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6153             }
6154         }
6155     }
6156     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6157 }
6158
6159 sub vm_stop_cleanup {
6160     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6161
6162     eval {
6163
6164         if (!$keepActive) {
6165             my $vollist = get_vm_volumes($conf);
6166             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6167
6168             if (my $tpmdrive = $conf->{tpmstate0}) {
6169                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6170                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6171                 if ($storeid) {
6172                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6173                 }
6174             }
6175         }
6176
6177         foreach my $ext (qw(mon qmp pid vnc qga)) {
6178             unlink "/var/run/qemu-server/${vmid}.$ext";
6179         }
6180
6181         if ($conf->{ivshmem}) {
6182             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6183             # just delete it for now, VMs which have this already open do not
6184             # are affected, but new VMs will get a separated one. If this
6185             # becomes an issue we either add some sort of ref-counting or just
6186             # add a "don't delete on stop" flag to the ivshmem format.
6187             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6188         }
6189
6190         cleanup_pci_devices($vmid, $conf);
6191
6192         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6193     };
6194     warn $@ if $@; # avoid errors - just warn
6195 }
6196
6197 # call only in locked context
6198 sub _do_vm_stop {
6199     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6200
6201     my $pid = check_running($vmid, $nocheck);
6202     return if !$pid;
6203
6204     my $conf;
6205     if (!$nocheck) {
6206         $conf = PVE::QemuConfig->load_config($vmid);
6207         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6208         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6209             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6210             $timeout = $opts->{down} if $opts->{down};
6211         }
6212         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6213     }
6214
6215     eval {
6216         if ($shutdown) {
6217             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6218                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6219             } else {
6220                 mon_cmd($vmid, "system_powerdown");
6221             }
6222         } else {
6223             mon_cmd($vmid, "quit");
6224         }
6225     };
6226     my $err = $@;
6227
6228     if (!$err) {
6229         $timeout = 60 if !defined($timeout);
6230
6231         my $count = 0;
6232         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6233             $count++;
6234             sleep 1;
6235         }
6236
6237         if ($count >= $timeout) {
6238             if ($force) {
6239                 warn "VM still running - terminating now with SIGTERM\n";
6240                 kill 15, $pid;
6241             } else {
6242                 die "VM quit/powerdown failed - got timeout\n";
6243             }
6244         } else {
6245             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6246             return;
6247         }
6248     } else {
6249        if (!check_running($vmid, $nocheck)) {
6250             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6251             return;
6252        }
6253        if ($force) {
6254             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6255             kill 15, $pid;
6256         } else {
6257             die "VM quit/powerdown failed\n";
6258         }
6259     }
6260
6261     # wait again
6262     $timeout = 10;
6263
6264     my $count = 0;
6265     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6266         $count++;
6267         sleep 1;
6268     }
6269
6270     if ($count >= $timeout) {
6271         warn "VM still running - terminating now with SIGKILL\n";
6272         kill 9, $pid;
6273         sleep 1;
6274     }
6275
6276     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6277 }
6278
6279 # Note: use $nocheck to skip tests if VM configuration file exists.
6280 # We need that when migration VMs to other nodes (files already moved)
6281 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6282 sub vm_stop {
6283     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6284
6285     $force = 1 if !defined($force) && !$shutdown;
6286
6287     if ($migratedfrom){
6288         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6289         kill 15, $pid if $pid;
6290         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6291         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6292         return;
6293     }
6294
6295     PVE::QemuConfig->lock_config($vmid, sub {
6296         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6297    });
6298 }
6299
6300 sub vm_reboot {
6301     my ($vmid, $timeout) = @_;
6302
6303     PVE::QemuConfig->lock_config($vmid, sub {
6304         eval {
6305
6306             # only reboot if running, as qmeventd starts it again on a stop event
6307             return if !check_running($vmid);
6308
6309             create_reboot_request($vmid);
6310
6311             my $storecfg = PVE::Storage::config();
6312             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6313
6314         };
6315         if (my $err = $@) {
6316             # avoid that the next normal shutdown will be confused for a reboot
6317             clear_reboot_request($vmid);
6318             die $err;
6319         }
6320    });
6321 }
6322
6323 # note: if using the statestorage parameter, the caller has to check privileges
6324 sub vm_suspend {
6325     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6326
6327     my $conf;
6328     my $path;
6329     my $storecfg;
6330     my $vmstate;
6331
6332     PVE::QemuConfig->lock_config($vmid, sub {
6333
6334         $conf = PVE::QemuConfig->load_config($vmid);
6335
6336         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6337         PVE::QemuConfig->check_lock($conf)
6338             if !($skiplock || $is_backing_up);
6339
6340         die "cannot suspend to disk during backup\n"
6341             if $is_backing_up && $includestate;
6342
6343         if ($includestate) {
6344             $conf->{lock} = 'suspending';
6345             my $date = strftime("%Y-%m-%d", localtime(time()));
6346             $storecfg = PVE::Storage::config();
6347             if (!$statestorage) {
6348                 $statestorage = find_vmstate_storage($conf, $storecfg);
6349                 # check permissions for the storage
6350                 my $rpcenv = PVE::RPCEnvironment::get();
6351                 if ($rpcenv->{type} ne 'cli') {
6352                     my $authuser = $rpcenv->get_user();
6353                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6354                 }
6355             }
6356
6357
6358             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6359                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6360             $path = PVE::Storage::path($storecfg, $vmstate);
6361             PVE::QemuConfig->write_config($vmid, $conf);
6362         } else {
6363             mon_cmd($vmid, "stop");
6364         }
6365     });
6366
6367     if ($includestate) {
6368         # save vm state
6369         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6370
6371         eval {
6372             set_migration_caps($vmid, 1);
6373             mon_cmd($vmid, "savevm-start", statefile => $path);
6374             for(;;) {
6375                 my $state = mon_cmd($vmid, "query-savevm");
6376                 if (!$state->{status}) {
6377                     die "savevm not active\n";
6378                 } elsif ($state->{status} eq 'active') {
6379                     sleep(1);
6380                     next;
6381                 } elsif ($state->{status} eq 'completed') {
6382                     print "State saved, quitting\n";
6383                     last;
6384                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6385                     die "query-savevm failed with error '$state->{error}'\n"
6386                 } else {
6387                     die "query-savevm returned status '$state->{status}'\n";
6388                 }
6389             }
6390         };
6391         my $err = $@;
6392
6393         PVE::QemuConfig->lock_config($vmid, sub {
6394             $conf = PVE::QemuConfig->load_config($vmid);
6395             if ($err) {
6396                 # cleanup, but leave suspending lock, to indicate something went wrong
6397                 eval {
6398                     mon_cmd($vmid, "savevm-end");
6399                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6400                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6401                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6402                     PVE::QemuConfig->write_config($vmid, $conf);
6403                 };
6404                 warn $@ if $@;
6405                 die $err;
6406             }
6407
6408             die "lock changed unexpectedly\n"
6409                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6410
6411             mon_cmd($vmid, "quit");
6412             $conf->{lock} = 'suspended';
6413             PVE::QemuConfig->write_config($vmid, $conf);
6414         });
6415     }
6416 }
6417
6418 # $nocheck is set when called as part of a migration - in this context the
6419 # location of the config file (source or target node) is not deterministic,
6420 # since migration cannot wait for pmxcfs to process the rename
6421 sub vm_resume {
6422     my ($vmid, $skiplock, $nocheck) = @_;
6423
6424     PVE::QemuConfig->lock_config($vmid, sub {
6425         my $res = mon_cmd($vmid, 'query-status');
6426         my $resume_cmd = 'cont';
6427         my $reset = 0;
6428         my $conf;
6429         if ($nocheck) {
6430             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6431             if ($@) {
6432                 my $vmlist = PVE::Cluster::get_vmlist();
6433                 if (exists($vmlist->{ids}->{$vmid})) {
6434                     my $node = $vmlist->{ids}->{$vmid}->{node};
6435                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6436                 }
6437                 if (!$conf) {
6438                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6439                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6440                 }
6441             }
6442         } else {
6443             $conf = PVE::QemuConfig->load_config($vmid);
6444         }
6445
6446         if ($res->{status}) {
6447             return if $res->{status} eq 'running'; # job done, go home
6448             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6449             $reset = 1 if $res->{status} eq 'shutdown';
6450         }
6451
6452         if (!$nocheck) {
6453             PVE::QemuConfig->check_lock($conf)
6454                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6455         }
6456
6457         if ($reset) {
6458             # required if a VM shuts down during a backup and we get a resume
6459             # request before the backup finishes for example
6460             mon_cmd($vmid, "system_reset");
6461         }
6462
6463         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6464
6465         mon_cmd($vmid, $resume_cmd);
6466     });
6467 }
6468
6469 sub vm_sendkey {
6470     my ($vmid, $skiplock, $key) = @_;
6471
6472     PVE::QemuConfig->lock_config($vmid, sub {
6473
6474         my $conf = PVE::QemuConfig->load_config($vmid);
6475
6476         # there is no qmp command, so we use the human monitor command
6477         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6478         die $res if $res ne '';
6479     });
6480 }
6481
6482 sub check_bridge_access {
6483     my ($rpcenv, $authuser, $conf) = @_;
6484
6485     return 1 if $authuser eq 'root@pam';
6486
6487     for my $opt (sort keys $conf->%*) {
6488         next if $opt !~ m/^net\d+$/;
6489         my $net = parse_net($conf->{$opt});
6490         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6491         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6492     }
6493     return 1;
6494 };
6495
6496 sub check_mapping_access {
6497     my ($rpcenv, $user, $conf) = @_;
6498
6499     for my $opt (keys $conf->%*) {
6500         if ($opt =~ m/^usb\d+$/) {
6501             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6502             if (my $host = $device->{host}) {
6503                 die "only root can set '$opt' config for real devices\n"
6504                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6505             } elsif ($device->{mapping}) {
6506                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6507             } else {
6508                 die "either 'host' or 'mapping' must be set.\n";
6509             }
6510         } elsif ($opt =~ m/^hostpci\d+$/) {
6511             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6512             if ($device->{host}) {
6513                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6514             } elsif ($device->{mapping}) {
6515                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6516             } else {
6517                 die "either 'host' or 'mapping' must be set.\n";
6518             }
6519        }
6520    }
6521 };
6522
6523 sub check_restore_permissions {
6524     my ($rpcenv, $user, $conf) = @_;
6525
6526     check_bridge_access($rpcenv, $user, $conf);
6527     check_mapping_access($rpcenv, $user, $conf);
6528 }
6529 # vzdump restore implementaion
6530
6531 sub tar_archive_read_firstfile {
6532     my $archive = shift;
6533
6534     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6535
6536     # try to detect archive type first
6537     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6538         die "unable to open file '$archive'\n";
6539     my $firstfile = <$fh>;
6540     kill 15, $pid;
6541     close $fh;
6542
6543     die "ERROR: archive contaions no data\n" if !$firstfile;
6544     chomp $firstfile;
6545
6546     return $firstfile;
6547 }
6548
6549 sub tar_restore_cleanup {
6550     my ($storecfg, $statfile) = @_;
6551
6552     print STDERR "starting cleanup\n";
6553
6554     if (my $fd = IO::File->new($statfile, "r")) {
6555         while (defined(my $line = <$fd>)) {
6556             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6557                 my $volid = $2;
6558                 eval {
6559                     if ($volid =~ m|^/|) {
6560                         unlink $volid || die 'unlink failed\n';
6561                     } else {
6562                         PVE::Storage::vdisk_free($storecfg, $volid);
6563                     }
6564                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6565                 };
6566                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6567             } else {
6568                 print STDERR "unable to parse line in statfile - $line";
6569             }
6570         }
6571         $fd->close();
6572     }
6573 }
6574
6575 sub restore_file_archive {
6576     my ($archive, $vmid, $user, $opts) = @_;
6577
6578     return restore_vma_archive($archive, $vmid, $user, $opts)
6579         if $archive eq '-';
6580
6581     my $info = PVE::Storage::archive_info($archive);
6582     my $format = $opts->{format} // $info->{format};
6583     my $comp = $info->{compression};
6584
6585     # try to detect archive format
6586     if ($format eq 'tar') {
6587         return restore_tar_archive($archive, $vmid, $user, $opts);
6588     } else {
6589         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6590     }
6591 }
6592
6593 # hepler to remove disks that will not be used after restore
6594 my $restore_cleanup_oldconf = sub {
6595     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6596
6597     my $kept_disks = {};
6598
6599     PVE::QemuConfig->foreach_volume($oldconf, sub {
6600         my ($ds, $drive) = @_;
6601
6602         return if drive_is_cdrom($drive, 1);
6603
6604         my $volid = $drive->{file};
6605         return if !$volid || $volid =~ m|^/|;
6606
6607         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6608         return if !$path || !$owner || ($owner != $vmid);
6609
6610         # Note: only delete disk we want to restore
6611         # other volumes will become unused
6612         if ($virtdev_hash->{$ds}) {
6613             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6614             if (my $err = $@) {
6615                 warn $err;
6616             }
6617         } else {
6618             $kept_disks->{$volid} = 1;
6619         }
6620     });
6621
6622     # after the restore we have no snapshots anymore
6623     for my $snapname (keys $oldconf->{snapshots}->%*) {
6624         my $snap = $oldconf->{snapshots}->{$snapname};
6625         if ($snap->{vmstate}) {
6626             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6627             if (my $err = $@) {
6628                 warn $err;
6629             }
6630         }
6631
6632         for my $volid (keys $kept_disks->%*) {
6633             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6634             warn $@ if $@;
6635         }
6636     }
6637 };
6638
6639 # Helper to parse vzdump backup device hints
6640 #
6641 # $rpcenv: Environment, used to ckeck storage permissions
6642 # $user: User ID, to check storage permissions
6643 # $storecfg: Storage configuration
6644 # $fh: the file handle for reading the configuration
6645 # $devinfo: should contain device sizes for all backu-up'ed devices
6646 # $options: backup options (pool, default storage)
6647 #
6648 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6649 my $parse_backup_hints = sub {
6650     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6651
6652     my $check_storage = sub { # assert if an image can be allocate
6653         my ($storeid, $scfg) = @_;
6654         die "Content type 'images' is not available on storage '$storeid'\n"
6655             if !$scfg->{content}->{images};
6656         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6657             if $user ne 'root@pam';
6658     };
6659
6660     my $virtdev_hash = {};
6661     while (defined(my $line = <$fh>)) {
6662         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6663             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6664             die "archive does not contain data for drive '$virtdev'\n"
6665                 if !$devinfo->{$devname};
6666
6667             if (defined($options->{storage})) {
6668                 $storeid = $options->{storage} || 'local';
6669             } elsif (!$storeid) {
6670                 $storeid = 'local';
6671             }
6672             $format = 'raw' if !$format;
6673             $devinfo->{$devname}->{devname} = $devname;
6674             $devinfo->{$devname}->{virtdev} = $virtdev;
6675             $devinfo->{$devname}->{format} = $format;
6676             $devinfo->{$devname}->{storeid} = $storeid;
6677
6678             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6679             $check_storage->($storeid, $scfg); # permission and content type check
6680
6681             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6682         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6683             my $virtdev = $1;
6684             my $drive = parse_drive($virtdev, $2);
6685
6686             if (drive_is_cloudinit($drive)) {
6687                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6688                 $storeid = $options->{storage} if defined ($options->{storage});
6689                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6690                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6691
6692                 $check_storage->($storeid, $scfg); # permission and content type check
6693
6694                 $virtdev_hash->{$virtdev} = {
6695                     format => $format,
6696                     storeid => $storeid,
6697                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6698                     is_cloudinit => 1,
6699                 };
6700             }
6701         }
6702     }
6703
6704     return $virtdev_hash;
6705 };
6706
6707 # Helper to allocate and activate all volumes required for a restore
6708 #
6709 # $storecfg: Storage configuration
6710 # $virtdev_hash: as returned by parse_backup_hints()
6711 #
6712 # Returns: { $virtdev => $volid }
6713 my $restore_allocate_devices = sub {
6714     my ($storecfg, $virtdev_hash, $vmid) = @_;
6715
6716     my $map = {};
6717     foreach my $virtdev (sort keys %$virtdev_hash) {
6718         my $d = $virtdev_hash->{$virtdev};
6719         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6720         my $storeid = $d->{storeid};
6721         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6722
6723         # test if requested format is supported
6724         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6725         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6726         $d->{format} = $defFormat if !$supported;
6727
6728         my $name;
6729         if ($d->{is_cloudinit}) {
6730             $name = "vm-$vmid-cloudinit";
6731             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6732             if ($scfg->{path}) {
6733                 $name .= ".$d->{format}";
6734             }
6735         }
6736
6737         my $volid = PVE::Storage::vdisk_alloc(
6738             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6739
6740         print STDERR "new volume ID is '$volid'\n";
6741         $d->{volid} = $volid;
6742
6743         PVE::Storage::activate_volumes($storecfg, [$volid]);
6744
6745         $map->{$virtdev} = $volid;
6746     }
6747
6748     return $map;
6749 };
6750
6751 sub restore_update_config_line {
6752     my ($cookie, $map, $line, $unique) = @_;
6753
6754     return '' if $line =~ m/^\#qmdump\#/;
6755     return '' if $line =~ m/^\#vzdump\#/;
6756     return '' if $line =~ m/^lock:/;
6757     return '' if $line =~ m/^unused\d+:/;
6758     return '' if $line =~ m/^parent:/;
6759
6760     my $res = '';
6761
6762     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6763     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6764         # try to convert old 1.X settings
6765         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6766         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6767             my ($model, $macaddr) = split(/\=/, $devconfig);
6768             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6769             my $net = {
6770                 model => $model,
6771                 bridge => "vmbr$ind",
6772                 macaddr => $macaddr,
6773             };
6774             my $netstr = print_net($net);
6775
6776             $res .= "net$cookie->{netcount}: $netstr\n";
6777             $cookie->{netcount}++;
6778         }
6779     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6780         my ($id, $netstr) = ($1, $2);
6781         my $net = parse_net($netstr);
6782         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6783         $netstr = print_net($net);
6784         $res .= "$id: $netstr\n";
6785     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6786         my $virtdev = $1;
6787         my $value = $3;
6788         my $di = parse_drive($virtdev, $value);
6789         if (defined($di->{backup}) && !$di->{backup}) {
6790             $res .= "#$line";
6791         } elsif ($map->{$virtdev}) {
6792             delete $di->{format}; # format can change on restore
6793             $di->{file} = $map->{$virtdev};
6794             $value = print_drive($di);
6795             $res .= "$virtdev: $value\n";
6796         } else {
6797             $res .= $line;
6798         }
6799     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6800         my $vmgenid = $1;
6801         if ($vmgenid ne '0') {
6802             # always generate a new vmgenid if there was a valid one setup
6803             $vmgenid = generate_uuid();
6804         }
6805         $res .= "vmgenid: $vmgenid\n";
6806     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6807         my ($uuid, $uuid_str);
6808         UUID::generate($uuid);
6809         UUID::unparse($uuid, $uuid_str);
6810         my $smbios1 = parse_smbios1($2);
6811         $smbios1->{uuid} = $uuid_str;
6812         $res .= $1.print_smbios1($smbios1)."\n";
6813     } else {
6814         $res .= $line;
6815     }
6816
6817     return $res;
6818 }
6819
6820 my $restore_deactivate_volumes = sub {
6821     my ($storecfg, $virtdev_hash) = @_;
6822
6823     my $vollist = [];
6824     for my $dev (values $virtdev_hash->%*) {
6825         push $vollist->@*, $dev->{volid} if $dev->{volid};
6826     }
6827
6828     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6829     print STDERR $@ if $@;
6830 };
6831
6832 my $restore_destroy_volumes = sub {
6833     my ($storecfg, $virtdev_hash) = @_;
6834
6835     for my $dev (values $virtdev_hash->%*) {
6836         my $volid = $dev->{volid} or next;
6837         eval {
6838             PVE::Storage::vdisk_free($storecfg, $volid);
6839             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6840         };
6841         print STDERR "unable to cleanup '$volid' - $@" if $@;
6842     }
6843 };
6844
6845 sub restore_merge_config {
6846     my ($filename, $backup_conf_raw, $override_conf) = @_;
6847
6848     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6849     for my $key (keys $override_conf->%*) {
6850         $backup_conf->{$key} = $override_conf->{$key};
6851     }
6852
6853     return $backup_conf;
6854 }
6855
6856 sub scan_volids {
6857     my ($cfg, $vmid) = @_;
6858
6859     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6860
6861     my $volid_hash = {};
6862     foreach my $storeid (keys %$info) {
6863         foreach my $item (@{$info->{$storeid}}) {
6864             next if !($item->{volid} && $item->{size});
6865             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6866             $volid_hash->{$item->{volid}} = $item;
6867         }
6868     }
6869
6870     return $volid_hash;
6871 }
6872
6873 sub update_disk_config {
6874     my ($vmid, $conf, $volid_hash) = @_;
6875
6876     my $changes;
6877     my $prefix = "VM $vmid";
6878
6879     # used and unused disks
6880     my $referenced = {};
6881
6882     # Note: it is allowed to define multiple storages with same path (alias), so
6883     # we need to check both 'volid' and real 'path' (two different volid can point
6884     # to the same path).
6885
6886     my $referencedpath = {};
6887
6888     # update size info
6889     PVE::QemuConfig->foreach_volume($conf, sub {
6890         my ($opt, $drive) = @_;
6891
6892         my $volid = $drive->{file};
6893         return if !$volid;
6894         my $volume = $volid_hash->{$volid};
6895
6896         # mark volid as "in-use" for next step
6897         $referenced->{$volid} = 1;
6898         if ($volume && (my $path = $volume->{path})) {
6899             $referencedpath->{$path} = 1;
6900         }
6901
6902         return if drive_is_cdrom($drive);
6903         return if !$volume;
6904
6905         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6906         if (defined($updated)) {
6907             $changes = 1;
6908             $conf->{$opt} = print_drive($updated);
6909             print "$prefix ($opt): $msg\n";
6910         }
6911     });
6912
6913     # remove 'unusedX' entry if volume is used
6914     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6915         my ($opt, $drive) = @_;
6916
6917         my $volid = $drive->{file};
6918         return if !$volid;
6919
6920         my $path;
6921         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6922         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6923             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6924             $changes = 1;
6925             delete $conf->{$opt};
6926         }
6927
6928         $referenced->{$volid} = 1;
6929         $referencedpath->{$path} = 1 if $path;
6930     });
6931
6932     foreach my $volid (sort keys %$volid_hash) {
6933         next if $volid =~ m/vm-$vmid-state-/;
6934         next if $referenced->{$volid};
6935         my $path = $volid_hash->{$volid}->{path};
6936         next if !$path; # just to be sure
6937         next if $referencedpath->{$path};
6938         $changes = 1;
6939         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6940         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6941         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6942     }
6943
6944     return $changes;
6945 }
6946
6947 sub rescan {
6948     my ($vmid, $nolock, $dryrun) = @_;
6949
6950     my $cfg = PVE::Storage::config();
6951
6952     print "rescan volumes...\n";
6953     my $volid_hash = scan_volids($cfg, $vmid);
6954
6955     my $updatefn =  sub {
6956         my ($vmid) = @_;
6957
6958         my $conf = PVE::QemuConfig->load_config($vmid);
6959
6960         PVE::QemuConfig->check_lock($conf);
6961
6962         my $vm_volids = {};
6963         foreach my $volid (keys %$volid_hash) {
6964             my $info = $volid_hash->{$volid};
6965             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6966         }
6967
6968         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6969
6970         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6971     };
6972
6973     if (defined($vmid)) {
6974         if ($nolock) {
6975             &$updatefn($vmid);
6976         } else {
6977             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6978         }
6979     } else {
6980         my $vmlist = config_list();
6981         foreach my $vmid (keys %$vmlist) {
6982             if ($nolock) {
6983                 &$updatefn($vmid);
6984             } else {
6985                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6986             }
6987         }
6988     }
6989 }
6990
6991 sub restore_proxmox_backup_archive {
6992     my ($archive, $vmid, $user, $options) = @_;
6993
6994     my $storecfg = PVE::Storage::config();
6995
6996     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6997     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6998
6999     my $fingerprint = $scfg->{fingerprint};
7000     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7001
7002     my $repo = PVE::PBSClient::get_repository($scfg);
7003     my $namespace = $scfg->{namespace};
7004
7005     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7006     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7007     local $ENV{PBS_PASSWORD} = $password;
7008     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7009
7010     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7011         PVE::Storage::parse_volname($storecfg, $archive);
7012
7013     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7014
7015     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7016
7017     my $tmpdir = "/var/tmp/vzdumptmp$$";
7018     rmtree $tmpdir;
7019     mkpath $tmpdir;
7020
7021     my $conffile = PVE::QemuConfig->config_file($vmid);
7022      # disable interrupts (always do cleanups)
7023     local $SIG{INT} =
7024         local $SIG{TERM} =
7025         local $SIG{QUIT} =
7026         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7027
7028     # Note: $oldconf is undef if VM does not exists
7029     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7030     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7031     my $new_conf_raw = '';
7032
7033     my $rpcenv = PVE::RPCEnvironment::get();
7034     my $devinfo = {}; # info about drives included in backup
7035     my $virtdev_hash = {}; # info about allocated drives
7036
7037     eval {
7038         # enable interrupts
7039         local $SIG{INT} =
7040             local $SIG{TERM} =
7041             local $SIG{QUIT} =
7042             local $SIG{HUP} =
7043             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7044
7045         my $cfgfn = "$tmpdir/qemu-server.conf";
7046         my $firewall_config_fn = "$tmpdir/fw.conf";
7047         my $index_fn = "$tmpdir/index.json";
7048
7049         my $cmd = "restore";
7050
7051         my $param = [$pbs_backup_name, "index.json", $index_fn];
7052         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7053         my $index = PVE::Tools::file_get_contents($index_fn);
7054         $index = decode_json($index);
7055
7056         foreach my $info (@{$index->{files}}) {
7057             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7058                 my $devname = $1;
7059                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7060                     $devinfo->{$devname}->{size} = $1;
7061                 } else {
7062                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7063                 }
7064             }
7065         }
7066
7067         my $is_qemu_server_backup = scalar(
7068             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7069         );
7070         if (!$is_qemu_server_backup) {
7071             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7072         }
7073         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7074
7075         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7076         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7077
7078         if ($has_firewall_config) {
7079             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7080             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7081
7082             my $pve_firewall_dir = '/etc/pve/firewall';
7083             mkdir $pve_firewall_dir; # make sure the dir exists
7084             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7085         }
7086
7087         my $fh = IO::File->new($cfgfn, "r") ||
7088             die "unable to read qemu-server.conf - $!\n";
7089
7090         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7091
7092         # fixme: rate limit?
7093
7094         # create empty/temp config
7095         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7096
7097         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7098
7099         # allocate volumes
7100         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7101
7102         foreach my $virtdev (sort keys %$virtdev_hash) {
7103             my $d = $virtdev_hash->{$virtdev};
7104             next if $d->{is_cloudinit}; # no need to restore cloudinit
7105
7106             # this fails if storage is unavailable
7107             my $volid = $d->{volid};
7108             my $path = PVE::Storage::path($storecfg, $volid);
7109
7110             # for live-restore we only want to preload the efidisk and TPM state
7111             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7112
7113             my @ns_arg;
7114             if (defined(my $ns = $scfg->{namespace})) {
7115                 @ns_arg = ('--ns', $ns);
7116             }
7117
7118             my $pbs_restore_cmd = [
7119                 '/usr/bin/pbs-restore',
7120                 '--repository', $repo,
7121                 @ns_arg,
7122                 $pbs_backup_name,
7123                 "$d->{devname}.img.fidx",
7124                 $path,
7125                 '--verbose',
7126                 ];
7127
7128             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7129             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7130
7131             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7132                 push @$pbs_restore_cmd, '--skip-zero';
7133             }
7134
7135             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7136             print "restore proxmox backup image: $dbg_cmdstring\n";
7137             run_command($pbs_restore_cmd);
7138         }
7139
7140         $fh->seek(0, 0) || die "seek failed - $!\n";
7141
7142         my $cookie = { netcount => 0 };
7143         while (defined(my $line = <$fh>)) {
7144             $new_conf_raw .= restore_update_config_line(
7145                 $cookie,
7146                 $map,
7147                 $line,
7148                 $options->{unique},
7149             );
7150         }
7151
7152         $fh->close();
7153     };
7154     my $err = $@;
7155
7156     if ($err || !$options->{live}) {
7157         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7158     }
7159
7160     rmtree $tmpdir;
7161
7162     if ($err) {
7163         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7164         die $err;
7165     }
7166
7167     if ($options->{live}) {
7168         # keep lock during live-restore
7169         $new_conf_raw .= "\nlock: create";
7170     }
7171
7172     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7173     check_restore_permissions($rpcenv, $user, $new_conf);
7174     PVE::QemuConfig->write_config($vmid, $new_conf);
7175
7176     eval { rescan($vmid, 1); };
7177     warn $@ if $@;
7178
7179     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7180
7181     if ($options->{live}) {
7182         # enable interrupts
7183         local $SIG{INT} =
7184             local $SIG{TERM} =
7185             local $SIG{QUIT} =
7186             local $SIG{HUP} =
7187             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7188
7189         my $conf = PVE::QemuConfig->load_config($vmid);
7190         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7191
7192         # these special drives are already restored before start
7193         delete $devinfo->{'drive-efidisk0'};
7194         delete $devinfo->{'drive-tpmstate0-backup'};
7195
7196         my $pbs_opts = {
7197             repo => $repo,
7198             keyfile => $keyfile,
7199             snapshot => $pbs_backup_name,
7200             namespace => $namespace,
7201         };
7202         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7203
7204         PVE::QemuConfig->remove_lock($vmid, "create");
7205     }
7206 }
7207
7208 sub pbs_live_restore {
7209     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7210
7211     print "starting VM for live-restore\n";
7212     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7213
7214     my $live_restore_backing = {};
7215     for my $ds (keys %$restored_disks) {
7216         $ds =~ m/^drive-(.*)$/;
7217         my $confname = $1;
7218         my $pbs_conf = {};
7219         $pbs_conf = {
7220             repository => $opts->{repo},
7221             snapshot => $opts->{snapshot},
7222             archive => "$ds.img.fidx",
7223         };
7224         $pbs_conf->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7225         $pbs_conf->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7226
7227         my $drive = parse_drive($confname, $conf->{$confname});
7228         print "restoring '$ds' to '$drive->{file}'\n";
7229
7230         my $pbs_name = "drive-${confname}-pbs";
7231         $live_restore_backing->{$confname} = {
7232             name => $pbs_name,
7233             blockdev => print_pbs_blockdev($pbs_conf, $pbs_name),
7234         };
7235     }
7236
7237     my $drives_streamed = 0;
7238     eval {
7239         # make sure HA doesn't interrupt our restore by stopping the VM
7240         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7241             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7242         }
7243
7244         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7245         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7246         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
7247
7248         my $qmeventd_fd = register_qmeventd_handle($vmid);
7249
7250         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7251         # this will effectively collapse the backing image chain consisting of
7252         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7253         # removes itself once all backing images vanish with 'auto-remove=on')
7254         my $jobs = {};
7255         for my $ds (sort keys %$restored_disks) {
7256             my $job_id = "restore-$ds";
7257             mon_cmd($vmid, 'block-stream',
7258                 'job-id' => $job_id,
7259                 device => "$ds",
7260             );
7261             $jobs->{$job_id} = {};
7262         }
7263
7264         mon_cmd($vmid, 'cont');
7265         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7266
7267         print "restore-drive jobs finished successfully, removing all tracking block devices"
7268             ." to disconnect from Proxmox Backup Server\n";
7269
7270         for my $ds (sort keys %$restored_disks) {
7271             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7272         }
7273
7274         close($qmeventd_fd);
7275     };
7276
7277     my $err = $@;
7278
7279     if ($err) {
7280         warn "An error occurred during live-restore: $err\n";
7281         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7282         die "live-restore failed\n";
7283     }
7284 }
7285
7286 sub restore_vma_archive {
7287     my ($archive, $vmid, $user, $opts, $comp) = @_;
7288
7289     my $readfrom = $archive;
7290
7291     my $cfg = PVE::Storage::config();
7292     my $commands = [];
7293     my $bwlimit = $opts->{bwlimit};
7294
7295     my $dbg_cmdstring = '';
7296     my $add_pipe = sub {
7297         my ($cmd) = @_;
7298         push @$commands, $cmd;
7299         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7300         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7301         $readfrom = '-';
7302     };
7303
7304     my $input = undef;
7305     if ($archive eq '-') {
7306         $input = '<&STDIN';
7307     } else {
7308         # If we use a backup from a PVE defined storage we also consider that
7309         # storage's rate limit:
7310         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7311         if (defined($volid)) {
7312             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7313             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7314             if ($readlimit) {
7315                 print STDERR "applying read rate limit: $readlimit\n";
7316                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7317                 $add_pipe->($cstream);
7318             }
7319         }
7320     }
7321
7322     if ($comp) {
7323         my $info = PVE::Storage::decompressor_info('vma', $comp);
7324         my $cmd = $info->{decompressor};
7325         push @$cmd, $readfrom;
7326         $add_pipe->($cmd);
7327     }
7328
7329     my $tmpdir = "/var/tmp/vzdumptmp$$";
7330     rmtree $tmpdir;
7331
7332     # disable interrupts (always do cleanups)
7333     local $SIG{INT} =
7334         local $SIG{TERM} =
7335         local $SIG{QUIT} =
7336         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7337
7338     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7339     POSIX::mkfifo($mapfifo, 0600);
7340     my $fifofh;
7341     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7342
7343     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7344
7345     my $devinfo = {}; # info about drives included in backup
7346     my $virtdev_hash = {}; # info about allocated drives
7347
7348     my $rpcenv = PVE::RPCEnvironment::get();
7349
7350     my $conffile = PVE::QemuConfig->config_file($vmid);
7351
7352     # Note: $oldconf is undef if VM does not exist
7353     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7354     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7355     my $new_conf_raw = '';
7356
7357     my %storage_limits;
7358
7359     my $print_devmap = sub {
7360         my $cfgfn = "$tmpdir/qemu-server.conf";
7361
7362         # we can read the config - that is already extracted
7363         my $fh = IO::File->new($cfgfn, "r") ||
7364             die "unable to read qemu-server.conf - $!\n";
7365
7366         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7367         if (-f $fwcfgfn) {
7368             my $pve_firewall_dir = '/etc/pve/firewall';
7369             mkdir $pve_firewall_dir; # make sure the dir exists
7370             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7371         }
7372
7373         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7374
7375         foreach my $info (values %{$virtdev_hash}) {
7376             my $storeid = $info->{storeid};
7377             next if defined($storage_limits{$storeid});
7378
7379             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7380             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7381             $storage_limits{$storeid} = $limit * 1024;
7382         }
7383
7384         foreach my $devname (keys %$devinfo) {
7385             die "found no device mapping information for device '$devname'\n"
7386                 if !$devinfo->{$devname}->{virtdev};
7387         }
7388
7389         # create empty/temp config
7390         if ($oldconf) {
7391             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7392             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7393         }
7394
7395         # allocate volumes
7396         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7397
7398         # print restore information to $fifofh
7399         foreach my $virtdev (sort keys %$virtdev_hash) {
7400             my $d = $virtdev_hash->{$virtdev};
7401             next if $d->{is_cloudinit}; # no need to restore cloudinit
7402
7403             my $storeid = $d->{storeid};
7404             my $volid = $d->{volid};
7405
7406             my $map_opts = '';
7407             if (my $limit = $storage_limits{$storeid}) {
7408                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7409             }
7410
7411             my $write_zeros = 1;
7412             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7413                 $write_zeros = 0;
7414             }
7415
7416             my $path = PVE::Storage::path($cfg, $volid);
7417
7418             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7419
7420             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7421         }
7422
7423         $fh->seek(0, 0) || die "seek failed - $!\n";
7424
7425         my $cookie = { netcount => 0 };
7426         while (defined(my $line = <$fh>)) {
7427             $new_conf_raw .= restore_update_config_line(
7428                 $cookie,
7429                 $map,
7430                 $line,
7431                 $opts->{unique},
7432             );
7433         }
7434
7435         $fh->close();
7436     };
7437
7438     my $oldtimeout;
7439
7440     eval {
7441         # enable interrupts
7442         local $SIG{INT} =
7443             local $SIG{TERM} =
7444             local $SIG{QUIT} =
7445             local $SIG{HUP} =
7446             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7447         local $SIG{ALRM} = sub { die "got timeout\n"; };
7448
7449         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7450
7451         my $parser = sub {
7452             my $line = shift;
7453
7454             print "$line\n";
7455
7456             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7457                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7458                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7459             } elsif ($line =~ m/^CTIME: /) {
7460                 # we correctly received the vma config, so we can disable
7461                 # the timeout now for disk allocation
7462                 alarm($oldtimeout || 0);
7463                 $oldtimeout = undef;
7464                 &$print_devmap();
7465                 print $fifofh "done\n";
7466                 close($fifofh);
7467                 $fifofh = undef;
7468             }
7469         };
7470
7471         print "restore vma archive: $dbg_cmdstring\n";
7472         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7473     };
7474     my $err = $@;
7475
7476     alarm($oldtimeout) if $oldtimeout;
7477
7478     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7479
7480     close($fifofh) if $fifofh;
7481     unlink $mapfifo;
7482     rmtree $tmpdir;
7483
7484     if ($err) {
7485         $restore_destroy_volumes->($cfg, $virtdev_hash);
7486         die $err;
7487     }
7488
7489     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7490     check_restore_permissions($rpcenv, $user, $new_conf);
7491     PVE::QemuConfig->write_config($vmid, $new_conf);
7492
7493     eval { rescan($vmid, 1); };
7494     warn $@ if $@;
7495
7496     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7497 }
7498
7499 sub restore_tar_archive {
7500     my ($archive, $vmid, $user, $opts) = @_;
7501
7502     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7503         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7504         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7505     }
7506
7507     if ($archive ne '-') {
7508         my $firstfile = tar_archive_read_firstfile($archive);
7509         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7510             if $firstfile ne 'qemu-server.conf';
7511     }
7512
7513     my $storecfg = PVE::Storage::config();
7514
7515     # avoid zombie disks when restoring over an existing VM -> cleanup first
7516     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7517     # skiplock=1 because qmrestore has set the 'create' lock itself already
7518     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7519     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7520
7521     my $tocmd = "/usr/lib/qemu-server/qmextract";
7522
7523     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7524     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7525     $tocmd .= ' --prealloc' if $opts->{prealloc};
7526     $tocmd .= ' --info' if $opts->{info};
7527
7528     # tar option "xf" does not autodetect compression when read from STDIN,
7529     # so we pipe to zcat
7530     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7531         PVE::Tools::shellquote("--to-command=$tocmd");
7532
7533     my $tmpdir = "/var/tmp/vzdumptmp$$";
7534     mkpath $tmpdir;
7535
7536     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7537     local $ENV{VZDUMP_VMID} = $vmid;
7538     local $ENV{VZDUMP_USER} = $user;
7539
7540     my $conffile = PVE::QemuConfig->config_file($vmid);
7541     my $new_conf_raw = '';
7542
7543     # disable interrupts (always do cleanups)
7544     local $SIG{INT} =
7545         local $SIG{TERM} =
7546         local $SIG{QUIT} =
7547         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7548
7549     eval {
7550         # enable interrupts
7551         local $SIG{INT} =
7552             local $SIG{TERM} =
7553             local $SIG{QUIT} =
7554             local $SIG{HUP} =
7555             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7556
7557         if ($archive eq '-') {
7558             print "extracting archive from STDIN\n";
7559             run_command($cmd, input => "<&STDIN");
7560         } else {
7561             print "extracting archive '$archive'\n";
7562             run_command($cmd);
7563         }
7564
7565         return if $opts->{info};
7566
7567         # read new mapping
7568         my $map = {};
7569         my $statfile = "$tmpdir/qmrestore.stat";
7570         if (my $fd = IO::File->new($statfile, "r")) {
7571             while (defined (my $line = <$fd>)) {
7572                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7573                     $map->{$1} = $2 if $1;
7574                 } else {
7575                     print STDERR "unable to parse line in statfile - $line\n";
7576                 }
7577             }
7578             $fd->close();
7579         }
7580
7581         my $confsrc = "$tmpdir/qemu-server.conf";
7582
7583         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7584
7585         my $cookie = { netcount => 0 };
7586         while (defined (my $line = <$srcfd>)) {
7587             $new_conf_raw .= restore_update_config_line(
7588                 $cookie,
7589                 $map,
7590                 $line,
7591                 $opts->{unique},
7592             );
7593         }
7594
7595         $srcfd->close();
7596     };
7597     if (my $err = $@) {
7598         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7599         die $err;
7600     }
7601
7602     rmtree $tmpdir;
7603
7604     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7605
7606     PVE::Cluster::cfs_update(); # make sure we read new file
7607
7608     eval { rescan($vmid, 1); };
7609     warn $@ if $@;
7610 };
7611
7612 sub foreach_storage_used_by_vm {
7613     my ($conf, $func) = @_;
7614
7615     my $sidhash = {};
7616
7617     PVE::QemuConfig->foreach_volume($conf, sub {
7618         my ($ds, $drive) = @_;
7619         return if drive_is_cdrom($drive);
7620
7621         my $volid = $drive->{file};
7622
7623         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7624         $sidhash->{$sid} = $sid if $sid;
7625     });
7626
7627     foreach my $sid (sort keys %$sidhash) {
7628         &$func($sid);
7629     }
7630 }
7631
7632 my $qemu_snap_storage = {
7633     rbd => 1,
7634 };
7635 sub do_snapshots_with_qemu {
7636     my ($storecfg, $volid, $deviceid) = @_;
7637
7638     return if $deviceid =~ m/tpmstate0/;
7639
7640     my $storage_name = PVE::Storage::parse_volume_id($volid);
7641     my $scfg = $storecfg->{ids}->{$storage_name};
7642     die "could not find storage '$storage_name'\n" if !defined($scfg);
7643
7644     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7645         return 1;
7646     }
7647
7648     if ($volid =~ m/\.(qcow2|qed)$/){
7649         return 1;
7650     }
7651
7652     return;
7653 }
7654
7655 sub qga_check_running {
7656     my ($vmid, $nowarn) = @_;
7657
7658     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7659     if ($@) {
7660         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7661         return 0;
7662     }
7663     return 1;
7664 }
7665
7666 sub template_create {
7667     my ($vmid, $conf, $disk) = @_;
7668
7669     my $storecfg = PVE::Storage::config();
7670
7671     PVE::QemuConfig->foreach_volume($conf, sub {
7672         my ($ds, $drive) = @_;
7673
7674         return if drive_is_cdrom($drive);
7675         return if $disk && $ds ne $disk;
7676
7677         my $volid = $drive->{file};
7678         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7679
7680         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7681         $drive->{file} = $voliddst;
7682         $conf->{$ds} = print_drive($drive);
7683         PVE::QemuConfig->write_config($vmid, $conf);
7684     });
7685 }
7686
7687 sub convert_iscsi_path {
7688     my ($path) = @_;
7689
7690     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7691         my $portal = $1;
7692         my $target = $2;
7693         my $lun = $3;
7694
7695         my $initiator_name = get_initiator_name();
7696
7697         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7698                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7699     }
7700
7701     die "cannot convert iscsi path '$path', unkown format\n";
7702 }
7703
7704 sub qemu_img_convert {
7705     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7706
7707     my $storecfg = PVE::Storage::config();
7708     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7709     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7710
7711     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7712
7713     my $cachemode;
7714     my $src_path;
7715     my $src_is_iscsi = 0;
7716     my $src_format;
7717
7718     if ($src_storeid) {
7719         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7720         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7721         $src_format = qemu_img_format($src_scfg, $src_volname);
7722         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7723         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7724         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7725     } elsif (-f $src_volid || -b $src_volid) {
7726         $src_path = $src_volid;
7727         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7728             $src_format = $1;
7729         }
7730     }
7731
7732     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7733
7734     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7735     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7736     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7737     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7738
7739     my $cmd = [];
7740     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7741     push @$cmd, '-l', "snapshot.name=$snapname"
7742         if $snapname && $src_format && $src_format eq "qcow2";
7743     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7744     push @$cmd, '-T', $cachemode if defined($cachemode);
7745     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7746
7747     if ($src_is_iscsi) {
7748         push @$cmd, '--image-opts';
7749         $src_path = convert_iscsi_path($src_path);
7750     } elsif ($src_format) {
7751         push @$cmd, '-f', $src_format;
7752     }
7753
7754     if ($dst_is_iscsi) {
7755         push @$cmd, '--target-image-opts';
7756         $dst_path = convert_iscsi_path($dst_path);
7757     } else {
7758         push @$cmd, '-O', $dst_format;
7759     }
7760
7761     push @$cmd, $src_path;
7762
7763     if (!$dst_is_iscsi && $is_zero_initialized) {
7764         push @$cmd, "zeroinit:$dst_path";
7765     } else {
7766         push @$cmd, $dst_path;
7767     }
7768
7769     my $parser = sub {
7770         my $line = shift;
7771         if($line =~ m/\((\S+)\/100\%\)/){
7772             my $percent = $1;
7773             my $transferred = int($size * $percent / 100);
7774             my $total_h = render_bytes($size, 1);
7775             my $transferred_h = render_bytes($transferred, 1);
7776
7777             print "transferred $transferred_h of $total_h ($percent%)\n";
7778         }
7779
7780     };
7781
7782     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7783     my $err = $@;
7784     die "copy failed: $err" if $err;
7785 }
7786
7787 sub qemu_img_format {
7788     my ($scfg, $volname) = @_;
7789
7790     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7791         return $1;
7792     } else {
7793         return "raw";
7794     }
7795 }
7796
7797 sub qemu_drive_mirror {
7798     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7799
7800     $jobs = {} if !$jobs;
7801
7802     my $qemu_target;
7803     my $format;
7804     $jobs->{"drive-$drive"} = {};
7805
7806     if ($dst_volid =~ /^nbd:/) {
7807         $qemu_target = $dst_volid;
7808         $format = "nbd";
7809     } else {
7810         my $storecfg = PVE::Storage::config();
7811         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7812
7813         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7814
7815         $format = qemu_img_format($dst_scfg, $dst_volname);
7816
7817         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7818
7819         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7820     }
7821
7822     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7823     $opts->{format} = $format if $format;
7824
7825     if (defined($src_bitmap)) {
7826         $opts->{sync} = 'incremental';
7827         $opts->{bitmap} = $src_bitmap;
7828         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7829     }
7830
7831     if (defined($bwlimit)) {
7832         $opts->{speed} = $bwlimit * 1024;
7833         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7834     } else {
7835         print "drive mirror is starting for drive-$drive\n";
7836     }
7837
7838     # if a job already runs for this device we get an error, catch it for cleanup
7839     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7840     if (my $err = $@) {
7841         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7842         warn "$@\n" if $@;
7843         die "mirroring error: $err\n";
7844     }
7845
7846     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7847 }
7848
7849 # $completion can be either
7850 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7851 # 'cancel': wait until all jobs are ready, block-job-cancel them
7852 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7853 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7854 sub qemu_drive_mirror_monitor {
7855     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7856
7857     $completion //= 'complete';
7858     $op //= "mirror";
7859
7860     eval {
7861         my $err_complete = 0;
7862
7863         my $starttime = time ();
7864         while (1) {
7865             die "block job ('$op') timed out\n" if $err_complete > 300;
7866
7867             my $stats = mon_cmd($vmid, "query-block-jobs");
7868             my $ctime = time();
7869
7870             my $running_jobs = {};
7871             for my $stat (@$stats) {
7872                 next if $stat->{type} ne $op;
7873                 $running_jobs->{$stat->{device}} = $stat;
7874             }
7875
7876             my $readycounter = 0;
7877
7878             for my $job_id (sort keys %$jobs) {
7879                 my $job = $running_jobs->{$job_id};
7880
7881                 my $vanished = !defined($job);
7882                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7883                 if($complete || ($vanished && $completion eq 'auto')) {
7884                     print "$job_id: $op-job finished\n";
7885                     delete $jobs->{$job_id};
7886                     next;
7887                 }
7888
7889                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7890
7891                 my $busy = $job->{busy};
7892                 my $ready = $job->{ready};
7893                 if (my $total = $job->{len}) {
7894                     my $transferred = $job->{offset} || 0;
7895                     my $remaining = $total - $transferred;
7896                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7897
7898                     my $duration = $ctime - $starttime;
7899                     my $total_h = render_bytes($total, 1);
7900                     my $transferred_h = render_bytes($transferred, 1);
7901
7902                     my $status = sprintf(
7903                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7904                         render_duration($duration),
7905                     );
7906
7907                     if ($ready) {
7908                         if ($busy) {
7909                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7910                         } else {
7911                             $status .= ", ready";
7912                         }
7913                     }
7914                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7915                     $jobs->{$job_id}->{ready} = $ready;
7916                 }
7917
7918                 $readycounter++ if $job->{ready};
7919             }
7920
7921             last if scalar(keys %$jobs) == 0;
7922
7923             if ($readycounter == scalar(keys %$jobs)) {
7924                 print "all '$op' jobs are ready\n";
7925
7926                 # do the complete later (or has already been done)
7927                 last if $completion eq 'skip' || $completion eq 'auto';
7928
7929                 if ($vmiddst && $vmiddst != $vmid) {
7930                     my $agent_running = $qga && qga_check_running($vmid);
7931                     if ($agent_running) {
7932                         print "freeze filesystem\n";
7933                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7934                         warn $@ if $@;
7935                     } else {
7936                         print "suspend vm\n";
7937                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7938                         warn $@ if $@;
7939                     }
7940
7941                     # if we clone a disk for a new target vm, we don't switch the disk
7942                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7943
7944                     if ($agent_running) {
7945                         print "unfreeze filesystem\n";
7946                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7947                         warn $@ if $@;
7948                     } else {
7949                         print "resume vm\n";
7950                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7951                         warn $@ if $@;
7952                     }
7953
7954                     last;
7955                 } else {
7956
7957                     for my $job_id (sort keys %$jobs) {
7958                         # try to switch the disk if source and destination are on the same guest
7959                         print "$job_id: Completing block job_id...\n";
7960
7961                         my $op;
7962                         if ($completion eq 'complete') {
7963                             $op = 'block-job-complete';
7964                         } elsif ($completion eq 'cancel') {
7965                             $op = 'block-job-cancel';
7966                         } else {
7967                             die "invalid completion value: $completion\n";
7968                         }
7969                         eval { mon_cmd($vmid, $op, device => $job_id) };
7970                         if ($@ =~ m/cannot be completed/) {
7971                             print "$job_id: block job cannot be completed, trying again.\n";
7972                             $err_complete++;
7973                         }else {
7974                             print "$job_id: Completed successfully.\n";
7975                             $jobs->{$job_id}->{complete} = 1;
7976                         }
7977                     }
7978                 }
7979             }
7980             sleep 1;
7981         }
7982     };
7983     my $err = $@;
7984
7985     if ($err) {
7986         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7987         die "block job ($op) error: $err";
7988     }
7989 }
7990
7991 sub qemu_blockjobs_cancel {
7992     my ($vmid, $jobs) = @_;
7993
7994     foreach my $job (keys %$jobs) {
7995         print "$job: Cancelling block job\n";
7996         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7997         $jobs->{$job}->{cancel} = 1;
7998     }
7999
8000     while (1) {
8001         my $stats = mon_cmd($vmid, "query-block-jobs");
8002
8003         my $running_jobs = {};
8004         foreach my $stat (@$stats) {
8005             $running_jobs->{$stat->{device}} = $stat;
8006         }
8007
8008         foreach my $job (keys %$jobs) {
8009
8010             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8011                 print "$job: Done.\n";
8012                 delete $jobs->{$job};
8013             }
8014         }
8015
8016         last if scalar(keys %$jobs) == 0;
8017
8018         sleep 1;
8019     }
8020 }
8021
8022 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8023 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8024 my sub clone_disk_check_io_uring {
8025     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8026
8027     return if !$use_drive_mirror;
8028
8029     # Don't complain when not changing storage.
8030     # Assume if it works for the source, it'll work for the target too.
8031     return if $src_storeid eq $dst_storeid;
8032
8033     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8034     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8035
8036     my $cache_direct = drive_uses_cache_direct($src_drive);
8037
8038     my $src_uses_io_uring;
8039     if ($src_drive->{aio}) {
8040         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8041     } else {
8042         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8043     }
8044
8045     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8046         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8047 }
8048
8049 sub clone_disk {
8050     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8051
8052     my ($vmid, $running) = $source->@{qw(vmid running)};
8053     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8054
8055     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8056     my ($storage, $format) = $dest->@{qw(storage format)};
8057
8058     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8059
8060     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8061         die "cloning from/to EFI disk requires EFI disk\n"
8062             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8063         die "cloning from/to TPM state requires TPM state\n"
8064             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8065
8066         # This would lead to two device nodes in QEMU pointing to the same backing image!
8067         die "cannot change drive name when cloning disk from/to the same VM\n"
8068             if $use_drive_mirror && $vmid == $newvmid;
8069     }
8070
8071     die "cannot move TPM state while VM is running\n"
8072         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8073
8074     my $newvolid;
8075
8076     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8077     print "$src_drivename " if $src_drivename;
8078     print "($drive->{file})\n";
8079
8080     if (!$full) {
8081         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8082         push @$newvollist, $newvolid;
8083     } else {
8084         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8085         my $storeid = $storage || $src_storeid;
8086
8087         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8088
8089         my $name = undef;
8090         my $size = undef;
8091         if (drive_is_cloudinit($drive)) {
8092             $name = "vm-$newvmid-cloudinit";
8093             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8094             if ($scfg->{path}) {
8095                 $name .= ".$dst_format";
8096             }
8097             $snapname = undef;
8098             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8099         } elsif ($dst_drivename eq 'efidisk0') {
8100             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8101         } elsif ($dst_drivename eq 'tpmstate0') {
8102             $dst_format = 'raw';
8103             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8104         } else {
8105             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8106
8107             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8108         }
8109         $newvolid = PVE::Storage::vdisk_alloc(
8110             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8111         );
8112         push @$newvollist, $newvolid;
8113
8114         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8115
8116         if (drive_is_cloudinit($drive)) {
8117             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8118             # if this is the case, we have to complete any block-jobs still there from
8119             # previous drive-mirrors
8120             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8121                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8122             }
8123             goto no_data_clone;
8124         }
8125
8126         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8127         if ($use_drive_mirror) {
8128             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8129                 $completion, $qga, $bwlimit);
8130         } else {
8131             if ($dst_drivename eq 'efidisk0') {
8132                 # the relevant data on the efidisk may be smaller than the source
8133                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8134                 # that is given by the OVMF_VARS.fd
8135                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8136                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8137
8138                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8139
8140                 # better for Ceph if block size is not too small, see bug #3324
8141                 my $bs = 1024*1024;
8142
8143                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8144
8145                 if ($src_format eq 'qcow2' && $snapname) {
8146                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8147                         if !min_version(kvm_user_version(), 6, 2);
8148                     push $cmd->@*, '-l', $snapname;
8149                 }
8150                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8151                 run_command($cmd);
8152             } else {
8153                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8154             }
8155         }
8156     }
8157
8158 no_data_clone:
8159     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8160
8161     my $disk = dclone($drive);
8162     delete $disk->{format};
8163     $disk->{file} = $newvolid;
8164     $disk->{size} = $size if defined($size);
8165
8166     return $disk;
8167 }
8168
8169 sub get_running_qemu_version {
8170     my ($vmid) = @_;
8171     my $res = mon_cmd($vmid, "query-version");
8172     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8173 }
8174
8175 sub qemu_use_old_bios_files {
8176     my ($machine_type) = @_;
8177
8178     return if !$machine_type;
8179
8180     my $use_old_bios_files = undef;
8181
8182     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8183         $machine_type = $1;
8184         $use_old_bios_files = 1;
8185     } else {
8186         my $version = extract_version($machine_type, kvm_user_version());
8187         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8188         # load new efi bios files on migration. So this hack is required to allow
8189         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8190         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8191         $use_old_bios_files = !min_version($version, 2, 4);
8192     }
8193
8194     return ($use_old_bios_files, $machine_type);
8195 }
8196
8197 sub get_efivars_size {
8198     my ($conf, $efidisk) = @_;
8199
8200     my $arch = get_vm_arch($conf);
8201     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8202     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8203     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8204     return -s $ovmf_vars;
8205 }
8206
8207 sub update_efidisk_size {
8208     my ($conf) = @_;
8209
8210     return if !defined($conf->{efidisk0});
8211
8212     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8213     $disk->{size} = get_efivars_size($conf);
8214     $conf->{efidisk0} = print_drive($disk);
8215
8216     return;
8217 }
8218
8219 sub update_tpmstate_size {
8220     my ($conf) = @_;
8221
8222     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8223     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8224     $conf->{tpmstate0} = print_drive($disk);
8225 }
8226
8227 sub create_efidisk($$$$$$$) {
8228     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8229
8230     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8231
8232     my $vars_size_b = -s $ovmf_vars;
8233     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8234     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8235     PVE::Storage::activate_volumes($storecfg, [$volid]);
8236
8237     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8238     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8239
8240     return ($volid, $size/1024);
8241 }
8242
8243 sub vm_iothreads_list {
8244     my ($vmid) = @_;
8245
8246     my $res = mon_cmd($vmid, 'query-iothreads');
8247
8248     my $iothreads = {};
8249     foreach my $iothread (@$res) {
8250         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8251     }
8252
8253     return $iothreads;
8254 }
8255
8256 sub scsihw_infos {
8257     my ($conf, $drive) = @_;
8258
8259     my $maxdev = 0;
8260
8261     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8262         $maxdev = 7;
8263     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8264         $maxdev = 1;
8265     } else {
8266         $maxdev = 256;
8267     }
8268
8269     my $controller = int($drive->{index} / $maxdev);
8270     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8271         ? "virtioscsi"
8272         : "scsihw";
8273
8274     return ($maxdev, $controller, $controller_prefix);
8275 }
8276
8277 sub resolve_dst_disk_format {
8278         my ($storecfg, $storeid, $src_volname, $format) = @_;
8279         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8280
8281         if (!$format) {
8282             # if no target format is specified, use the source disk format as hint
8283             if ($src_volname) {
8284                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8285                 $format = qemu_img_format($scfg, $src_volname);
8286             } else {
8287                 return $defFormat;
8288             }
8289         }
8290
8291         # test if requested format is supported - else use default
8292         my $supported = grep { $_ eq $format } @$validFormats;
8293         $format = $defFormat if !$supported;
8294         return $format;
8295 }
8296
8297 # NOTE: if this logic changes, please update docs & possibly gui logic
8298 sub find_vmstate_storage {
8299     my ($conf, $storecfg) = @_;
8300
8301     # first, return storage from conf if set
8302     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8303
8304     my ($target, $shared, $local);
8305
8306     foreach_storage_used_by_vm($conf, sub {
8307         my ($sid) = @_;
8308         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8309         my $dst = $scfg->{shared} ? \$shared : \$local;
8310         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8311     });
8312
8313     # second, use shared storage where VM has at least one disk
8314     # third, use local storage where VM has at least one disk
8315     # fall back to local storage
8316     $target = $shared // $local // 'local';
8317
8318     return $target;
8319 }
8320
8321 sub generate_uuid {
8322     my ($uuid, $uuid_str);
8323     UUID::generate($uuid);
8324     UUID::unparse($uuid, $uuid_str);
8325     return $uuid_str;
8326 }
8327
8328 sub generate_smbios1_uuid {
8329     return "uuid=".generate_uuid();
8330 }
8331
8332 sub nbd_stop {
8333     my ($vmid) = @_;
8334
8335     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8336 }
8337
8338 sub create_reboot_request {
8339     my ($vmid) = @_;
8340     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8341         or die "failed to create reboot trigger file: $!\n";
8342     close($fh);
8343 }
8344
8345 sub clear_reboot_request {
8346     my ($vmid) = @_;
8347     my $path = "/run/qemu-server/$vmid.reboot";
8348     my $res = 0;
8349
8350     $res = unlink($path);
8351     die "could not remove reboot request for $vmid: $!"
8352         if !$res && $! != POSIX::ENOENT;
8353
8354     return $res;
8355 }
8356
8357 sub bootorder_from_legacy {
8358     my ($conf, $bootcfg) = @_;
8359
8360     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8361     my $bootindex_hash = {};
8362     my $i = 1;
8363     foreach my $o (split(//, $boot)) {
8364         $bootindex_hash->{$o} = $i*100;
8365         $i++;
8366     }
8367
8368     my $bootorder = {};
8369
8370     PVE::QemuConfig->foreach_volume($conf, sub {
8371         my ($ds, $drive) = @_;
8372
8373         if (drive_is_cdrom ($drive, 1)) {
8374             if ($bootindex_hash->{d}) {
8375                 $bootorder->{$ds} = $bootindex_hash->{d};
8376                 $bootindex_hash->{d} += 1;
8377             }
8378         } elsif ($bootindex_hash->{c}) {
8379             $bootorder->{$ds} = $bootindex_hash->{c}
8380                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8381             $bootindex_hash->{c} += 1;
8382         }
8383     });
8384
8385     if ($bootindex_hash->{n}) {
8386         for (my $i = 0; $i < $MAX_NETS; $i++) {
8387             my $netname = "net$i";
8388             next if !$conf->{$netname};
8389             $bootorder->{$netname} = $bootindex_hash->{n};
8390             $bootindex_hash->{n} += 1;
8391         }
8392     }
8393
8394     return $bootorder;
8395 }
8396
8397 # Generate default device list for 'boot: order=' property. Matches legacy
8398 # default boot order, but with explicit device names. This is important, since
8399 # the fallback for when neither 'order' nor the old format is specified relies
8400 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8401 sub get_default_bootdevices {
8402     my ($conf) = @_;
8403
8404     my @ret = ();
8405
8406     # harddisk
8407     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8408     push @ret, $first if $first;
8409
8410     # cdrom
8411     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8412     push @ret, $first if $first;
8413
8414     # network
8415     for (my $i = 0; $i < $MAX_NETS; $i++) {
8416         my $netname = "net$i";
8417         next if !$conf->{$netname};
8418         push @ret, $netname;
8419         last;
8420     }
8421
8422     return \@ret;
8423 }
8424
8425 sub device_bootorder {
8426     my ($conf) = @_;
8427
8428     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8429
8430     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8431
8432     my $bootorder = {};
8433     if (!defined($boot) || $boot->{legacy}) {
8434         $bootorder = bootorder_from_legacy($conf, $boot);
8435     } elsif ($boot->{order}) {
8436         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8437         for my $dev (PVE::Tools::split_list($boot->{order})) {
8438             $bootorder->{$dev} = $i++;
8439         }
8440     }
8441
8442     return $bootorder;
8443 }
8444
8445 sub register_qmeventd_handle {
8446     my ($vmid) = @_;
8447
8448     my $fh;
8449     my $peer = "/var/run/qmeventd.sock";
8450     my $count = 0;
8451
8452     for (;;) {
8453         $count++;
8454         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8455         last if $fh;
8456         if ($! != EINTR && $! != EAGAIN) {
8457             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8458         }
8459         if ($count > 4) {
8460             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8461               . "after $count retries\n";
8462         }
8463         usleep(25000);
8464     }
8465
8466     # send handshake to mark VM as backing up
8467     print $fh to_json({vzdump => {vmid => "$vmid"}});
8468
8469     # return handle to be closed later when inhibit is no longer required
8470     return $fh;
8471 }
8472
8473 # bash completion helper
8474
8475 sub complete_backup_archives {
8476     my ($cmdname, $pname, $cvalue) = @_;
8477
8478     my $cfg = PVE::Storage::config();
8479
8480     my $storeid;
8481
8482     if ($cvalue =~ m/^([^:]+):/) {
8483         $storeid = $1;
8484     }
8485
8486     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8487
8488     my $res = [];
8489     foreach my $id (keys %$data) {
8490         foreach my $item (@{$data->{$id}}) {
8491             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8492             push @$res, $item->{volid} if defined($item->{volid});
8493         }
8494     }
8495
8496     return $res;
8497 }
8498
8499 my $complete_vmid_full = sub {
8500     my ($running) = @_;
8501
8502     my $idlist = vmstatus();
8503
8504     my $res = [];
8505
8506     foreach my $id (keys %$idlist) {
8507         my $d = $idlist->{$id};
8508         if (defined($running)) {
8509             next if $d->{template};
8510             next if $running && $d->{status} ne 'running';
8511             next if !$running && $d->{status} eq 'running';
8512         }
8513         push @$res, $id;
8514
8515     }
8516     return $res;
8517 };
8518
8519 sub complete_vmid {
8520     return &$complete_vmid_full();
8521 }
8522
8523 sub complete_vmid_stopped {
8524     return &$complete_vmid_full(0);
8525 }
8526
8527 sub complete_vmid_running {
8528     return &$complete_vmid_full(1);
8529 }
8530
8531 sub complete_storage {
8532
8533     my $cfg = PVE::Storage::config();
8534     my $ids = $cfg->{ids};
8535
8536     my $res = [];
8537     foreach my $sid (keys %$ids) {
8538         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8539         next if !$ids->{$sid}->{content}->{images};
8540         push @$res, $sid;
8541     }
8542
8543     return $res;
8544 }
8545
8546 sub complete_migration_storage {
8547     my ($cmd, $param, $current_value, $all_args) = @_;
8548
8549     my $targetnode = @$all_args[1];
8550
8551     my $cfg = PVE::Storage::config();
8552     my $ids = $cfg->{ids};
8553
8554     my $res = [];
8555     foreach my $sid (keys %$ids) {
8556         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8557         next if !$ids->{$sid}->{content}->{images};
8558         push @$res, $sid;
8559     }
8560
8561     return $res;
8562 }
8563
8564 sub vm_is_paused {
8565     my ($vmid, $include_suspended) = @_;
8566     my $qmpstatus = eval {
8567         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8568         mon_cmd($vmid, "query-status");
8569     };
8570     warn "$@\n" if $@;
8571     return $qmpstatus && (
8572         $qmpstatus->{status} eq "paused" ||
8573         $qmpstatus->{status} eq "prelaunch" ||
8574         ($include_suspended && $qmpstatus->{status} eq "suspended")
8575     );
8576 }
8577
8578 sub check_volume_storage_type {
8579     my ($storecfg, $vol) = @_;
8580
8581     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8582     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8583     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8584
8585     die "storage '$storeid' does not support content-type '$vtype'\n"
8586         if !$scfg->{content}->{$vtype};
8587
8588     return 1;
8589 }
8590
8591 sub add_nets_bridge_fdb {
8592     my ($conf, $vmid) = @_;
8593
8594     for my $opt (keys %$conf) {
8595         next if $opt !~ m/^net(\d+)$/;
8596         my $iface = "tap${vmid}i$1";
8597         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8598         my $net = parse_net($conf->{$opt}, 1) or next;
8599
8600         my $mac = $net->{macaddr};
8601         if (!$mac) {
8602             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8603                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8604             next;
8605         }
8606
8607         my $bridge = $net->{bridge};
8608         if (!$bridge) {
8609             log_warn("Interface '$iface' not attached to any bridge.");
8610             next;
8611         }
8612         if ($have_sdn) {
8613             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8614         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8615             PVE::Network::add_bridge_fdb($iface, $mac);
8616         }
8617     }
8618 }
8619
8620 sub del_nets_bridge_fdb {
8621     my ($conf, $vmid) = @_;
8622
8623     for my $opt (keys %$conf) {
8624         next if $opt !~ m/^net(\d+)$/;
8625         my $iface = "tap${vmid}i$1";
8626
8627         my $net = parse_net($conf->{$opt}) or next;
8628         my $mac = $net->{macaddr} or next;
8629
8630         my $bridge = $net->{bridge};
8631         if ($have_sdn) {
8632             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8633         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8634             PVE::Network::del_bridge_fdb($iface, $mac);
8635         }
8636     }
8637 }
8638
8639 sub create_ifaces_ipams_ips {
8640     my ($conf, $vmid) = @_;
8641
8642     return if !$have_sdn;
8643
8644     foreach my $opt (keys %$conf) {
8645         if ($opt =~ m/^net(\d+)$/) {
8646             my $value = $conf->{$opt};
8647             my $net = PVE::QemuServer::parse_net($value);
8648             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8649             warn $@ if $@;
8650         }
8651     }
8652 }
8653
8654 sub delete_ifaces_ipams_ips {
8655     my ($conf, $vmid) = @_;
8656
8657     return if !$have_sdn;
8658
8659     foreach my $opt (keys %$conf) {
8660         if ($opt =~ m/^net(\d+)$/) {
8661             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8662             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8663             warn $@ if $@;
8664         }
8665     }
8666 }
8667
8668 1;