PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::HA::Config;
  38 use PVE::Mapping::PCI;
  39 use PVE::Mapping::USB;
  40 use PVE::INotify;
  41 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  42 use PVE::ProcFSTools;
  43 use PVE::PBSClient;
  44 use PVE::RESTEnvironment qw(log_warn);
  45 use PVE::RPCEnvironment;
  46 use PVE::Storage;
  47 use PVE::SysFSTools;
  48 use PVE::Systemd;
  49 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  50
  51 use PVE::QMPClient;
  52 use PVE::QemuConfig;
  53 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  54 use PVE::QemuServer::Cloudinit;
  55 use PVE::QemuServer::CGroup;
  56 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
  57 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  58 use PVE::QemuServer::Machine;
  59 use PVE::QemuServer::Memory qw(get_current_memory);
  60 use PVE::QemuServer::Monitor qw(mon_cmd);
  61 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  62 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  63 use PVE::QemuServer::USB;
  64
  65 my $have_sdn;
  66 eval {
  67     require PVE::Network::SDN::Zones;
  68     require PVE::Network::SDN::Vnets;
  69     $have_sdn = 1;
  70 };
  71
  72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  73 my $OVMF = {
  74     x86_64 => {
  75         '4m-no-smm' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-no-smm-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         '4m' => [
  84             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  86         ],
  87         '4m-ms' => [
  88             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  89             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  90         ],
  91         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  92         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  93         # older backups and snapshot?
  94         default => [
  95             "$EDK2_FW_BASE/OVMF_CODE.fd",
  96             "$EDK2_FW_BASE/OVMF_VARS.fd",
  97         ],
  98     },
  99     aarch64 => {
 100         default => [
 101             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 102             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 103         ],
 104     },
 105 };
 106
 107 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 108
 109 # Note about locking: we use flock on the config file protect against concurent actions.
 110 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 112 # But you can ignore this kind of lock with the --skiplock flag.
 113
 114 cfs_register_file(
 115     '/qemu-server/',
 116     \&parse_vm_config,
 117     \&write_vm_config
 118 );
 119
 120 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 121     description => "Some command save/restore state from this location.",
 122     type => 'string',
 123     maxLength => 128,
 124     optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196     clipboard => {
 197         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 198             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 199         type => 'string',
 200         enum => ['vnc'],
 201         optional => 1,
 202     },
 203 };
 204
 205 my $ivshmem_fmt = {
 206     size => {
 207         type => 'integer',
 208         minimum => 1,
 209         description => "The size of the file in MB.",
 210     },
 211     name => {
 212         type => 'string',
 213         pattern => '[a-zA-Z0-9\-]+',
 214         optional => 1,
 215         format_description => 'string',
 216         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 217     },
 218 };
 219
 220 my $audio_fmt = {
 221     device => {
 222         type => 'string',
 223         enum => [qw(ich9-intel-hda intel-hda AC97)],
 224         description =>  "Configure an audio device."
 225     },
 226     driver =>  {
 227         type => 'string',
 228         enum => ['spice', 'none'],
 229         default => 'spice',
 230         optional => 1,
 231         description => "Driver backend for the audio device."
 232     },
 233 };
 234
 235 my $spice_enhancements_fmt = {
 236     foldersharing => {
 237         type => 'boolean',
 238         optional => 1,
 239         default => '0',
 240         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 241     },
 242     videostreaming =>  {
 243         type => 'string',
 244         enum => ['off', 'all', 'filter'],
 245         default => 'off',
 246         optional => 1,
 247         description => "Enable video streaming. Uses compression for detected video streams."
 248     },
 249 };
 250
 251 my $rng_fmt = {
 252     source => {
 253         type => 'string',
 254         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 255         default_key => 1,
 256         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 257             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 258             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 259             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 260             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 261             ." a hardware RNG from the host.",
 262     },
 263     max_bytes => {
 264         type => 'integer',
 265         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 266             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 267             ." `0` to disable limiting (potentially dangerous!).",
 268         optional => 1,
 269
 270         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 271         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 272         # reading from /dev/urandom
 273         default => 1024,
 274     },
 275     period => {
 276         type => 'integer',
 277         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 278             ." the guest to retrieve another 'max_bytes' of entropy.",
 279         optional => 1,
 280         default => 1000,
 281     },
 282 };
 283
 284 my $meta_info_fmt = {
 285     'ctime' => {
 286         type => 'integer',
 287         description => "The guest creation timestamp as UNIX epoch time",
 288         minimum => 0,
 289         optional => 1,
 290     },
 291     'creation-qemu' => {
 292         type => 'string',
 293         description => "The QEMU (machine) version from the time this VM was created.",
 294         pattern => '\d+(\.\d+)+',
 295         optional => 1,
 296     },
 297 };
 298
 299 my $confdesc = {
 300     onboot => {
 301         optional => 1,
 302         type => 'boolean',
 303         description => "Specifies whether a VM will be started during system bootup.",
 304         default => 0,
 305     },
 306     autostart => {
 307         optional => 1,
 308         type => 'boolean',
 309         description => "Automatic restart after crash (currently ignored).",
 310         default => 0,
 311     },
 312     hotplug => {
 313         optional => 1,
 314         type => 'string', format => 'pve-hotplug-features',
 315         description => "Selectively enable hotplug features. This is a comma separated list of"
 316             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 317             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 318             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 319             ." windows > 7.",
 320         default => 'network,disk,usb',
 321     },
 322     reboot => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 326         default => 1,
 327     },
 328     lock => {
 329         optional => 1,
 330         type => 'string',
 331         description => "Lock/unlock the VM.",
 332         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 333     },
 334     cpulimit => {
 335         optional => 1,
 336         type => 'number',
 337         description => "Limit of CPU usage.",
 338         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 339             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 340         minimum => 0,
 341         maximum => 128,
 342         default => 0,
 343     },
 344     cpuunits => {
 345         optional => 1,
 346         type => 'integer',
 347         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 348         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 349             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 350             ." weights of all the other running VMs.",
 351         minimum => 1,
 352         maximum => 262144,
 353         default => 'cgroup v1: 1024, cgroup v2: 100',
 354     },
 355     memory => {
 356         optional => 1,
 357         type => 'string',
 358         description => "Memory properties.",
 359         format => $PVE::QemuServer::Memory::memory_fmt
 360     },
 361     balloon => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 365         minimum => 0,
 366     },
 367     shares => {
 368         optional => 1,
 369         type => 'integer',
 370         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 371             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 372             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 373         minimum => 0,
 374         maximum => 50000,
 375         default => 1000,
 376     },
 377     keyboard => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Keyboard layout for VNC server. This option is generally not required and"
 381         ." is often better handled from within the guest OS.",
 382         enum => PVE::Tools::kvmkeymaplist(),
 383         default => undef,
 384     },
 385     name => {
 386         optional => 1,
 387         type => 'string', format => 'dns-name',
 388         description => "Set a name for the VM. Only used on the configuration web interface.",
 389     },
 390     scsihw => {
 391         optional => 1,
 392         type => 'string',
 393         description => "SCSI controller model",
 394         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 395         default => 'lsi',
 396     },
 397     description => {
 398         optional => 1,
 399         type => 'string',
 400         description => "Description for the VM. Shown in the web-interface VM's summary."
 401             ." This is saved as comment inside the configuration file.",
 402         maxLength => 1024 * 8,
 403     },
 404     ostype => {
 405         optional => 1,
 406         type => 'string',
 407         # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`.
 408         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 409         description => "Specify guest operating system.",
 410         verbose_description => <<EODESC,
 411 Specify guest operating system. This is used to enable special
 412 optimization/features for specific operating systems:
 413
 414 [horizontal]
 415 other;; unspecified OS
 416 wxp;; Microsoft Windows XP
 417 w2k;; Microsoft Windows 2000
 418 w2k3;; Microsoft Windows 2003
 419 w2k8;; Microsoft Windows 2008
 420 wvista;; Microsoft Windows Vista
 421 win7;; Microsoft Windows 7
 422 win8;; Microsoft Windows 8/2012/2012r2
 423 win10;; Microsoft Windows 10/2016/2019
 424 win11;; Microsoft Windows 11/2022/2025
 425 l24;; Linux 2.4 Kernel
 426 l26;; Linux 2.6 - 6.X Kernel
 427 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 428 EODESC
 429     },
 430     boot => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-boot',
 433         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 434             ." key or 'legacy=' is deprecated.",
 435     },
 436     bootdisk => {
 437         optional => 1,
 438         type => 'string', format => 'pve-qm-bootdisk',
 439         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 440         pattern => '(ide|sata|scsi|virtio)\d+',
 441     },
 442     smp => {
 443         optional => 1,
 444         type => 'integer',
 445         description => "The number of CPUs. Please use option -sockets instead.",
 446         minimum => 1,
 447         default => 1,
 448     },
 449     sockets => {
 450         optional => 1,
 451         type => 'integer',
 452         description => "The number of CPU sockets.",
 453         minimum => 1,
 454         default => 1,
 455     },
 456     cores => {
 457         optional => 1,
 458         type => 'integer',
 459         description => "The number of cores per socket.",
 460         minimum => 1,
 461         default => 1,
 462     },
 463     numa => {
 464         optional => 1,
 465         type => 'boolean',
 466         description => "Enable/disable NUMA.",
 467         default => 0,
 468     },
 469     hugepages => {
 470         optional => 1,
 471         type => 'string',
 472         description => "Enable/disable hugepages memory.",
 473         enum => [qw(any 2 1024)],
 474     },
 475     keephugepages => {
 476         optional => 1,
 477         type => 'boolean',
 478         default => 0,
 479         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 480             ." after VM shutdown and can be used for subsequent starts.",
 481     },
 482     vcpus => {
 483         optional => 1,
 484         type => 'integer',
 485         description => "Number of hotplugged vcpus.",
 486         minimum => 1,
 487         default => 0,
 488     },
 489     acpi => {
 490         optional => 1,
 491         type => 'boolean',
 492         description => "Enable/disable ACPI.",
 493         default => 1,
 494     },
 495     agent => {
 496         optional => 1,
 497         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 498         type => 'string',
 499         format => $agent_fmt,
 500     },
 501     kvm => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable KVM hardware virtualization.",
 505         default => 1,
 506     },
 507     tdf => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Enable/disable time drift fix.",
 511         default => 0,
 512     },
 513     localtime => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 517             ." the `ostype` indicates a Microsoft Windows OS.",
 518     },
 519     freeze => {
 520         optional => 1,
 521         type => 'boolean',
 522         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 523     },
 524     vga => {
 525         optional => 1,
 526         type => 'string', format => $vga_fmt,
 527         description => "Configure the VGA hardware.",
 528         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 529             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 530             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 531             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 532             ." display server. For win* OS you can select how many independent displays you want,"
 533             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 534             ." using a serial device as terminal.",
 535     },
 536     watchdog => {
 537         optional => 1,
 538         type => 'string', format => 'pve-qm-watchdog',
 539         description => "Create a virtual hardware watchdog device.",
 540         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 541             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 542             ." else the watchdog will reset the guest (or execute the respective action specified)",
 543     },
 544     startdate => {
 545         optional => 1,
 546         type => 'string',
 547         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 548         description => "Set the initial date of the real time clock. Valid format for date are:"
 549             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 550         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 551         default => 'now',
 552     },
 553     startup =>  get_standard_option('pve-startup-order'),
 554     template => {
 555         optional => 1,
 556         type => 'boolean',
 557         description => "Enable/disable Template.",
 558         default => 0,
 559     },
 560     args => {
 561         optional => 1,
 562         type => 'string',
 563         description => "Arbitrary arguments passed to kvm.",
 564         verbose_description => <<EODESCR,
 565 Arbitrary arguments passed to kvm, for example:
 566
 567 args: -no-reboot -smbios 'type=0,vendor=FOO'
 568
 569 NOTE: this option is for experts only.
 570 EODESCR
 571     },
 572     tablet => {
 573         optional => 1,
 574         type => 'boolean',
 575         default => 1,
 576         description => "Enable/disable the USB tablet device.",
 577         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 578             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 579             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 580             ." may consider disabling this to save some context switches. This is turned off by"
 581             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 582     },
 583     migrate_speed => {
 584         optional => 1,
 585         type => 'integer',
 586         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 587         minimum => 0,
 588         default => 0,
 589     },
 590     migrate_downtime => {
 591         optional => 1,
 592         type => 'number',
 593         description => "Set maximum tolerated downtime (in seconds) for migrations. Should the"
 594             ." migration not be able to converge in the very end, because too much newly dirtied"
 595             ." RAM needs to be transferred, the limit will be increased automatically step-by-step"
 596             ." until migration can converge.",
 597         minimum => 0,
 598         default => 0.1,
 599     },
 600     cdrom => {
 601         optional => 1,
 602         type => 'string', format => 'pve-qm-ide',
 603         typetext => '<volume>',
 604         description => "This is an alias for option -ide2",
 605     },
 606     cpu => {
 607         optional => 1,
 608         description => "Emulated CPU type.",
 609         type => 'string',
 610         format => 'pve-vm-cpu-conf',
 611     },
 612     parent => get_standard_option('pve-snapshot-name', {
 613         optional => 1,
 614         description => "Parent snapshot name. This is used internally, and should not be modified.",
 615     }),
 616     snaptime => {
 617         optional => 1,
 618         description => "Timestamp for snapshots.",
 619         type => 'integer',
 620         minimum => 0,
 621     },
 622     vmstate => {
 623         optional => 1,
 624         type => 'string', format => 'pve-volume-id',
 625         description => "Reference to a volume which stores the VM state. This is used internally"
 626             ." for snapshots.",
 627     },
 628     vmstatestorage => get_standard_option('pve-storage-id', {
 629         description => "Default storage for VM state volumes/files.",
 630         optional => 1,
 631     }),
 632     runningmachine => get_standard_option('pve-qemu-machine', {
 633         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 634             ." for snapshots.",
 635     }),
 636     runningcpu => {
 637         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 638             ." internally for snapshots.",
 639         optional => 1,
 640         type => 'string',
 641         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 642         format_description => 'QEMU -cpu parameter'
 643     },
 644     machine => get_standard_option('pve-qemu-machine'),
 645     arch => {
 646         description => "Virtual processor architecture. Defaults to the host.",
 647         optional => 1,
 648         type => 'string',
 649         enum => [qw(x86_64 aarch64)],
 650     },
 651     smbios1 => {
 652         description => "Specify SMBIOS type 1 fields.",
 653         type => 'string', format => 'pve-qm-smbios1',
 654         maxLength => 512,
 655         optional => 1,
 656     },
 657     protection => {
 658         optional => 1,
 659         type => 'boolean',
 660         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 661             ." remove disk operations.",
 662         default => 0,
 663     },
 664     bios => {
 665         optional => 1,
 666         type => 'string',
 667         enum => [ qw(seabios ovmf) ],
 668         description => "Select BIOS implementation.",
 669         default => 'seabios',
 670     },
 671     vmgenid => {
 672         type => 'string',
 673         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 674         format_description => 'UUID',
 675         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 676             ." to disable explicitly.",
 677         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 678             ." value identifier to the guest OS. This allows to notify the guest operating system"
 679             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 680             ." execution or creation from a template). The guest operating system notices the"
 681             ." change, and is then able to react as appropriate by marking its copies of"
 682             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 683             ."Note that auto-creation only works when done through API/CLI create or update methods"
 684             .", but not when manually editing the config file.",
 685         default => "1 (autogenerated)",
 686         optional => 1,
 687     },
 688     hookscript => {
 689         type => 'string',
 690         format => 'pve-volume-id',
 691         optional => 1,
 692         description => "Script that will be executed during various steps in the vms lifetime.",
 693     },
 694     ivshmem => {
 695         type => 'string',
 696         format => $ivshmem_fmt,
 697         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 698             ." the host.",
 699         optional => 1,
 700     },
 701     audio0 => {
 702         type => 'string',
 703         format => $audio_fmt,
 704         description => "Configure a audio device, useful in combination with QXL/Spice.",
 705         optional => 1
 706     },
 707     spice_enhancements => {
 708         type => 'string',
 709         format => $spice_enhancements_fmt,
 710         description => "Configure additional enhancements for SPICE.",
 711         optional => 1
 712     },
 713     tags => {
 714         type => 'string', format => 'pve-tag-list',
 715         description => 'Tags of the VM. This is only meta information.',
 716         optional => 1,
 717     },
 718     rng0 => {
 719         type => 'string',
 720         format => $rng_fmt,
 721         description => "Configure a VirtIO-based Random Number Generator.",
 722         optional => 1,
 723     },
 724     meta => {
 725         type => 'string',
 726         format => $meta_info_fmt,
 727         description => "Some (read-only) meta-information about this guest.",
 728         optional => 1,
 729     },
 730     affinity => {
 731         type => 'string', format => 'pve-cpuset',
 732         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 733         optional => 1,
 734     },
 735 };
 736
 737 my $cicustom_fmt = {
 738     meta => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all meta data passed to the VM via"
 742             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     network => {
 747         type => 'string',
 748         optional => 1,
 749         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 750         format => 'pve-volume-id',
 751         format_description => 'volume',
 752     },
 753     user => {
 754         type => 'string',
 755         optional => 1,
 756         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 757         format => 'pve-volume-id',
 758         format_description => 'volume',
 759     },
 760     vendor => {
 761         type => 'string',
 762         optional => 1,
 763         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 764         format => 'pve-volume-id',
 765         format_description => 'volume',
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 769
 770 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 771 my $confdesc_cloudinit = {
 772     citype => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'Specifies the cloud-init configuration format. The default depends on the'
 776             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 777             .' and `configdrive2` for windows.',
 778         enum => ['configdrive2', 'nocloud', 'opennebula'],
 779     },
 780     ciuser => {
 781         optional => 1,
 782         type => 'string',
 783         description => "cloud-init: User name to change ssh keys and password for instead of the"
 784             ." image's configured default user.",
 785     },
 786     cipassword => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Password to assign the user. Using this is generally not'
 790             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 791             .' support hashed passwords.',
 792     },
 793     ciupgrade => {
 794         optional => 1,
 795         type => 'boolean',
 796         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 797         default => 1,
 798     },
 799     cicustom => {
 800         optional => 1,
 801         type => 'string',
 802         description => 'cloud-init: Specify custom files to replace the automatically generated'
 803             .' ones at start.',
 804         format => 'pve-qm-cicustom',
 805     },
 806     searchdomain => {
 807         optional => 1,
 808         type => 'string',
 809         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 810             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 811             .' are set.',
 812     },
 813     nameserver => {
 814         optional => 1,
 815         type => 'string', format => 'address-list',
 816         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 817             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 818             .' are set.',
 819     },
 820     sshkeys => {
 821         optional => 1,
 822         type => 'string',
 823         format => 'urlencoded',
 824         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 825     },
 826 };
 827
 828 # what about other qemu settings ?
 829 #cpu => 'string',
 830 #machine => 'string',
 831 #fda => 'file',
 832 #fdb => 'file',
 833 #mtdblock => 'file',
 834 #sd => 'file',
 835 #pflash => 'file',
 836 #snapshot => 'bool',
 837 #bootp => 'file',
 838 ##tftp => 'dir',
 839 ##smb => 'dir',
 840 #kernel => 'file',
 841 #append => 'string',
 842 #initrd => 'file',
 843 ##soundhw => 'string',
 844
 845 while (my ($k, $v) = each %$confdesc) {
 846     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 847 }
 848
 849 my $MAX_NETS = 32;
 850 my $MAX_SERIAL_PORTS = 4;
 851 my $MAX_PARALLEL_PORTS = 3;
 852
 853 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 854     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 855 }
 856
 857 my $nic_model_list = [
 858     'e1000',
 859     'e1000-82540em',
 860     'e1000-82544gc',
 861     'e1000-82545em',
 862     'e1000e',
 863     'i82551',
 864     'i82557b',
 865     'i82559er',
 866     'ne2k_isa',
 867     'ne2k_pci',
 868     'pcnet',
 869     'rtl8139',
 870     'virtio',
 871     'vmxnet3',
 872 ];
 873 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 874
 875 my $net_fmt_bridge_descr = <<__EOD__;
 876 Bridge to attach the network device to. The Proxmox VE standard bridge
 877 is called 'vmbr0'.
 878
 879 If you do not specify a bridge, we create a kvm user (NATed) network
 880 device, which provides DHCP and DNS services. The following addresses
 881 are used:
 882
 883  10.0.2.2   Gateway
 884  10.0.2.3   DNS Server
 885  10.0.2.4   SMB Server
 886
 887 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 888 __EOD__
 889
 890 my $net_fmt = {
 891     macaddr  => get_standard_option('mac-addr', {
 892         description => "MAC address. That address must be unique withing your network. This is"
 893             ." automatically generated if not specified.",
 894     }),
 895     model => {
 896         type => 'string',
 897         description => "Network Card Model. The 'virtio' model provides the best performance with"
 898             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 899             ." best to use 'e1000'.",
 900         enum => $nic_model_list,
 901         default_key => 1,
 902     },
 903     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 904     bridge => get_standard_option('pve-bridge-id', {
 905         description => $net_fmt_bridge_descr,
 906         optional => 1,
 907     }),
 908     queues => {
 909         type => 'integer',
 910         minimum => 0, maximum => 64,
 911         description => 'Number of packet queues to be used on the device.',
 912         optional => 1,
 913     },
 914     rate => {
 915         type => 'number',
 916         minimum => 0,
 917         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 918         optional => 1,
 919     },
 920     tag => {
 921         type => 'integer',
 922         minimum => 1, maximum => 4094,
 923         description => 'VLAN tag to apply to packets on this interface.',
 924         optional => 1,
 925     },
 926     trunks => {
 927         type => 'string',
 928         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 929         description => 'VLAN trunks to pass through this interface.',
 930         format_description => 'vlanid[;vlanid...]',
 931         optional => 1,
 932     },
 933     firewall => {
 934         type => 'boolean',
 935         description => 'Whether this interface should be protected by the firewall.',
 936         optional => 1,
 937     },
 938     link_down => {
 939         type => 'boolean',
 940         description => 'Whether this interface should be disconnected (like pulling the plug).',
 941         optional => 1,
 942     },
 943     mtu => {
 944         type => 'integer',
 945         minimum => 1, maximum => 65520,
 946         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 947         optional => 1,
 948     },
 949 };
 950
 951 my $netdesc = {
 952     optional => 1,
 953     type => 'string', format => $net_fmt,
 954     description => "Specify network devices.",
 955 };
 956
 957 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 958
 959 my $ipconfig_fmt = {
 960     ip => {
 961         type => 'string',
 962         format => 'pve-ipv4-config',
 963         format_description => 'IPv4Format/CIDR',
 964         description => 'IPv4 address in CIDR format.',
 965         optional => 1,
 966         default => 'dhcp',
 967     },
 968     gw => {
 969         type => 'string',
 970         format => 'ipv4',
 971         format_description => 'GatewayIPv4',
 972         description => 'Default gateway for IPv4 traffic.',
 973         optional => 1,
 974         requires => 'ip',
 975     },
 976     ip6 => {
 977         type => 'string',
 978         format => 'pve-ipv6-config',
 979         format_description => 'IPv6Format/CIDR',
 980         description => 'IPv6 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw6 => {
 985         type => 'string',
 986         format => 'ipv6',
 987         format_description => 'GatewayIPv6',
 988         description => 'Default gateway for IPv6 traffic.',
 989         optional => 1,
 990         requires => 'ip6',
 991     },
 992 };
 993 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 994 my $ipconfigdesc = {
 995     optional => 1,
 996     type => 'string', format => 'pve-qm-ipconfig',
 997     description => <<'EODESCR',
 998 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 999
1000 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1001
1002 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1003 gateway should be provided.
1004 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1005 cloud-init 19.4 or newer.
1006
1007 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1008 dhcp on IPv4.
1009 EODESCR
1010 };
1011 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1012
1013 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1014     $confdesc->{"net$i"} = $netdesc;
1015     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1016 }
1017
1018 foreach my $key (keys %$confdesc_cloudinit) {
1019     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1020 }
1021
1022 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1023 sub pve_verify_cpuset {
1024     my ($set_text, $noerr) = @_;
1025
1026     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1027
1028     if ($@) {
1029         return if $noerr;
1030         die "unable to parse cpuset option\n";
1031     }
1032
1033     return PVE::CpuSet->new($members)->short_string();
1034 }
1035
1036 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1037 sub verify_volume_id_or_qm_path {
1038     my ($volid, $noerr) = @_;
1039
1040     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1041
1042     return verify_volume_id_or_absolute_path($volid, $noerr);
1043 }
1044
1045 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1046 sub verify_volume_id_or_absolute_path {
1047     my ($volid, $noerr) = @_;
1048
1049     return $volid if $volid =~ m|^/|;
1050
1051     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1052     if ($@) {
1053         return if $noerr;
1054         die $@;
1055     }
1056     return $volid;
1057 }
1058
1059 my $serialdesc = {
1060         optional => 1,
1061         type => 'string',
1062         pattern => '(/dev/.+|socket)',
1063         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1064         verbose_description =>  <<EODESCR,
1065 Create a serial device inside the VM (n is 0 to 3), and pass through a
1066 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1067 host side (use 'qm terminal' to open a terminal connection).
1068
1069 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1070 use with special care.
1071
1072 CAUTION: Experimental! User reported problems with this option.
1073 EODESCR
1074 };
1075
1076 my $paralleldesc= {
1077         optional => 1,
1078         type => 'string',
1079         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1080         description =>  "Map host parallel devices (n is 0 to 2).",
1081         verbose_description =>  <<EODESCR,
1082 Map host parallel devices (n is 0 to 2).
1083
1084 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085 machines - use with special care.
1086
1087 CAUTION: Experimental! User reported problems with this option.
1088 EODESCR
1089 };
1090
1091 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1092     $confdesc->{"parallel$i"} = $paralleldesc;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1096     $confdesc->{"serial$i"} = $serialdesc;
1097 }
1098
1099 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1100     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1101 }
1102
1103 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1104     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1105 }
1106
1107 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1108     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1109 }
1110
1111 my $boot_fmt = {
1112     legacy => {
1113         optional => 1,
1114         default_key => 1,
1115         type => 'string',
1116         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1117                      . " Deprecated, use 'order=' instead.",
1118         pattern => '[acdn]{1,4}',
1119         format_description => "[acdn]{1,4}",
1120
1121         # note: this is also the fallback if boot: is not given at all
1122         default => 'cdn',
1123     },
1124     order => {
1125         optional => 1,
1126         type => 'string',
1127         format => 'pve-qm-bootdev-list',
1128         format_description => "device[;device...]",
1129         description => <<EODESC,
1130 The guest will attempt to boot from devices in the order they appear here.
1131
1132 Disks, optical drives and passed-through storage USB devices will be directly
1133 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1134 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1135
1136 Note that only devices in this list will be marked as bootable and thus loaded
1137 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1138 (e.g. software-raid), you need to specify all of them here.
1139
1140 Overrides the deprecated 'legacy=[acdn]*' value when given.
1141 EODESC
1142     },
1143 };
1144 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1145
1146 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1147 sub verify_bootdev {
1148     my ($dev, $noerr) = @_;
1149
1150     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1151     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1152
1153     my $check = sub {
1154         my ($base) = @_;
1155         return 0 if $dev !~ m/^$base\d+$/;
1156         return 0 if !$confdesc->{$dev};
1157         return 1;
1158     };
1159
1160     return $dev if $check->("net");
1161     return $dev if $check->("usb");
1162     return $dev if $check->("hostpci");
1163
1164     return if $noerr;
1165     die "invalid boot device '$dev'\n";
1166 }
1167
1168 sub print_bootorder {
1169     my ($devs) = @_;
1170     return "" if !@$devs;
1171     my $data = { order => join(';', @$devs) };
1172     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1173 }
1174
1175 my $kvm_api_version = 0;
1176
1177 sub kvm_version {
1178     return $kvm_api_version if $kvm_api_version;
1179
1180     open my $fh, '<', '/dev/kvm' or return;
1181
1182     # 0xae00 => KVM_GET_API_VERSION
1183     $kvm_api_version = ioctl($fh, 0xae00, 0);
1184     close($fh);
1185
1186     return $kvm_api_version;
1187 }
1188
1189 my $kvm_user_version = {};
1190 my $kvm_mtime = {};
1191
1192 sub kvm_user_version {
1193     my ($binary) = @_;
1194
1195     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1196     my $st = stat($binary);
1197
1198     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1199     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1200         $cachedmtime == $st->mtime;
1201
1202     $kvm_user_version->{$binary} = 'unknown';
1203     $kvm_mtime->{$binary} = $st->mtime;
1204
1205     my $code = sub {
1206         my $line = shift;
1207         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1208             $kvm_user_version->{$binary} = $2;
1209         }
1210     };
1211
1212     eval { run_command([$binary, '--version'], outfunc => $code); };
1213     warn $@ if $@;
1214
1215     return $kvm_user_version->{$binary};
1216
1217 }
1218 my sub extract_version {
1219     my ($machine_type, $version) = @_;
1220     $version = kvm_user_version() if !defined($version);
1221     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1222 }
1223
1224 sub kernel_has_vhost_net {
1225     return -c '/dev/vhost-net';
1226 }
1227
1228 sub option_exists {
1229     my $key = shift;
1230     return defined($confdesc->{$key});
1231 }
1232
1233 my $cdrom_path;
1234 sub get_cdrom_path {
1235
1236     return $cdrom_path if defined($cdrom_path);
1237
1238     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1239
1240     if (!defined($cdrom_path)) {
1241         log_warn("no physical CD-ROM available, ignoring");
1242         $cdrom_path = '';
1243     }
1244
1245     return $cdrom_path;
1246 }
1247
1248 sub get_iso_path {
1249     my ($storecfg, $vmid, $cdrom) = @_;
1250
1251     if ($cdrom eq 'cdrom') {
1252         return get_cdrom_path();
1253     } elsif ($cdrom eq 'none') {
1254         return '';
1255     } elsif ($cdrom =~ m|^/|) {
1256         return $cdrom;
1257     } else {
1258         return PVE::Storage::path($storecfg, $cdrom);
1259     }
1260 }
1261
1262 # try to convert old style file names to volume IDs
1263 sub filename_to_volume_id {
1264     my ($vmid, $file, $media) = @_;
1265
1266      if (!($file eq 'none' || $file eq 'cdrom' ||
1267           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1268
1269         return if $file =~ m|/|;
1270
1271         if ($media && $media eq 'cdrom') {
1272             $file = "local:iso/$file";
1273         } else {
1274             $file = "local:$vmid/$file";
1275         }
1276     }
1277
1278     return $file;
1279 }
1280
1281 sub verify_media_type {
1282     my ($opt, $vtype, $media) = @_;
1283
1284     return if !$media;
1285
1286     my $etype;
1287     if ($media eq 'disk') {
1288         $etype = 'images';
1289     } elsif ($media eq 'cdrom') {
1290         $etype = 'iso';
1291     } else {
1292         die "internal error";
1293     }
1294
1295     return if ($vtype eq $etype);
1296
1297     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1298 }
1299
1300 sub cleanup_drive_path {
1301     my ($opt, $storecfg, $drive) = @_;
1302
1303     # try to convert filesystem paths to volume IDs
1304
1305     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1306         ($drive->{file} !~ m|^/dev/.+|) &&
1307         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1308         ($drive->{file} !~ m/^\d+$/)) {
1309         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1310         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1311             if !$vtype;
1312         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1313         verify_media_type($opt, $vtype, $drive->{media});
1314         $drive->{file} = $volid;
1315     }
1316
1317     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1318 }
1319
1320 sub parse_hotplug_features {
1321     my ($data) = @_;
1322
1323     my $res = {};
1324
1325     return $res if $data eq '0';
1326
1327     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1328
1329     foreach my $feature (PVE::Tools::split_list($data)) {
1330         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1331             $res->{$1} = 1;
1332         } else {
1333             die "invalid hotplug feature '$feature'\n";
1334         }
1335     }
1336     return $res;
1337 }
1338
1339 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1340 sub pve_verify_hotplug_features {
1341     my ($value, $noerr) = @_;
1342
1343     return $value if parse_hotplug_features($value);
1344
1345     return if $noerr;
1346
1347     die "unable to parse hotplug option\n";
1348 }
1349
1350 sub assert_clipboard_config {
1351     my ($vga) = @_;
1352
1353     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1354
1355     if (
1356         $vga->{'clipboard'}
1357         && $vga->{'clipboard'} eq 'vnc'
1358         && $vga->{type}
1359         && $vga->{type} !~ $clipboard_regex
1360     ) {
1361         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1362     }
1363 }
1364
1365 sub print_tabletdevice_full {
1366     my ($conf, $arch) = @_;
1367
1368     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1369
1370     # we use uhci for old VMs because tablet driver was buggy in older qemu
1371     my $usbbus;
1372     if ($q35 || $arch eq 'aarch64') {
1373         $usbbus = 'ehci';
1374     } else {
1375         $usbbus = 'uhci';
1376     }
1377
1378     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1379 }
1380
1381 sub print_keyboarddevice_full {
1382     my ($conf, $arch) = @_;
1383
1384     return if $arch ne 'aarch64';
1385
1386     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1387 }
1388
1389 my sub get_drive_id {
1390     my ($drive) = @_;
1391     return "$drive->{interface}$drive->{index}";
1392 }
1393
1394 sub print_drivedevice_full {
1395     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1396
1397     my $device = '';
1398     my $maxdev = 0;
1399
1400     my $drive_id = get_drive_id($drive);
1401     if ($drive->{interface} eq 'virtio') {
1402         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1403         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1404         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1405     } elsif ($drive->{interface} eq 'scsi') {
1406
1407         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1408         my $unit = $drive->{index} % $maxdev;
1409
1410         my $machine_version = extract_version($machine_type, kvm_user_version());
1411         my $device_type = PVE::QemuServer::Drive::get_scsi_device_type(
1412             $drive, $storecfg, $machine_version);
1413
1414         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1415             $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit";
1416         } else {
1417             $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1418                 .",lun=$drive->{index}";
1419         }
1420         $device .= ",drive=drive-$drive_id,id=$drive_id";
1421
1422         if ($drive->{ssd} && ($device_type eq 'block' || $device_type eq 'hd')) {
1423             $device .= ",rotation_rate=1";
1424         }
1425         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1426
1427         # only scsi-hd and scsi-cd support passing vendor and product information
1428         if ($device_type eq 'hd' || $device_type eq 'cd') {
1429             if (my $vendor = $drive->{vendor}) {
1430                 $device .= ",vendor=$vendor";
1431             }
1432             if (my $product = $drive->{product}) {
1433                 $device .= ",product=$product";
1434             }
1435         }
1436
1437     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1438         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1439         my $controller = int($drive->{index} / $maxdev);
1440         my $unit = $drive->{index} % $maxdev;
1441
1442         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1443         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1444         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1445         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1446         # were before. Move odd ones up by 2 where they don't clash.
1447         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1448             $controller += 2 * ($unit % 2);
1449             $unit = 0;
1450         }
1451
1452         my $device_type = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1453
1454         $device = "ide-$device_type";
1455         if ($drive->{interface} eq 'ide') {
1456             $device .= ",bus=ide.$controller,unit=$unit";
1457         } else {
1458             $device .= ",bus=ahci$controller.$unit";
1459         }
1460         $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462         if ($device_type eq 'hd') {
1463             if (my $model = $drive->{model}) {
1464                 $model = URI::Escape::uri_unescape($model);
1465                 $device .= ",model=$model";
1466             }
1467             if ($drive->{ssd}) {
1468                 $device .= ",rotation_rate=1";
1469             }
1470         }
1471         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1472     } elsif ($drive->{interface} eq 'usb') {
1473         die "implement me";
1474         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1475     } else {
1476         die "unsupported interface type";
1477     }
1478
1479     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1480
1481     if (my $serial = $drive->{serial}) {
1482         $serial = URI::Escape::uri_unescape($serial);
1483         $device .= ",serial=$serial";
1484     }
1485
1486
1487     return $device;
1488 }
1489
1490 sub get_initiator_name {
1491     my $initiator;
1492
1493     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1494     while (defined(my $line = <$fh>)) {
1495         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1496         $initiator = $1;
1497         last;
1498     }
1499     $fh->close();
1500
1501     return $initiator;
1502 }
1503
1504 my sub storage_allows_io_uring_default {
1505     my ($scfg, $cache_direct) = @_;
1506
1507     # io_uring with cache mode writeback or writethrough on krbd will hang...
1508     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1509
1510     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1511     # sometimes, just plain disable...
1512     return if $scfg && $scfg->{type} eq 'lvm';
1513
1514     # io_uring causes problems when used with CIFS since kernel 5.15
1515     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1516     return if $scfg && $scfg->{type} eq 'cifs';
1517
1518     return 1;
1519 }
1520
1521 my sub drive_uses_cache_direct {
1522     my ($drive, $scfg) = @_;
1523
1524     my $cache_direct = 0;
1525
1526     if (my $cache = $drive->{cache}) {
1527         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1528     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1529         $cache_direct = 1;
1530     }
1531
1532     return $cache_direct;
1533 }
1534
1535 sub print_drive_commandline_full {
1536     my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
1537
1538     my $path;
1539     my $volid = $drive->{file};
1540     my $format = $drive->{format};
1541     my $drive_id = get_drive_id($drive);
1542
1543     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1544     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1545
1546     if (drive_is_cdrom($drive)) {
1547         $path = get_iso_path($storecfg, $vmid, $volid);
1548         die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
1549     } else {
1550         if ($storeid) {
1551             $path = PVE::Storage::path($storecfg, $volid);
1552             $format //= qemu_img_format($scfg, $volname);
1553         } else {
1554             $path = $volid;
1555             $format //= "raw";
1556         }
1557    }
1558
1559    my $is_rbd = $path =~ m/^rbd:/;
1560
1561     my $opts = '';
1562     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1563     foreach my $o (@qemu_drive_options) {
1564         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1565     }
1566
1567     # snapshot only accepts on|off
1568     if (defined($drive->{snapshot})) {
1569         my $v = $drive->{snapshot} ? 'on' : 'off';
1570         $opts .= ",snapshot=$v";
1571     }
1572
1573     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1574         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1575     }
1576
1577     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1578         my ($dir, $qmpname) = @$type;
1579         if (my $v = $drive->{"mbps$dir"}) {
1580             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1581         }
1582         if (my $v = $drive->{"mbps${dir}_max"}) {
1583             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1584         }
1585         if (my $v = $drive->{"bps${dir}_max_length"}) {
1586             $opts .= ",throttling.bps$qmpname-max-length=$v";
1587         }
1588         if (my $v = $drive->{"iops${dir}"}) {
1589             $opts .= ",throttling.iops$qmpname=$v";
1590         }
1591         if (my $v = $drive->{"iops${dir}_max"}) {
1592             $opts .= ",throttling.iops$qmpname-max=$v";
1593         }
1594         if (my $v = $drive->{"iops${dir}_max_length"}) {
1595             $opts .= ",throttling.iops$qmpname-max-length=$v";
1596         }
1597     }
1598
1599     if ($live_restore_name) {
1600         $format = "rbd" if $is_rbd;
1601         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1602             if !$format;
1603         $opts .= ",format=alloc-track,file.driver=$format";
1604     } elsif ($format) {
1605         $opts .= ",format=$format";
1606     }
1607
1608     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1609
1610     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1611
1612     if (!$drive->{aio}) {
1613         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1614             # io_uring supports all cache modes
1615             $opts .= ",aio=io_uring";
1616         } else {
1617             # aio native works only with O_DIRECT
1618             if($cache_direct) {
1619                 $opts .= ",aio=native";
1620             } else {
1621                 $opts .= ",aio=threads";
1622             }
1623         }
1624     }
1625
1626     if (!drive_is_cdrom($drive)) {
1627         my $detectzeroes;
1628         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1629             $detectzeroes = 'off';
1630         } elsif ($drive->{discard}) {
1631             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1632         } else {
1633             # This used to be our default with discard not being specified:
1634             $detectzeroes = 'on';
1635         }
1636
1637         # note: 'detect-zeroes' works per blockdev and we want it to persist
1638         # after the alloc-track is removed, so put it on 'file' directly
1639         my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes";
1640         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1641     }
1642
1643     if ($live_restore_name) {
1644         $opts .= ",backing=$live_restore_name";
1645         $opts .= ",auto-remove=on";
1646     }
1647
1648     # my $file_param = $live_restore_name ? "file.file.filename" : "file";
1649     my $file_param = "file";
1650     if ($live_restore_name) {
1651         # non-rbd drivers require the underlying file to be a seperate block
1652         # node, so add a second .file indirection
1653         $file_param .= ".file" if !$is_rbd;
1654         $file_param .= ".filename";
1655     }
1656     my $pathinfo = $path ? "$file_param=$path," : '';
1657
1658     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1659 }
1660
1661 sub print_pbs_blockdev {
1662     my ($pbs_conf, $pbs_name) = @_;
1663     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1664     $blockdev .= ",repository=$pbs_conf->{repository}";
1665     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1666     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1667     $blockdev .= ",archive=$pbs_conf->{archive}";
1668     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1669     return $blockdev;
1670 }
1671
1672 sub print_netdevice_full {
1673     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1674
1675     my $device = $net->{model};
1676     if ($net->{model} eq 'virtio') {
1677          $device = 'virtio-net-pci';
1678      };
1679
1680     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1681     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1682     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1683         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1684         # and out of each queue plus one config interrupt and control vector queue
1685         my $vectors = $net->{queues} * 2 + 2;
1686         $tmpstr .= ",vectors=$vectors,mq=on";
1687         if (min_version($machine_version, 7, 1)) {
1688             $tmpstr .= ",packed=on";
1689         }
1690     }
1691
1692     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1693         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1694     }
1695
1696     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1697
1698     if (my $mtu = $net->{mtu}) {
1699         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1700             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1701             if ($mtu == 1) {
1702                  $mtu = $bridge_mtu;
1703             } elsif ($mtu < 576) {
1704                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1705             } elsif ($mtu > $bridge_mtu) {
1706                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1707             }
1708             $tmpstr .= ",host_mtu=$mtu";
1709         } else {
1710             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1711         }
1712     }
1713
1714     if ($use_old_bios_files) {
1715         my $romfile;
1716         if ($device eq 'virtio-net-pci') {
1717             $romfile = 'pxe-virtio.rom';
1718         } elsif ($device eq 'e1000') {
1719             $romfile = 'pxe-e1000.rom';
1720         } elsif ($device eq 'e1000e') {
1721             $romfile = 'pxe-e1000e.rom';
1722         } elsif ($device eq 'ne2k') {
1723             $romfile = 'pxe-ne2k_pci.rom';
1724         } elsif ($device eq 'pcnet') {
1725             $romfile = 'pxe-pcnet.rom';
1726         } elsif ($device eq 'rtl8139') {
1727             $romfile = 'pxe-rtl8139.rom';
1728         }
1729         $tmpstr .= ",romfile=$romfile" if $romfile;
1730     }
1731
1732     return $tmpstr;
1733 }
1734
1735 sub print_netdev_full {
1736     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1737
1738     my $i = '';
1739     if ($netid =~ m/^net(\d+)$/) {
1740         $i = int($1);
1741     }
1742
1743     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1744
1745     my $ifname = "tap${vmid}i$i";
1746
1747     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1748     die "interface name '$ifname' is too long (max 15 character)\n"
1749         if length($ifname) >= 16;
1750
1751     my $vhostparam = '';
1752     if (is_native_arch($arch)) {
1753         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1754     }
1755
1756     my $vmname = $conf->{name} || "vm$vmid";
1757
1758     my $netdev = "";
1759     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1760
1761     if ($net->{bridge}) {
1762         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1763             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1764     } else {
1765         $netdev = "type=user,id=$netid,hostname=$vmname";
1766     }
1767
1768     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1769
1770     return $netdev;
1771 }
1772
1773 my $vga_map = {
1774     'cirrus' => 'cirrus-vga',
1775     'std' => 'VGA',
1776     'vmware' => 'vmware-svga',
1777     'virtio' => 'virtio-vga',
1778     'virtio-gl' => 'virtio-vga-gl',
1779 };
1780
1781 sub print_vga_device {
1782     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1783
1784     my $type = $vga_map->{$vga->{type}};
1785     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1786         $type = 'virtio-gpu';
1787     }
1788     my $vgamem_mb = $vga->{memory};
1789
1790     my $max_outputs = '';
1791     if ($qxlnum) {
1792         $type = $id ? 'qxl' : 'qxl-vga';
1793
1794         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1795             # set max outputs so linux can have up to 4 qxl displays with one device
1796             if (min_version($machine_version, 4, 1)) {
1797                 $max_outputs = ",max_outputs=4";
1798             }
1799         }
1800     }
1801
1802     die "no device-type for $vga->{type}\n" if !$type;
1803
1804     my $memory = "";
1805     if ($vgamem_mb) {
1806         if ($vga->{type} =~ /^virtio/) {
1807             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1808             $memory = ",max_hostmem=$bytes";
1809         } elsif ($qxlnum) {
1810             # from https://www.spice-space.org/multiple-monitors.html
1811             $memory = ",vgamem_mb=$vga->{memory}";
1812             my $ram = $vgamem_mb * 4;
1813             my $vram = $vgamem_mb * 2;
1814             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1815         } else {
1816             $memory = ",vgamem_mb=$vga->{memory}";
1817         }
1818     } elsif ($qxlnum && $id) {
1819         $memory = ",ram_size=67108864,vram_size=33554432";
1820     }
1821
1822     my $edidoff = "";
1823     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1824         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1825     }
1826
1827     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1828     my $vgaid = "vga" . ($id // '');
1829     my $pciaddr;
1830     if ($q35 && $vgaid eq 'vga') {
1831         # the first display uses pcie.0 bus on q35 machines
1832         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1833     } else {
1834         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1835     }
1836
1837     if ($vga->{type} eq 'virtio-gl') {
1838         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1839         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1840             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1841
1842         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1843             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1844     }
1845
1846     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1847 }
1848
1849 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1850 sub parse_net {
1851     my ($data, $disable_mac_autogen) = @_;
1852
1853     my $res = eval { parse_property_string($net_fmt, $data) };
1854     if ($@) {
1855         warn $@;
1856         return;
1857     }
1858     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1859         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1860         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1861     }
1862     return $res;
1863 }
1864
1865 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1866 sub parse_ipconfig {
1867     my ($data) = @_;
1868
1869     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1870     if ($@) {
1871         warn $@;
1872         return;
1873     }
1874
1875     if ($res->{gw} && !$res->{ip}) {
1876         warn 'gateway specified without specifying an IP address';
1877         return;
1878     }
1879     if ($res->{gw6} && !$res->{ip6}) {
1880         warn 'IPv6 gateway specified without specifying an IPv6 address';
1881         return;
1882     }
1883     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1884         warn 'gateway specified together with DHCP';
1885         return;
1886     }
1887     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1888         # gw6 + auto/dhcp
1889         warn "IPv6 gateway specified together with $res->{ip6} address";
1890         return;
1891     }
1892
1893     if (!$res->{ip} && !$res->{ip6}) {
1894         return { ip => 'dhcp', ip6 => 'dhcp' };
1895     }
1896
1897     return $res;
1898 }
1899
1900 sub print_net {
1901     my $net = shift;
1902
1903     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1904 }
1905
1906 sub add_random_macs {
1907     my ($settings) = @_;
1908
1909     foreach my $opt (keys %$settings) {
1910         next if $opt !~ m/^net(\d+)$/;
1911         my $net = parse_net($settings->{$opt});
1912         next if !$net;
1913         $settings->{$opt} = print_net($net);
1914     }
1915 }
1916
1917 sub vm_is_volid_owner {
1918     my ($storecfg, $vmid, $volid) = @_;
1919
1920     if ($volid !~  m|^/|) {
1921         my ($path, $owner);
1922         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1923         if ($owner && ($owner == $vmid)) {
1924             return 1;
1925         }
1926     }
1927
1928     return;
1929 }
1930
1931 sub vmconfig_register_unused_drive {
1932     my ($storecfg, $vmid, $conf, $drive) = @_;
1933
1934     if (drive_is_cloudinit($drive)) {
1935         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1936         warn $@ if $@;
1937         delete $conf->{cloudinit};
1938     } elsif (!drive_is_cdrom($drive)) {
1939         my $volid = $drive->{file};
1940         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1941             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1942         }
1943     }
1944 }
1945
1946 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1947 my $smbios1_fmt = {
1948     uuid => {
1949         type => 'string',
1950         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1951         format_description => 'UUID',
1952         description => "Set SMBIOS1 UUID.",
1953         optional => 1,
1954     },
1955     version => {
1956         type => 'string',
1957         pattern => '[A-Za-z0-9+\/]+={0,2}',
1958         format_description => 'Base64 encoded string',
1959         description => "Set SMBIOS1 version.",
1960         optional => 1,
1961     },
1962     serial => {
1963         type => 'string',
1964         pattern => '[A-Za-z0-9+\/]+={0,2}',
1965         format_description => 'Base64 encoded string',
1966         description => "Set SMBIOS1 serial number.",
1967         optional => 1,
1968     },
1969     manufacturer => {
1970         type => 'string',
1971         pattern => '[A-Za-z0-9+\/]+={0,2}',
1972         format_description => 'Base64 encoded string',
1973         description => "Set SMBIOS1 manufacturer.",
1974         optional => 1,
1975     },
1976     product => {
1977         type => 'string',
1978         pattern => '[A-Za-z0-9+\/]+={0,2}',
1979         format_description => 'Base64 encoded string',
1980         description => "Set SMBIOS1 product ID.",
1981         optional => 1,
1982     },
1983     sku => {
1984         type => 'string',
1985         pattern => '[A-Za-z0-9+\/]+={0,2}',
1986         format_description => 'Base64 encoded string',
1987         description => "Set SMBIOS1 SKU string.",
1988         optional => 1,
1989     },
1990     family => {
1991         type => 'string',
1992         pattern => '[A-Za-z0-9+\/]+={0,2}',
1993         format_description => 'Base64 encoded string',
1994         description => "Set SMBIOS1 family string.",
1995         optional => 1,
1996     },
1997     base64 => {
1998         type => 'boolean',
1999         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2000         optional => 1,
2001     },
2002 };
2003
2004 sub parse_smbios1 {
2005     my ($data) = @_;
2006
2007     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2008     warn $@ if $@;
2009     return $res;
2010 }
2011
2012 sub print_smbios1 {
2013     my ($smbios1) = @_;
2014     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2015 }
2016
2017 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2018
2019 sub parse_watchdog {
2020     my ($value) = @_;
2021
2022     return if !$value;
2023
2024     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 sub parse_guest_agent {
2030     my ($conf) = @_;
2031
2032     return {} if !defined($conf->{agent});
2033
2034     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2035     warn $@ if $@;
2036
2037     # if the agent is disabled ignore the other potentially set properties
2038     return {} if !$res->{enabled};
2039     return $res;
2040 }
2041
2042 sub get_qga_key {
2043     my ($conf, $key) = @_;
2044     return undef if !defined($conf->{agent});
2045
2046     my $agent = parse_guest_agent($conf);
2047     return $agent->{$key};
2048 }
2049
2050 sub parse_vga {
2051     my ($value) = @_;
2052
2053     return {} if !$value;
2054     my $res = eval { parse_property_string($vga_fmt, $value) };
2055     warn $@ if $@;
2056     return $res;
2057 }
2058
2059 sub parse_rng {
2060     my ($value) = @_;
2061
2062     return if !$value;
2063
2064     my $res = eval { parse_property_string($rng_fmt, $value) };
2065     warn $@ if $@;
2066     return $res;
2067 }
2068
2069 sub parse_meta_info {
2070     my ($value) = @_;
2071
2072     return if !$value;
2073
2074     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2075     warn $@ if $@;
2076     return $res;
2077 }
2078
2079 sub new_meta_info_string {
2080     my () = @_; # for now do not allow to override any value
2081
2082     return PVE::JSONSchema::print_property_string(
2083         {
2084             'creation-qemu' => kvm_user_version(),
2085             ctime => "". int(time()),
2086         },
2087         $meta_info_fmt
2088     );
2089 }
2090
2091 sub qemu_created_version_fixups {
2092     my ($conf, $forcemachine, $kvmver) = @_;
2093
2094     my $meta = parse_meta_info($conf->{meta}) // {};
2095     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2096
2097     # check if we need to apply some handling for VMs that always use the latest machine version but
2098     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2099     # would be required (we do not want to pin machine version for non-windows OS type)
2100     my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
2101     if (
2102         (!defined($machine_conf->{type}) || $machine_conf->{type} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2103         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2104         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2105         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2106     ) {
2107         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2108         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2109             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2110             # and thus with the predictable interface naming of systemd
2111             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2112         }
2113     }
2114     return;
2115 }
2116
2117 # add JSON properties for create and set function
2118 sub json_config_properties {
2119     my ($prop, $with_disk_alloc) = @_;
2120
2121     my $skip_json_config_opts = {
2122         parent => 1,
2123         snaptime => 1,
2124         vmstate => 1,
2125         runningmachine => 1,
2126         runningcpu => 1,
2127         meta => 1,
2128     };
2129
2130     foreach my $opt (keys %$confdesc) {
2131         next if $skip_json_config_opts->{$opt};
2132
2133         if ($with_disk_alloc && is_valid_drivename($opt)) {
2134             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2135         } else {
2136             $prop->{$opt} = $confdesc->{$opt};
2137         }
2138     }
2139
2140     return $prop;
2141 }
2142
2143 # Properties that we can read from an OVF file
2144 sub json_ovf_properties {
2145     my $prop = {};
2146
2147     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2148         $prop->{$device} = {
2149             type => 'string',
2150             format => 'pve-volume-id-or-absolute-path',
2151             description => "Disk image that gets imported to $device",
2152             optional => 1,
2153         };
2154     }
2155
2156     $prop->{cores} = {
2157         type => 'integer',
2158         description => "The number of CPU cores.",
2159         optional => 1,
2160     };
2161     $prop->{memory} = {
2162         type => 'integer',
2163         description => "Amount of RAM for the VM in MB.",
2164         optional => 1,
2165     };
2166     $prop->{name} = {
2167         type => 'string',
2168         description => "Name of the VM.",
2169         optional => 1,
2170     };
2171
2172     return $prop;
2173 }
2174
2175 # return copy of $confdesc_cloudinit to generate documentation
2176 sub cloudinit_config_properties {
2177
2178     return dclone($confdesc_cloudinit);
2179 }
2180
2181 sub cloudinit_pending_properties {
2182     my $p = {
2183         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2184         name => 1,
2185     };
2186     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2187     return $p;
2188 }
2189
2190 sub check_type {
2191     my ($key, $value) = @_;
2192
2193     die "unknown setting '$key'\n" if !$confdesc->{$key};
2194
2195     my $type = $confdesc->{$key}->{type};
2196
2197     if (!defined($value)) {
2198         die "got undefined value\n";
2199     }
2200
2201     if ($value =~ m/[\n\r]/) {
2202         die "property contains a line feed\n";
2203     }
2204
2205     if ($type eq 'boolean') {
2206         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2207         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2208         die "type check ('boolean') failed - got '$value'\n";
2209     } elsif ($type eq 'integer') {
2210         return int($1) if $value =~ m/^(\d+)$/;
2211         die "type check ('integer') failed - got '$value'\n";
2212     } elsif ($type eq 'number') {
2213         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2214         die "type check ('number') failed - got '$value'\n";
2215     } elsif ($type eq 'string') {
2216         if (my $fmt = $confdesc->{$key}->{format}) {
2217             PVE::JSONSchema::check_format($fmt, $value);
2218             return $value;
2219         }
2220         $value =~ s/^\"(.*)\"$/$1/;
2221         return $value;
2222     } else {
2223         die "internal error"
2224     }
2225 }
2226
2227 sub destroy_vm {
2228     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2229
2230     my $conf = PVE::QemuConfig->load_config($vmid);
2231
2232     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2233         PVE::QemuConfig->check_lock($conf);
2234     }
2235
2236     if ($conf->{template}) {
2237         # check if any base image is still used by a linked clone
2238         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2239                 my ($ds, $drive) = @_;
2240                 return if drive_is_cdrom($drive);
2241
2242                 my $volid = $drive->{file};
2243                 return if !$volid || $volid =~ m|^/|;
2244
2245                 die "base volume '$volid' is still in use by linked cloned\n"
2246                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2247
2248         });
2249     }
2250
2251     my $volids = {};
2252     my $remove_owned_drive = sub {
2253         my ($ds, $drive) = @_;
2254         return if drive_is_cdrom($drive, 1);
2255
2256         my $volid = $drive->{file};
2257         return if !$volid || $volid =~ m|^/|;
2258         return if $volids->{$volid};
2259
2260         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2261         return if !$path || !$owner || ($owner != $vmid);
2262
2263         $volids->{$volid} = 1;
2264         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2265         warn "Could not remove disk '$volid', check manually: $@" if $@;
2266     };
2267
2268     # only remove disks owned by this VM (referenced in the config)
2269     my $include_opts = {
2270         include_unused => 1,
2271         extra_keys => ['vmstate'],
2272     };
2273     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2274
2275     for my $snap (values %{$conf->{snapshots}}) {
2276         next if !defined($snap->{vmstate});
2277         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2278         next if !defined($drive);
2279         $remove_owned_drive->('vmstate', $drive);
2280     }
2281
2282     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2283
2284     if ($purge_unreferenced) { # also remove unreferenced disk
2285         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2286         PVE::Storage::foreach_volid($vmdisks, sub {
2287             my ($volid, $sid, $volname, $d) = @_;
2288             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2289             warn $@ if $@;
2290         });
2291     }
2292
2293     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2294     warn $@ if $@;
2295
2296     if (defined $replacement_conf) {
2297         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2298     } else {
2299         PVE::QemuConfig->destroy_config($vmid);
2300     }
2301 }
2302
2303 sub parse_vm_config {
2304     my ($filename, $raw, $strict) = @_;
2305
2306     return if !defined($raw);
2307
2308     my $res = {
2309         digest => Digest::SHA::sha1_hex($raw),
2310         snapshots => {},
2311         pending => {},
2312         cloudinit => {},
2313     };
2314
2315     my $handle_error = sub {
2316         my ($msg) = @_;
2317
2318         if ($strict) {
2319             die $msg;
2320         } else {
2321             warn $msg;
2322         }
2323     };
2324
2325     $filename =~ m|/qemu-server/(\d+)\.conf$|
2326         || die "got strange filename '$filename'";
2327
2328     my $vmid = $1;
2329
2330     my $conf = $res;
2331     my $descr;
2332     my $finish_description = sub {
2333         if (defined($descr)) {
2334             $descr =~ s/\s+$//;
2335             $conf->{description} = $descr;
2336         }
2337         $descr = undef;
2338     };
2339     my $section = '';
2340
2341     my @lines = split(/\n/, $raw);
2342     foreach my $line (@lines) {
2343         next if $line =~ m/^\s*$/;
2344
2345         if ($line =~ m/^\[PENDING\]\s*$/i) {
2346             $section = 'pending';
2347             $finish_description->();
2348             $conf = $res->{$section} = {};
2349             next;
2350         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2351             $section = 'cloudinit';
2352             $finish_description->();
2353             $conf = $res->{$section} = {};
2354             next;
2355
2356         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2357             $section = $1;
2358             $finish_description->();
2359             $conf = $res->{snapshots}->{$section} = {};
2360             next;
2361         }
2362
2363         if ($line =~ m/^\#(.*)$/) {
2364             $descr = '' if !defined($descr);
2365             $descr .= PVE::Tools::decode_text($1) . "\n";
2366             next;
2367         }
2368
2369         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2370             $descr = '' if !defined($descr);
2371             $descr .= PVE::Tools::decode_text($2);
2372         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2373             $conf->{snapstate} = $1;
2374         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2375             my $key = $1;
2376             my $value = $2;
2377             $conf->{$key} = $value;
2378         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2379             my $value = $1;
2380             if ($section eq 'pending') {
2381                 $conf->{delete} = $value; # we parse this later
2382             } else {
2383                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2384             }
2385         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2386             my $key = $1;
2387             my $value = $2;
2388             if ($section eq 'cloudinit') {
2389                 # ignore validation only used for informative purpose
2390                 $conf->{$key} = $value;
2391                 next;
2392             }
2393             eval { $value = check_type($key, $value); };
2394             if ($@) {
2395                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2396             } else {
2397                 $key = 'ide2' if $key eq 'cdrom';
2398                 my $fmt = $confdesc->{$key}->{format};
2399                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2400                     my $v = parse_drive($key, $value);
2401                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2402                         $v->{file} = $volid;
2403                         $value = print_drive($v);
2404                     } else {
2405                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2406                         next;
2407                     }
2408                 }
2409
2410                 $conf->{$key} = $value;
2411             }
2412         } else {
2413             $handle_error->("vm $vmid - unable to parse config: $line\n");
2414         }
2415     }
2416
2417     $finish_description->();
2418     delete $res->{snapstate}; # just to be sure
2419
2420     return $res;
2421 }
2422
2423 sub write_vm_config {
2424     my ($filename, $conf) = @_;
2425
2426     delete $conf->{snapstate}; # just to be sure
2427
2428     if ($conf->{cdrom}) {
2429         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2430         $conf->{ide2} = $conf->{cdrom};
2431         delete $conf->{cdrom};
2432     }
2433
2434     # we do not use 'smp' any longer
2435     if ($conf->{sockets}) {
2436         delete $conf->{smp};
2437     } elsif ($conf->{smp}) {
2438         $conf->{sockets} = $conf->{smp};
2439         delete $conf->{cores};
2440         delete $conf->{smp};
2441     }
2442
2443     my $used_volids = {};
2444
2445     my $cleanup_config = sub {
2446         my ($cref, $pending, $snapname) = @_;
2447
2448         foreach my $key (keys %$cref) {
2449             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2450                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2451             my $value = $cref->{$key};
2452             if ($key eq 'delete') {
2453                 die "propertry 'delete' is only allowed in [PENDING]\n"
2454                     if !$pending;
2455                 # fixme: check syntax?
2456                 next;
2457             }
2458             eval { $value = check_type($key, $value); };
2459             die "unable to parse value of '$key' - $@" if $@;
2460
2461             $cref->{$key} = $value;
2462
2463             if (!$snapname && is_valid_drivename($key)) {
2464                 my $drive = parse_drive($key, $value);
2465                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2466             }
2467         }
2468     };
2469
2470     &$cleanup_config($conf);
2471
2472     &$cleanup_config($conf->{pending}, 1);
2473
2474     foreach my $snapname (keys %{$conf->{snapshots}}) {
2475         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2476         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2477     }
2478
2479     # remove 'unusedX' settings if we re-add a volume
2480     foreach my $key (keys %$conf) {
2481         my $value = $conf->{$key};
2482         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2483             delete $conf->{$key};
2484         }
2485     }
2486
2487     my $generate_raw_config = sub {
2488         my ($conf, $pending) = @_;
2489
2490         my $raw = '';
2491
2492         # add description as comment to top of file
2493         if (defined(my $descr = $conf->{description})) {
2494             if ($descr) {
2495                 foreach my $cl (split(/\n/, $descr)) {
2496                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2497                 }
2498             } else {
2499                 $raw .= "#\n" if $pending;
2500             }
2501         }
2502
2503         foreach my $key (sort keys %$conf) {
2504             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2505             $raw .= "$key: $conf->{$key}\n";
2506         }
2507         return $raw;
2508     };
2509
2510     my $raw = &$generate_raw_config($conf);
2511
2512     if (scalar(keys %{$conf->{pending}})){
2513         $raw .= "\n[PENDING]\n";
2514         $raw .= &$generate_raw_config($conf->{pending}, 1);
2515     }
2516
2517     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2518         $raw .= "\n[special:cloudinit]\n";
2519         $raw .= &$generate_raw_config($conf->{cloudinit});
2520     }
2521
2522     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2523         $raw .= "\n[$snapname]\n";
2524         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2525     }
2526
2527     return $raw;
2528 }
2529
2530 sub load_defaults {
2531
2532     my $res = {};
2533
2534     # we use static defaults from our JSON schema configuration
2535     foreach my $key (keys %$confdesc) {
2536         if (defined(my $default = $confdesc->{$key}->{default})) {
2537             $res->{$key} = $default;
2538         }
2539     }
2540
2541     return $res;
2542 }
2543
2544 sub config_list {
2545     my $vmlist = PVE::Cluster::get_vmlist();
2546     my $res = {};
2547     return $res if !$vmlist || !$vmlist->{ids};
2548     my $ids = $vmlist->{ids};
2549     my $nodename = nodename();
2550
2551     foreach my $vmid (keys %$ids) {
2552         my $d = $ids->{$vmid};
2553         next if !$d->{node} || $d->{node} ne $nodename;
2554         next if !$d->{type} || $d->{type} ne 'qemu';
2555         $res->{$vmid}->{exists} = 1;
2556     }
2557     return $res;
2558 }
2559
2560 # test if VM uses local resources (to prevent migration)
2561 sub check_local_resources {
2562     my ($conf, $noerr) = @_;
2563
2564     my @loc_res = ();
2565     my $mapped_res = [];
2566
2567     my $nodelist = PVE::Cluster::get_nodelist();
2568     my $pci_map = PVE::Mapping::PCI::config();
2569     my $usb_map = PVE::Mapping::USB::config();
2570
2571     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2572
2573     my $add_missing_mapping = sub {
2574         my ($type, $key, $id) = @_;
2575         for my $node (@$nodelist) {
2576             my $entry;
2577             if ($type eq 'pci') {
2578                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2579             } elsif ($type eq 'usb') {
2580                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2581             }
2582             if (!scalar($entry->@*)) {
2583                 push @{$missing_mappings_by_node->{$node}}, $key;
2584             }
2585         }
2586     };
2587
2588     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2589     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2590
2591     push @loc_res, "ivshmem" if $conf->{ivshmem};
2592
2593     foreach my $k (keys %$conf) {
2594         if ($k =~ m/^usb/) {
2595             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2596             next if $entry->{host} && $entry->{host} =~ m/^spice$/i;
2597             if ($entry->{mapping}) {
2598                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2599                 push @$mapped_res, $k;
2600             }
2601         }
2602         if ($k =~ m/^hostpci/) {
2603             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2604             if ($entry->{mapping}) {
2605                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2606                 push @$mapped_res, $k;
2607             }
2608         }
2609         # sockets are safe: they will recreated be on the target side post-migrate
2610         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2611         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2612     }
2613
2614     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2615
2616     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2617 }
2618
2619 # check if used storages are available on all nodes (use by migrate)
2620 sub check_storage_availability {
2621     my ($storecfg, $conf, $node) = @_;
2622
2623     PVE::QemuConfig->foreach_volume($conf, sub {
2624         my ($ds, $drive) = @_;
2625
2626         my $volid = $drive->{file};
2627         return if !$volid;
2628
2629         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2630         return if !$sid;
2631
2632         # check if storage is available on both nodes
2633         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2634         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2635
2636         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2637
2638         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2639             if !$scfg->{content}->{$vtype};
2640    });
2641 }
2642
2643 # list nodes where all VM images are available (used by has_feature API)
2644 sub shared_nodes {
2645     my ($conf, $storecfg) = @_;
2646
2647     my $nodelist = PVE::Cluster::get_nodelist();
2648     my $nodehash = { map { $_ => 1 } @$nodelist };
2649     my $nodename = nodename();
2650
2651     PVE::QemuConfig->foreach_volume($conf, sub {
2652         my ($ds, $drive) = @_;
2653
2654         my $volid = $drive->{file};
2655         return if !$volid;
2656
2657         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2658         if ($storeid) {
2659             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2660             if ($scfg->{disable}) {
2661                 $nodehash = {};
2662             } elsif (my $avail = $scfg->{nodes}) {
2663                 foreach my $node (keys %$nodehash) {
2664                     delete $nodehash->{$node} if !$avail->{$node};
2665                 }
2666             } elsif (!$scfg->{shared}) {
2667                 foreach my $node (keys %$nodehash) {
2668                     delete $nodehash->{$node} if $node ne $nodename
2669                 }
2670             }
2671         }
2672     });
2673
2674     return $nodehash
2675 }
2676
2677 sub check_local_storage_availability {
2678     my ($conf, $storecfg) = @_;
2679
2680     my $nodelist = PVE::Cluster::get_nodelist();
2681     my $nodehash = { map { $_ => {} } @$nodelist };
2682
2683     PVE::QemuConfig->foreach_volume($conf, sub {
2684         my ($ds, $drive) = @_;
2685
2686         my $volid = $drive->{file};
2687         return if !$volid;
2688
2689         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2690         if ($storeid) {
2691             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2692
2693             if ($scfg->{disable}) {
2694                 foreach my $node (keys %$nodehash) {
2695                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2696                 }
2697             } elsif (my $avail = $scfg->{nodes}) {
2698                 foreach my $node (keys %$nodehash) {
2699                     if (!$avail->{$node}) {
2700                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2701                     }
2702                 }
2703             }
2704         }
2705     });
2706
2707     foreach my $node (values %$nodehash) {
2708         if (my $unavail = $node->{unavailable_storages}) {
2709             $node->{unavailable_storages} = [ sort keys %$unavail ];
2710         }
2711     }
2712
2713     return $nodehash
2714 }
2715
2716 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2717 sub check_running {
2718     my ($vmid, $nocheck, $node) = @_;
2719
2720     # $nocheck is set when called during a migration, in which case the config
2721     # file might still or already reside on the *other* node
2722     # - because rename has already happened, and current node is source
2723     # - because rename hasn't happened yet, and current node is target
2724     # - because rename has happened, current node is target, but hasn't yet
2725     # processed it yet
2726     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2727     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2728 }
2729
2730 sub vzlist {
2731
2732     my $vzlist = config_list();
2733
2734     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2735
2736     while (defined(my $de = $fd->read)) {
2737         next if $de !~ m/^(\d+)\.pid$/;
2738         my $vmid = $1;
2739         next if !defined($vzlist->{$vmid});
2740         if (my $pid = check_running($vmid)) {
2741             $vzlist->{$vmid}->{pid} = $pid;
2742         }
2743     }
2744
2745     return $vzlist;
2746 }
2747
2748 our $vmstatus_return_properties = {
2749     vmid => get_standard_option('pve-vmid'),
2750     status => {
2751         description => "QEMU process status.",
2752         type => 'string',
2753         enum => ['stopped', 'running'],
2754     },
2755     maxmem => {
2756         description => "Maximum memory in bytes.",
2757         type => 'integer',
2758         optional => 1,
2759         renderer => 'bytes',
2760     },
2761     maxdisk => {
2762         description => "Root disk size in bytes.",
2763         type => 'integer',
2764         optional => 1,
2765         renderer => 'bytes',
2766     },
2767     name => {
2768         description => "VM name.",
2769         type => 'string',
2770         optional => 1,
2771     },
2772     qmpstatus => {
2773         description => "VM run state from the 'query-status' QMP monitor command.",
2774         type => 'string',
2775         optional => 1,
2776     },
2777     pid => {
2778         description => "PID of running qemu process.",
2779         type => 'integer',
2780         optional => 1,
2781     },
2782     uptime => {
2783         description => "Uptime.",
2784         type => 'integer',
2785         optional => 1,
2786         renderer => 'duration',
2787     },
2788     cpus => {
2789         description => "Maximum usable CPUs.",
2790         type => 'number',
2791         optional => 1,
2792     },
2793     lock => {
2794         description => "The current config lock, if any.",
2795         type => 'string',
2796         optional => 1,
2797     },
2798     tags => {
2799         description  => "The current configured tags, if any",
2800         type => 'string',
2801         optional => 1,
2802     },
2803     'running-machine' => {
2804         description => "The currently running machine type (if running).",
2805         type => 'string',
2806         optional => 1,
2807     },
2808     'running-qemu' => {
2809         description => "The currently running QEMU version (if running).",
2810         type => 'string',
2811         optional => 1,
2812     },
2813 };
2814
2815 my $last_proc_pid_stat;
2816
2817 # get VM status information
2818 # This must be fast and should not block ($full == false)
2819 # We only query KVM using QMP if $full == true (this can be slow)
2820 sub vmstatus {
2821     my ($opt_vmid, $full) = @_;
2822
2823     my $res = {};
2824
2825     my $storecfg = PVE::Storage::config();
2826
2827     my $list = vzlist();
2828     my $defaults = load_defaults();
2829
2830     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2831
2832     my $cpucount = $cpuinfo->{cpus} || 1;
2833
2834     foreach my $vmid (keys %$list) {
2835         next if $opt_vmid && ($vmid ne $opt_vmid);
2836
2837         my $conf = PVE::QemuConfig->load_config($vmid);
2838
2839         my $d = { vmid => int($vmid) };
2840         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2841
2842         # fixme: better status?
2843         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2844
2845         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2846         if (defined($size)) {
2847             $d->{disk} = 0; # no info available
2848             $d->{maxdisk} = $size;
2849         } else {
2850             $d->{disk} = 0;
2851             $d->{maxdisk} = 0;
2852         }
2853
2854         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2855             * ($conf->{cores} || $defaults->{cores});
2856         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2857         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2858
2859         $d->{name} = $conf->{name} || "VM $vmid";
2860         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2861
2862         if ($conf->{balloon}) {
2863             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2864             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2865                 : $defaults->{shares};
2866         }
2867
2868         $d->{uptime} = 0;
2869         $d->{cpu} = 0;
2870         $d->{mem} = 0;
2871
2872         $d->{netout} = 0;
2873         $d->{netin} = 0;
2874
2875         $d->{diskread} = 0;
2876         $d->{diskwrite} = 0;
2877
2878         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2879
2880         $d->{serial} = 1 if conf_has_serial($conf);
2881         $d->{lock} = $conf->{lock} if $conf->{lock};
2882         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2883
2884         $res->{$vmid} = $d;
2885     }
2886
2887     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2888     foreach my $dev (keys %$netdev) {
2889         next if $dev !~ m/^tap([1-9]\d*)i/;
2890         my $vmid = $1;
2891         my $d = $res->{$vmid};
2892         next if !$d;
2893
2894         $d->{netout} += $netdev->{$dev}->{receive};
2895         $d->{netin} += $netdev->{$dev}->{transmit};
2896
2897         if ($full) {
2898             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2899             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2900         }
2901
2902     }
2903
2904     my $ctime = gettimeofday;
2905
2906     foreach my $vmid (keys %$list) {
2907
2908         my $d = $res->{$vmid};
2909         my $pid = $d->{pid};
2910         next if !$pid;
2911
2912         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2913         next if !$pstat; # not running
2914
2915         my $used = $pstat->{utime} + $pstat->{stime};
2916
2917         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2918
2919         if ($pstat->{vsize}) {
2920             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2921         }
2922
2923         my $old = $last_proc_pid_stat->{$pid};
2924         if (!$old) {
2925             $last_proc_pid_stat->{$pid} = {
2926                 time => $ctime,
2927                 used => $used,
2928                 cpu => 0,
2929             };
2930             next;
2931         }
2932
2933         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2934
2935         if ($dtime > 1000) {
2936             my $dutime = $used -  $old->{used};
2937
2938             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2939             $last_proc_pid_stat->{$pid} = {
2940                 time => $ctime,
2941                 used => $used,
2942                 cpu => $d->{cpu},
2943             };
2944         } else {
2945             $d->{cpu} = $old->{cpu};
2946         }
2947     }
2948
2949     return $res if !$full;
2950
2951     my $qmpclient = PVE::QMPClient->new();
2952
2953     my $ballooncb = sub {
2954         my ($vmid, $resp) = @_;
2955
2956         my $info = $resp->{'return'};
2957         return if !$info->{max_mem};
2958
2959         my $d = $res->{$vmid};
2960
2961         # use memory assigned to VM
2962         $d->{maxmem} = $info->{max_mem};
2963         $d->{balloon} = $info->{actual};
2964
2965         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2966             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2967             $d->{freemem} = $info->{free_mem};
2968         }
2969
2970         $d->{ballooninfo} = $info;
2971     };
2972
2973     my $blockstatscb = sub {
2974         my ($vmid, $resp) = @_;
2975         my $data = $resp->{'return'} || [];
2976         my $totalrdbytes = 0;
2977         my $totalwrbytes = 0;
2978
2979         for my $blockstat (@$data) {
2980             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2981             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2982
2983             $blockstat->{device} =~ s/drive-//;
2984             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2985         }
2986         $res->{$vmid}->{diskread} = $totalrdbytes;
2987         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2988     };
2989
2990     my $machinecb = sub {
2991         my ($vmid, $resp) = @_;
2992         my $data = $resp->{'return'} || [];
2993
2994         $res->{$vmid}->{'running-machine'} =
2995             PVE::QemuServer::Machine::current_from_query_machines($data);
2996     };
2997
2998     my $versioncb = sub {
2999         my ($vmid, $resp) = @_;
3000         my $data = $resp->{'return'} // {};
3001         my $version = 'unknown';
3002
3003         if (my $v = $data->{qemu}) {
3004             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3005         }
3006
3007         $res->{$vmid}->{'running-qemu'} = $version;
3008     };
3009
3010     my $statuscb = sub {
3011         my ($vmid, $resp) = @_;
3012
3013         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3014         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3015         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3016         # this fails if ballon driver is not loaded, so this must be
3017         # the last commnand (following command are aborted if this fails).
3018         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3019
3020         my $status = 'unknown';
3021         if (!defined($status = $resp->{'return'}->{status})) {
3022             warn "unable to get VM status\n";
3023             return;
3024         }
3025
3026         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3027     };
3028
3029     foreach my $vmid (keys %$list) {
3030         next if $opt_vmid && ($vmid ne $opt_vmid);
3031         next if !$res->{$vmid}->{pid}; # not running
3032         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3033     }
3034
3035     $qmpclient->queue_execute(undef, 2);
3036
3037     foreach my $vmid (keys %$list) {
3038         next if $opt_vmid && ($vmid ne $opt_vmid);
3039         next if !$res->{$vmid}->{pid}; #not running
3040
3041         # we can't use the $qmpclient since it might have already aborted on
3042         # 'query-balloon', but this might also fail for older versions...
3043         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3044         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3045     }
3046
3047     foreach my $vmid (keys %$list) {
3048         next if $opt_vmid && ($vmid ne $opt_vmid);
3049         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3050     }
3051
3052     return $res;
3053 }
3054
3055 sub conf_has_serial {
3056     my ($conf) = @_;
3057
3058     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3059         if ($conf->{"serial$i"}) {
3060             return 1;
3061         }
3062     }
3063
3064     return 0;
3065 }
3066
3067 sub conf_has_audio {
3068     my ($conf, $id) = @_;
3069
3070     $id //= 0;
3071     my $audio = $conf->{"audio$id"};
3072     return if !defined($audio);
3073
3074     my $audioproperties = parse_property_string($audio_fmt, $audio);
3075     my $audiodriver = $audioproperties->{driver} // 'spice';
3076
3077     return {
3078         dev => $audioproperties->{device},
3079         dev_id => "audiodev$id",
3080         backend => $audiodriver,
3081         backend_id => "$audiodriver-backend${id}",
3082     };
3083 }
3084
3085 sub audio_devs {
3086     my ($audio, $audiopciaddr, $machine_version) = @_;
3087
3088     my $devs = [];
3089
3090     my $id = $audio->{dev_id};
3091     my $audiodev = "";
3092     if (min_version($machine_version, 4, 2)) {
3093         $audiodev = ",audiodev=$audio->{backend_id}";
3094     }
3095
3096     if ($audio->{dev} eq 'AC97') {
3097         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3098     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3099         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3100         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3101         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3102     } else {
3103         die "unkown audio device '$audio->{dev}', implement me!";
3104     }
3105
3106     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3107
3108     return $devs;
3109 }
3110
3111 sub get_tpm_paths {
3112     my ($vmid) = @_;
3113     return {
3114         socket => "/var/run/qemu-server/$vmid.swtpm",
3115         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3116     };
3117 }
3118
3119 sub add_tpm_device {
3120     my ($vmid, $devices, $conf) = @_;
3121
3122     return if !$conf->{tpmstate0};
3123
3124     my $paths = get_tpm_paths($vmid);
3125
3126     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3127     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3128     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3129 }
3130
3131 sub start_swtpm {
3132     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3133
3134     return if !$tpmdrive;
3135
3136     my $state;
3137     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3138     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3139     if ($storeid) {
3140         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3141     } else {
3142         $state = $tpm->{file};
3143     }
3144
3145     my $paths = get_tpm_paths($vmid);
3146
3147     # during migration, we will get state from remote
3148     #
3149     if (!$migration) {
3150         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3151         my $setup_cmd = [
3152             "swtpm_setup",
3153             "--tpmstate",
3154             "file://$state",
3155             "--createek",
3156             "--create-ek-cert",
3157             "--create-platform-cert",
3158             "--lock-nvram",
3159             "--config",
3160             "/etc/swtpm_setup.conf", # do not use XDG configs
3161             "--runas",
3162             "0", # force creation as root, error if not possible
3163             "--not-overwrite", # ignore existing state, do not modify
3164         ];
3165
3166         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3167         # TPM 2.0 supports ECC crypto, use if possible
3168         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3169
3170         run_command($setup_cmd, outfunc => sub {
3171             print "swtpm_setup: $1\n";
3172         });
3173     }
3174
3175     # Used to distinguish different invocations in the log.
3176     my $log_prefix = "[id=" . int(time()) . "] ";
3177
3178     my $emulator_cmd = [
3179         "swtpm",
3180         "socket",
3181         "--tpmstate",
3182         "backend-uri=file://$state,mode=0600",
3183         "--ctrl",
3184         "type=unixio,path=$paths->{socket},mode=0600",
3185         "--pid",
3186         "file=$paths->{pid}",
3187         "--terminate", # terminate on QEMU disconnect
3188         "--daemon",
3189         "--log",
3190         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3191     ];
3192     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3193     run_command($emulator_cmd, outfunc => sub { print $1; });
3194
3195     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3196     while (! -e $paths->{pid}) {
3197         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3198         usleep(50_000);
3199     }
3200
3201     # return untainted PID of swtpm daemon so it can be killed on error
3202     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3203     return $1;
3204 }
3205
3206 sub vga_conf_has_spice {
3207     my ($vga) = @_;
3208
3209     my $vgaconf = parse_vga($vga);
3210     my $vgatype = $vgaconf->{type};
3211     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3212
3213     return $1 || 1;
3214 }
3215
3216 sub get_vm_arch {
3217     my ($conf) = @_;
3218     return $conf->{arch} // get_host_arch();
3219 }
3220
3221 my $default_machines = {
3222     x86_64 => 'pc',
3223     aarch64 => 'virt',
3224 };
3225
3226 sub get_installed_machine_version {
3227     my ($kvmversion) = @_;
3228     $kvmversion = kvm_user_version() if !defined($kvmversion);
3229     $kvmversion =~ m/^(\d+\.\d+)/;
3230     return $1;
3231 }
3232
3233 sub windows_get_pinned_machine_version {
3234     my ($machine, $base_version, $kvmversion) = @_;
3235
3236     my $pin_version = $base_version;
3237     if (!defined($base_version) ||
3238         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3239     ) {
3240         $pin_version = get_installed_machine_version($kvmversion);
3241     }
3242     if (!$machine || $machine eq 'pc') {
3243         $machine = "pc-i440fx-$pin_version";
3244     } elsif ($machine eq 'q35') {
3245         $machine = "pc-q35-$pin_version";
3246     } elsif ($machine eq 'virt') {
3247         $machine = "virt-$pin_version";
3248     } else {
3249         warn "unknown machine type '$machine', not touching that!\n";
3250     }
3251
3252     return $machine;
3253 }
3254
3255 sub get_vm_machine {
3256     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3257
3258     my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
3259     my $machine = $forcemachine || $machine_conf->{type};
3260
3261     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3262         $kvmversion //= kvm_user_version();
3263         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3264         # layout which confuses windows quite a bit and may result in various regressions..
3265         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3266         if (windows_version($conf->{ostype})) {
3267             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3268         }
3269         $arch //= 'x86_64';
3270         $machine ||= $default_machines->{$arch};
3271         if ($add_pve_version) {
3272             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3273             $machine .= "+pve$pvever";
3274         }
3275     }
3276
3277     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3278         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3279         $machine = $1 if $is_pxe;
3280
3281         # for version-pinned machines that do not include a pve-version (e.g.
3282         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3283         $machine .= '+pve0';
3284
3285         $machine .= '.pxe' if $is_pxe;
3286     }
3287
3288     return $machine;
3289 }
3290
3291 sub get_ovmf_files($$$) {
3292     my ($arch, $efidisk, $smm) = @_;
3293
3294     my $types = $OVMF->{$arch}
3295         or die "no OVMF images known for architecture '$arch'\n";
3296
3297     my $type = 'default';
3298     if ($arch eq 'x86_64') {
3299         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3300             $type = $smm ? "4m" : "4m-no-smm";
3301             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3302         } else {
3303             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3304         }
3305     }
3306
3307     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3308     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3309     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3310
3311     return ($ovmf_code, $ovmf_vars);
3312 }
3313
3314 my $Arch2Qemu = {
3315     aarch64 => '/usr/bin/qemu-system-aarch64',
3316     x86_64 => '/usr/bin/qemu-system-x86_64',
3317 };
3318 sub get_command_for_arch($) {
3319     my ($arch) = @_;
3320     return '/usr/bin/kvm' if is_native_arch($arch);
3321
3322     my $cmd = $Arch2Qemu->{$arch}
3323         or die "don't know how to emulate architecture '$arch'\n";
3324     return $cmd;
3325 }
3326
3327 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3328 # to use in a QEMU command line (-cpu element), first array_intersect the result
3329 # of query_supported_ with query_understood_. This is necessary because:
3330 #
3331 # a) query_understood_ returns flags the host cannot use and
3332 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3333 #    flags, but CPU settings - with most of them being flags. Those settings
3334 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3335 #
3336 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3337 # expensive. If you need the value returned from this, you can get it much
3338 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3339 # $accel being 'kvm' or 'tcg'.
3340 #
3341 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3342 # changes, automatically populating pmxcfs.
3343 #
3344 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3345 # since kvm and tcg machines support different flags
3346 #
3347 sub query_supported_cpu_flags {
3348     my ($arch) = @_;
3349
3350     $arch //= get_host_arch();
3351     my $default_machine = $default_machines->{$arch};
3352
3353     my $flags = {};
3354
3355     # FIXME: Once this is merged, the code below should work for ARM as well:
3356     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3357     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3358         $arch eq "aarch64";
3359
3360     my $kvm_supported = defined(kvm_version());
3361     my $qemu_cmd = get_command_for_arch($arch);
3362     my $fakevmid = -1;
3363     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3364
3365     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3366     my $query_supported_run_qemu = sub {
3367         my ($kvm) = @_;
3368
3369         my $flags = {};
3370         my $cmd = [
3371             $qemu_cmd,
3372             '-machine', $default_machine,
3373             '-display', 'none',
3374             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3375             '-mon', 'chardev=qmp,mode=control',
3376             '-pidfile', $pidfile,
3377             '-S', '-daemonize'
3378         ];
3379
3380         if (!$kvm) {
3381             push @$cmd, '-accel', 'tcg';
3382         }
3383
3384         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3385         die "QEMU flag querying VM exited with code " . $rc if $rc;
3386
3387         eval {
3388             my $cmd_result = mon_cmd(
3389                 $fakevmid,
3390                 'query-cpu-model-expansion',
3391                 type => 'full',
3392                 model => { name => 'host' }
3393             );
3394
3395             my $props = $cmd_result->{model}->{props};
3396             foreach my $prop (keys %$props) {
3397                 next if $props->{$prop} ne '1';
3398                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3399                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3400                 # We only keep those with underscores, to match /proc/cpuinfo
3401                 $prop =~ s/\.|-/_/g;
3402                 $flags->{$prop} = 1;
3403             }
3404         };
3405         my $err = $@;
3406
3407         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3408         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3409
3410         die $err if $err;
3411
3412         return [ sort keys %$flags ];
3413     };
3414
3415     # We need to query QEMU twice, since KVM and TCG have different supported flags
3416     PVE::QemuConfig->lock_config($fakevmid, sub {
3417         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3418         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3419
3420         if ($kvm_supported) {
3421             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3422             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3423         }
3424     });
3425
3426     return $flags;
3427 }
3428
3429 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3430 my $understood_cpu_flag_dir = "/usr/share/kvm";
3431 sub query_understood_cpu_flags {
3432     my $arch = get_host_arch();
3433     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3434
3435     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3436         if ! -e $filepath;
3437
3438     my $raw = file_get_contents($filepath);
3439     $raw =~ s/^\s+|\s+$//g;
3440     my @flags = split(/\s+/, $raw);
3441
3442     return \@flags;
3443 }
3444
3445 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3446 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3447 my sub should_disable_smm {
3448     my ($conf, $vga, $machine) = @_;
3449
3450     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3451
3452     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3453         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3454 }
3455
3456 my sub print_ovmf_drive_commandlines {
3457     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3458
3459     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3460
3461     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3462
3463     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3464     if ($d) {
3465         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3466         my ($path, $format) = $d->@{'file', 'format'};
3467         if ($storeid) {
3468             $path = PVE::Storage::path($storecfg, $d->{file});
3469             if (!defined($format)) {
3470                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3471                 $format = qemu_img_format($scfg, $volname);
3472             }
3473         } elsif (!defined($format)) {
3474             die "efidisk format must be specified\n";
3475         }
3476         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3477         if ($path =~ m/^rbd:/) {
3478             $var_drive_str .= ',cache=writeback';
3479             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3480         }
3481         $var_drive_str .= ",format=$format,file=$path";
3482
3483         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3484         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3485     } else {
3486         log_warn("no efidisk configured! Using temporary efivars disk.");
3487         my $path = "/tmp/$vmid-ovmf.fd";
3488         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3489         $var_drive_str .= ",format=raw,file=$path";
3490         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3491     }
3492
3493     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3494 }
3495
3496 sub config_to_command {
3497     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3498         $live_restore_backing) = @_;
3499
3500     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3501     my $devices = [];
3502     my $bridges = {};
3503     my $ostype = $conf->{ostype};
3504     my $winversion = windows_version($ostype);
3505     my $kvm = $conf->{kvm};
3506     my $nodename = nodename();
3507
3508     my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
3509
3510     my $arch = get_vm_arch($conf);
3511     my $kvm_binary = get_command_for_arch($arch);
3512     my $kvmver = kvm_user_version($kvm_binary);
3513
3514     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3515         $kvmver //= "undefined";
3516         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3517     }
3518
3519     my $add_pve_version = min_version($kvmver, 4, 1);
3520
3521     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3522     my $machine_version = extract_version($machine_type, $kvmver);
3523     $kvm //= 1 if is_native_arch($arch);
3524
3525     $machine_version =~ m/(\d+)\.(\d+)/;
3526     my ($machine_major, $machine_minor) = ($1, $2);
3527
3528     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3529         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3530     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3531         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3532             ." please upgrade node '$nodename'\n"
3533     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3534         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3535         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3536             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3537             ." node '$nodename'\n";
3538     }
3539
3540     # if a specific +pve version is required for a feature, use $version_guard
3541     # instead of min_version to allow machines to be run with the minimum
3542     # required version
3543     my $required_pve_version = 0;
3544     my $version_guard = sub {
3545         my ($major, $minor, $pve) = @_;
3546         return 0 if !min_version($machine_version, $major, $minor, $pve);
3547         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3548         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3549         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3550         return 1;
3551     };
3552
3553     if ($kvm && !defined kvm_version()) {
3554         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3555             ." or enable in BIOS.\n";
3556     }
3557
3558     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3559     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3560     my $use_old_bios_files = undef;
3561     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3562
3563     my $cmd = [];
3564     if ($conf->{affinity}) {
3565         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3566     }
3567
3568     push @$cmd, $kvm_binary;
3569
3570     push @$cmd, '-id', $vmid;
3571
3572     my $vmname = $conf->{name} || "vm$vmid";
3573
3574     push @$cmd, '-name', "$vmname,debug-threads=on";
3575
3576     push @$cmd, '-no-shutdown';
3577
3578     my $use_virtio = 0;
3579
3580     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3581     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3582     push @$cmd, '-mon', "chardev=qmp,mode=control";
3583
3584     if (min_version($machine_version, 2, 12)) {
3585         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3586         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3587     }
3588
3589     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3590
3591     push @$cmd, '-daemonize';
3592
3593     if ($conf->{smbios1}) {
3594         my $smbios_conf = parse_smbios1($conf->{smbios1});
3595         if ($smbios_conf->{base64}) {
3596             # Do not pass base64 flag to qemu
3597             delete $smbios_conf->{base64};
3598             my $smbios_string = "";
3599             foreach my $key (keys %$smbios_conf) {
3600                 my $value;
3601                 if ($key eq "uuid") {
3602                     $value = $smbios_conf->{uuid}
3603                 } else {
3604                     $value = decode_base64($smbios_conf->{$key});
3605                 }
3606                 # qemu accepts any binary data, only commas need escaping by double comma
3607                 $value =~ s/,/,,/g;
3608                 $smbios_string .= "," . $key . "=" . $value if $value;
3609             }
3610             push @$cmd, '-smbios', "type=1" . $smbios_string;
3611         } else {
3612             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3613         }
3614     }
3615
3616     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3617         die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3618             if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3619
3620         my ($code_drive_str, $var_drive_str) =
3621             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3622         push $cmd->@*, '-drive', $code_drive_str;
3623         push $cmd->@*, '-drive', $var_drive_str;
3624     }
3625
3626     if ($q35) { # tell QEMU to load q35 config early
3627         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628         if (min_version($machine_version, 4, 0)) {
3629             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3630         } else {
3631             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632         }
3633     }
3634
3635     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3636         push @$cmd, $fixups->@*;
3637     }
3638
3639     if ($conf->{vmgenid}) {
3640         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3641     }
3642
3643     # add usb controllers
3644     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3645         $conf, $bridges, $arch, $machine_type, $machine_version);
3646     push @$devices, @usbcontrollers if @usbcontrollers;
3647     my $vga = parse_vga($conf->{vga});
3648
3649     my $qxlnum = vga_conf_has_spice($conf->{vga});
3650     $vga->{type} = 'qxl' if $qxlnum;
3651
3652     if (!$vga->{type}) {
3653         if ($arch eq 'aarch64') {
3654             $vga->{type} = 'virtio';
3655         } elsif (min_version($machine_version, 2, 9)) {
3656             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657         } else {
3658             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659         }
3660     }
3661
3662     # enable absolute mouse coordinates (needed by vnc)
3663     my $tablet = $conf->{tablet};
3664     if (!defined($tablet)) {
3665         $tablet = $defaults->{tablet};
3666         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668     }
3669
3670     if ($tablet) {
3671         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3672         my $kbd = print_keyboarddevice_full($conf, $arch);
3673         push @$devices, '-device', $kbd if defined($kbd);
3674     }
3675
3676     my $bootorder = device_bootorder($conf);
3677
3678     # host pci device passthrough
3679     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3680         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3681
3682     # usb devices
3683     my $usb_dev_features = {};
3684     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3685
3686     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3687         $conf, $usb_dev_features, $bootorder, $machine_version);
3688     push @$devices, @usbdevices if @usbdevices;
3689
3690     # serial devices
3691     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3692         my $path = $conf->{"serial$i"} or next;
3693         if ($path eq 'socket') {
3694             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696             # On aarch64, serial0 is the UART device. QEMU only allows
3697             # connecting UART devices via the '-serial' command line, as
3698             # the device has a fixed slot on the hardware...
3699             if ($arch eq 'aarch64' && $i == 0) {
3700                 push @$devices, '-serial', "chardev:serial$i";
3701             } else {
3702                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703             }
3704         } else {
3705             die "no such serial device\n" if ! -c $path;
3706             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3707             push @$devices, '-device', "isa-serial,chardev=serial$i";
3708         }
3709     }
3710
3711     # parallel devices
3712     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3713         if (my $path = $conf->{"parallel$i"}) {
3714             die "no such parallel device\n" if ! -c $path;
3715             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3716             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718         }
3719     }
3720
3721     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3722         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3723         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3724         push @$devices, @$audio_devs;
3725     }
3726
3727     # Add a TPM only if the VM is not a template,
3728     # to support backing up template VMs even if the TPM disk is write-protected.
3729     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3730
3731     my $sockets = 1;
3732     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3733     $sockets = $conf->{sockets} if  $conf->{sockets};
3734
3735     my $cores = $conf->{cores} || 1;
3736
3737     my $maxcpus = $sockets * $cores;
3738
3739     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3740
3741     my $allowed_vcpus = $cpuinfo->{cpus};
3742
3743     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3744
3745     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3746         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747         for (my $i = 2; $i <= $vcpus; $i++)  {
3748             my $cpustr = print_cpu_device($conf, $arch, $i);
3749             push @$cmd, '-device', $cpustr;
3750         }
3751
3752     } else {
3753
3754         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755     }
3756     push @$cmd, '-nodefaults';
3757
3758     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3759
3760     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3761
3762     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3763
3764     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3765         push @$devices, '-device', print_vga_device(
3766             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3767
3768         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3769
3770         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3771         push @$cmd,  '-vnc', "unix:$socket,password=on";
3772     } else {
3773         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3774         push @$cmd, '-nographic';
3775     }
3776
3777     # time drift fix
3778     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3779     my $useLocaltime = $conf->{localtime};
3780
3781     if ($winversion >= 5) { # windows
3782         $useLocaltime = 1 if !defined($conf->{localtime});
3783
3784         # use time drift fix when acpi is enabled
3785         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3786             $tdf = 1 if !defined($conf->{tdf});
3787         }
3788     }
3789
3790     if ($winversion >= 6) {
3791         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3792         push @$machineFlags, 'hpet=off';
3793     }
3794
3795     push @$rtcFlags, 'driftfix=slew' if $tdf;
3796
3797     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3798         push @$rtcFlags, "base=$conf->{startdate}";
3799     } elsif ($useLocaltime) {
3800         push @$rtcFlags, 'base=localtime';
3801     }
3802
3803     if ($forcecpu) {
3804         push @$cmd, '-cpu', $forcecpu;
3805     } else {
3806         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3807     }
3808
3809     PVE::QemuServer::Memory::config(
3810         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3811
3812     push @$cmd, '-S' if $conf->{freeze};
3813
3814     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3815
3816     my $guest_agent = parse_guest_agent($conf);
3817
3818     if ($guest_agent->{enabled}) {
3819         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3820         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3821
3822         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3823             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3824             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3825             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3826         } elsif ($guest_agent->{type} eq 'isa') {
3827             push @$devices, '-device', "isa-serial,chardev=qga0";
3828         }
3829     }
3830
3831     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3832     if ($rng && $version_guard->(4, 1, 2)) {
3833         check_rng_source($rng->{source});
3834
3835         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3836         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3837         my $limiter_str = "";
3838         if ($max_bytes) {
3839             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3840         }
3841
3842         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3843         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3844         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3845     }
3846
3847     my $spice_port;
3848
3849     assert_clipboard_config($vga);
3850     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3851
3852     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3853         if ($qxlnum > 1) {
3854             if ($winversion){
3855                 for (my $i = 1; $i < $qxlnum; $i++){
3856                     push @$devices, '-device', print_vga_device(
3857                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3858                 }
3859             } else {
3860                 # assume other OS works like Linux
3861                 my ($ram, $vram) = ("134217728", "67108864");
3862                 if ($vga->{memory}) {
3863                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3864                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3865                 }
3866                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3867                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3868             }
3869         }
3870
3871         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3872
3873         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3874         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3875             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3876         } else {
3877             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3878         }
3879         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880
3881         if ($is_spice) {
3882             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3883             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3884             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3885
3886             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3887             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3888
3889             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3890             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3891             if ($spice_enhancement->{foldersharing}) {
3892                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3894             }
3895
3896             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898                 if $spice_enhancement->{videostreaming};
3899             push @$devices, '-spice', "$spice_opts";
3900         }
3901     }
3902
3903     # enable balloon by default, unless explicitly disabled
3904     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3905         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3906         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3908         push @$devices, '-device', $ballooncmd;
3909     }
3910
3911     if ($conf->{watchdog}) {
3912         my $wdopts = parse_watchdog($conf->{watchdog});
3913         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3914         my $watchdog = $wdopts->{model} || 'i6300esb';
3915         push @$devices, '-device', "$watchdog$pciaddr";
3916         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3917     }
3918
3919     my $vollist = [];
3920     my $scsicontroller = {};
3921     my $ahcicontroller = {};
3922     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3923
3924     # Add iscsi initiator name if available
3925     if (my $initiator = get_initiator_name()) {
3926         push @$devices, '-iscsi', "initiator-name=$initiator";
3927     }
3928
3929     PVE::QemuConfig->foreach_volume($conf, sub {
3930         my ($ds, $drive) = @_;
3931
3932         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3933             check_volume_storage_type($storecfg, $drive->{file});
3934             push @$vollist, $drive->{file};
3935         }
3936
3937         # ignore efidisk here, already added in bios/fw handling code above
3938         return if $drive->{interface} eq 'efidisk';
3939         # similar for TPM
3940         return if $drive->{interface} eq 'tpmstate';
3941
3942         $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3945
3946         if ($drive->{interface} eq 'virtio'){
3947            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3948         }
3949
3950         if ($drive->{interface} eq 'scsi') {
3951
3952             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3953
3954             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3956
3957             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3959
3960             my $iothread = '';
3961             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3962                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964             } elsif ($drive->{iothread}) {
3965                 log_warn(
3966                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3967                 );
3968             }
3969
3970             my $queues = '';
3971             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3972                 $queues = ",num_queues=$drive->{queues}";
3973             }
3974
3975             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3976                 if !$scsicontroller->{$controller};
3977             $scsicontroller->{$controller}=1;
3978         }
3979
3980         if ($drive->{interface} eq 'sata') {
3981             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3982             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3983             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3984                 if !$ahcicontroller->{$controller};
3985             $ahcicontroller->{$controller}=1;
3986         }
3987
3988         my $live_restore = $live_restore_backing->{$ds};
3989         my $live_blockdev_name = undef;
3990         if ($live_restore) {
3991             $live_blockdev_name = $live_restore->{name};
3992             push @$devices, '-blockdev', $live_restore->{blockdev};
3993         }
3994
3995         my $drive_cmd = print_drive_commandline_full(
3996             $storecfg, $vmid, $drive, $live_blockdev_name, min_version($kvmver, 6, 0));
3997
3998         # extra protection for templates, but SATA and IDE don't support it..
3999         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4000
4001         push @$devices, '-drive',$drive_cmd;
4002         push @$devices, '-device', print_drivedevice_full(
4003             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4004     });
4005
4006     for (my $i = 0; $i < $MAX_NETS; $i++) {
4007         my $netname = "net$i";
4008
4009         next if !$conf->{$netname};
4010         my $d = parse_net($conf->{$netname});
4011         next if !$d;
4012         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4013
4014         $use_virtio = 1 if $d->{model} eq 'virtio';
4015
4016         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4017
4018         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4019         push @$devices, '-netdev', $netdevfull;
4020
4021         my $netdevicefull = print_netdevice_full(
4022             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4023
4024         push @$devices, '-device', $netdevicefull;
4025     }
4026
4027     if ($conf->{ivshmem}) {
4028         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4029
4030         my $bus;
4031         if ($q35) {
4032             $bus = print_pcie_addr("ivshmem");
4033         } else {
4034             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4035         }
4036
4037         my $ivshmem_name = $ivshmem->{name} // $vmid;
4038         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4039
4040         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4042             .",size=$ivshmem->{size}M";
4043     }
4044
4045     # pci.4 is nested in pci.1
4046     $bridges->{1} = 1 if $bridges->{4};
4047
4048     if (!$q35) { # add pci bridges
4049         if (min_version($machine_version, 2, 3)) {
4050            $bridges->{1} = 1;
4051            $bridges->{2} = 1;
4052         }
4053         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4054     }
4055
4056     for my $k (sort {$b cmp $a} keys %$bridges) {
4057         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4058
4059         my $k_name = $k;
4060         if ($k == 2 && $legacy_igd) {
4061             $k_name = "$k-igd";
4062         }
4063         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4064         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4065
4066         if ($q35) { # add after -readconfig pve-q35.cfg
4067             splice @$devices, 2, 0, '-device', $devstr;
4068         } else {
4069             unshift @$devices, '-device', $devstr if $k > 0;
4070         }
4071     }
4072
4073     if (!$kvm) {
4074         push @$machineFlags, 'accel=tcg';
4075     }
4076
4077     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4078
4079     my $machine_type_min = $machine_type;
4080     if ($add_pve_version) {
4081         $machine_type_min =~ s/\+pve\d+$//;
4082         $machine_type_min .= "+pve$required_pve_version";
4083     }
4084     push @$machineFlags, "type=${machine_type_min}";
4085
4086     PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
4087
4088     if (my $viommu = $machine_conf->{viommu}) {
4089         if ($viommu eq 'intel') {
4090             unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on';
4091             push @$machineFlags, 'kernel-irqchip=split';
4092         } elsif ($viommu eq 'virtio') {
4093             push @$devices, '-device', 'virtio-iommu-pci';
4094         }
4095     }
4096
4097     push @$cmd, @$devices;
4098     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4099     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4100     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4101
4102     if (my $vmstate = $conf->{vmstate}) {
4103         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4104         push @$vollist, $vmstate;
4105         push @$cmd, '-loadstate', $statepath;
4106         print "activating and using '$vmstate' as vmstate\n";
4107     }
4108
4109     if (PVE::QemuConfig->is_template($conf)) {
4110         # needed to workaround base volumes being read-only
4111         push @$cmd, '-snapshot';
4112     }
4113
4114     # add custom args
4115     if ($conf->{args}) {
4116         my $aa = PVE::Tools::split_args($conf->{args});
4117         push @$cmd, @$aa;
4118     }
4119
4120     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4121 }
4122
4123 sub check_rng_source {
4124     my ($source) = @_;
4125
4126     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4127     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4128         if ! -e $source;
4129
4130     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4131     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4132         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4133         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4134         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4135             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4136             ." to the host.\n";
4137     }
4138 }
4139
4140 sub spice_port {
4141     my ($vmid) = @_;
4142
4143     my $res = mon_cmd($vmid, 'query-spice');
4144
4145     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4146 }
4147
4148 sub vm_devices_list {
4149     my ($vmid) = @_;
4150
4151     my $res = mon_cmd($vmid, 'query-pci');
4152     my $devices_to_check = [];
4153     my $devices = {};
4154     foreach my $pcibus (@$res) {
4155         push @$devices_to_check, @{$pcibus->{devices}},
4156     }
4157
4158     while (@$devices_to_check) {
4159         my $to_check = [];
4160         for my $d (@$devices_to_check) {
4161             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4162             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4163
4164             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4165             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4166         }
4167         $devices_to_check = $to_check;
4168     }
4169
4170     my $resblock = mon_cmd($vmid, 'query-block');
4171     foreach my $block (@$resblock) {
4172         if($block->{device} =~ m/^drive-(\S+)/){
4173                 $devices->{$1} = 1;
4174         }
4175     }
4176
4177     my $resmice = mon_cmd($vmid, 'query-mice');
4178     foreach my $mice (@$resmice) {
4179         if ($mice->{name} eq 'QEMU HID Tablet') {
4180             $devices->{tablet} = 1;
4181             last;
4182         }
4183     }
4184
4185     # for usb devices there is no query-usb
4186     # but we can iterate over the entries in
4187     # qom-list path=/machine/peripheral
4188     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4189     foreach my $per (@$resperipheral) {
4190         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4191             $devices->{$per->{name}} = 1;
4192         }
4193     }
4194
4195     return $devices;
4196 }
4197
4198 sub vm_deviceplug {
4199     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4200
4201     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4202
4203     my $devices_list = vm_devices_list($vmid);
4204     return 1 if defined($devices_list->{$deviceid});
4205
4206     # add PCI bridge if we need it for the device
4207     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4208
4209     if ($deviceid eq 'tablet') {
4210         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4211     } elsif ($deviceid eq 'keyboard') {
4212         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4213     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4214         my $id = $1;
4215         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4216         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4217     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4218         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4219     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4220         qemu_iothread_add($vmid, $deviceid, $device);
4221
4222         qemu_driveadd($storecfg, $vmid, $device);
4223         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4224
4225         qemu_deviceadd($vmid, $devicefull);
4226         eval { qemu_deviceaddverify($vmid, $deviceid); };
4227         if (my $err = $@) {
4228             eval { qemu_drivedel($vmid, $deviceid); };
4229             warn $@ if $@;
4230             die $err;
4231         }
4232     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4233         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4234         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4235         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4236
4237         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4238
4239         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4240             qemu_iothread_add($vmid, $deviceid, $device);
4241             $devicefull .= ",iothread=iothread-$deviceid";
4242         }
4243
4244         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4245             $devicefull .= ",num_queues=$device->{queues}";
4246         }
4247
4248         qemu_deviceadd($vmid, $devicefull);
4249         qemu_deviceaddverify($vmid, $deviceid);
4250     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4251         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4252         qemu_driveadd($storecfg, $vmid, $device);
4253
4254         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4255         eval { qemu_deviceadd($vmid, $devicefull); };
4256         if (my $err = $@) {
4257             eval { qemu_drivedel($vmid, $deviceid); };
4258             warn $@ if $@;
4259             die $err;
4260         }
4261     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4262         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4263
4264         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4265         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4266         my $use_old_bios_files = undef;
4267         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4268
4269         my $netdevicefull = print_netdevice_full(
4270             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4271         qemu_deviceadd($vmid, $netdevicefull);
4272         eval {
4273             qemu_deviceaddverify($vmid, $deviceid);
4274             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4275         };
4276         if (my $err = $@) {
4277             eval { qemu_netdevdel($vmid, $deviceid); };
4278             warn $@ if $@;
4279             die $err;
4280         }
4281     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4282         my $bridgeid = $2;
4283         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4284         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4285
4286         qemu_deviceadd($vmid, $devicefull);
4287         qemu_deviceaddverify($vmid, $deviceid);
4288     } else {
4289         die "can't hotplug device '$deviceid'\n";
4290     }
4291
4292     return 1;
4293 }
4294
4295 # fixme: this should raise exceptions on error!
4296 sub vm_deviceunplug {
4297     my ($vmid, $conf, $deviceid) = @_;
4298
4299     my $devices_list = vm_devices_list($vmid);
4300     return 1 if !defined($devices_list->{$deviceid});
4301
4302     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4303     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4304
4305     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4306         qemu_devicedel($vmid, $deviceid);
4307     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4308         qemu_devicedel($vmid, $deviceid);
4309         qemu_devicedelverify($vmid, $deviceid);
4310     } elsif ($deviceid =~ m/^usb\d+$/) {
4311         qemu_devicedel($vmid, $deviceid);
4312         qemu_devicedelverify($vmid, $deviceid);
4313     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4314         my $device = parse_drive($deviceid, $conf->{$deviceid});
4315
4316         qemu_devicedel($vmid, $deviceid);
4317         qemu_devicedelverify($vmid, $deviceid);
4318         qemu_drivedel($vmid, $deviceid);
4319         qemu_iothread_del($vmid, $deviceid, $device);
4320     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4321         qemu_devicedel($vmid, $deviceid);
4322         qemu_devicedelverify($vmid, $deviceid);
4323     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4324         my $device = parse_drive($deviceid, $conf->{$deviceid});
4325
4326         qemu_devicedel($vmid, $deviceid);
4327         qemu_devicedelverify($vmid, $deviceid);
4328         qemu_drivedel($vmid, $deviceid);
4329         qemu_deletescsihw($conf, $vmid, $deviceid);
4330
4331         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4332             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4333     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4334         qemu_devicedel($vmid, $deviceid);
4335         qemu_devicedelverify($vmid, $deviceid);
4336         qemu_netdevdel($vmid, $deviceid);
4337     } else {
4338         die "can't unplug device '$deviceid'\n";
4339     }
4340
4341     return 1;
4342 }
4343
4344 sub qemu_spice_usbredir_chardev_add {
4345     my ($vmid, $id) = @_;
4346
4347     mon_cmd($vmid, "chardev-add" , (
4348         id => $id,
4349         backend => {
4350             type => 'spicevmc',
4351             data => {
4352                 type => "usbredir",
4353             },
4354         },
4355     ));
4356 }
4357
4358 sub qemu_iothread_add {
4359     my ($vmid, $deviceid, $device) = @_;
4360
4361     if ($device->{iothread}) {
4362         my $iothreads = vm_iothreads_list($vmid);
4363         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4364     }
4365 }
4366
4367 sub qemu_iothread_del {
4368     my ($vmid, $deviceid, $device) = @_;
4369
4370     if ($device->{iothread}) {
4371         my $iothreads = vm_iothreads_list($vmid);
4372         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4373     }
4374 }
4375
4376 sub qemu_driveadd {
4377     my ($storecfg, $vmid, $device) = @_;
4378
4379     my $kvmver = get_running_qemu_version($vmid);
4380     my $io_uring = min_version($kvmver, 6, 0);
4381     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4382     $drive =~ s/\\/\\\\/g;
4383     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4384
4385     # If the command succeeds qemu prints: "OK"
4386     return 1 if $ret =~ m/OK/s;
4387
4388     die "adding drive failed: $ret\n";
4389 }
4390
4391 sub qemu_drivedel {
4392     my ($vmid, $deviceid) = @_;
4393
4394     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4395     $ret =~ s/^\s+//;
4396
4397     return 1 if $ret eq "";
4398
4399     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4400     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4401
4402     die "deleting drive $deviceid failed : $ret\n";
4403 }
4404
4405 sub qemu_deviceaddverify {
4406     my ($vmid, $deviceid) = @_;
4407
4408     for (my $i = 0; $i <= 5; $i++) {
4409          my $devices_list = vm_devices_list($vmid);
4410          return 1 if defined($devices_list->{$deviceid});
4411          sleep 1;
4412     }
4413
4414     die "error on hotplug device '$deviceid'\n";
4415 }
4416
4417
4418 sub qemu_devicedelverify {
4419     my ($vmid, $deviceid) = @_;
4420
4421     # need to verify that the device is correctly removed as device_del
4422     # is async and empty return is not reliable
4423
4424     for (my $i = 0; $i <= 5; $i++) {
4425          my $devices_list = vm_devices_list($vmid);
4426          return 1 if !defined($devices_list->{$deviceid});
4427          sleep 1;
4428     }
4429
4430     die "error on hot-unplugging device '$deviceid'\n";
4431 }
4432
4433 sub qemu_findorcreatescsihw {
4434     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4435
4436     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4437
4438     my $scsihwid="$controller_prefix$controller";
4439     my $devices_list = vm_devices_list($vmid);
4440
4441     if (!defined($devices_list->{$scsihwid})) {
4442         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4443     }
4444
4445     return 1;
4446 }
4447
4448 sub qemu_deletescsihw {
4449     my ($conf, $vmid, $opt) = @_;
4450
4451     my $device = parse_drive($opt, $conf->{$opt});
4452
4453     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4454         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4455         return 1;
4456     }
4457
4458     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4459
4460     my $devices_list = vm_devices_list($vmid);
4461     foreach my $opt (keys %{$devices_list}) {
4462         if (is_valid_drivename($opt)) {
4463             my $drive = parse_drive($opt, $conf->{$opt});
4464             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4465                 return 1;
4466             }
4467         }
4468     }
4469
4470     my $scsihwid="scsihw$controller";
4471
4472     vm_deviceunplug($vmid, $conf, $scsihwid);
4473
4474     return 1;
4475 }
4476
4477 sub qemu_add_pci_bridge {
4478     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4479
4480     my $bridges = {};
4481
4482     my $bridgeid;
4483
4484     print_pci_addr($device, $bridges, $arch, $machine_type);
4485
4486     while (my ($k, $v) = each %$bridges) {
4487         $bridgeid = $k;
4488     }
4489     return 1 if !defined($bridgeid) || $bridgeid < 1;
4490
4491     my $bridge = "pci.$bridgeid";
4492     my $devices_list = vm_devices_list($vmid);
4493
4494     if (!defined($devices_list->{$bridge})) {
4495         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4496     }
4497
4498     return 1;
4499 }
4500
4501 sub qemu_set_link_status {
4502     my ($vmid, $device, $up) = @_;
4503
4504     mon_cmd($vmid, "set_link", name => $device,
4505                up => $up ? JSON::true : JSON::false);
4506 }
4507
4508 sub qemu_netdevadd {
4509     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4510
4511     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4512     my %options =  split(/[=,]/, $netdev);
4513
4514     if (defined(my $vhost = $options{vhost})) {
4515         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4516     }
4517
4518     if (defined(my $queues = $options{queues})) {
4519         $options{queues} = $queues + 0;
4520     }
4521
4522     mon_cmd($vmid, "netdev_add",  %options);
4523     return 1;
4524 }
4525
4526 sub qemu_netdevdel {
4527     my ($vmid, $deviceid) = @_;
4528
4529     mon_cmd($vmid, "netdev_del", id => $deviceid);
4530 }
4531
4532 sub qemu_usb_hotplug {
4533     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4534
4535     return if !$device;
4536
4537     # remove the old one first
4538     vm_deviceunplug($vmid, $conf, $deviceid);
4539
4540     # check if xhci controller is necessary and available
4541     my $devicelist = vm_devices_list($vmid);
4542
4543     if (!$devicelist->{xhci}) {
4544         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4545         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4546     }
4547
4548     # add the new one
4549     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4550 }
4551
4552 sub qemu_cpu_hotplug {
4553     my ($vmid, $conf, $vcpus) = @_;
4554
4555     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4556
4557     my $sockets = 1;
4558     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4559     $sockets = $conf->{sockets} if  $conf->{sockets};
4560     my $cores = $conf->{cores} || 1;
4561     my $maxcpus = $sockets * $cores;
4562
4563     $vcpus = $maxcpus if !$vcpus;
4564
4565     die "you can't add more vcpus than maxcpus\n"
4566         if $vcpus > $maxcpus;
4567
4568     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4569
4570     if ($vcpus < $currentvcpus) {
4571
4572         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4573
4574             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4575                 qemu_devicedel($vmid, "cpu$i");
4576                 my $retry = 0;
4577                 my $currentrunningvcpus = undef;
4578                 while (1) {
4579                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4580                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4581                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4582                     $retry++;
4583                     sleep 1;
4584                 }
4585                 #update conf after each succesfull cpu unplug
4586                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4587                 PVE::QemuConfig->write_config($vmid, $conf);
4588             }
4589         } else {
4590             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4591         }
4592
4593         return;
4594     }
4595
4596     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4597     die "vcpus in running vm does not match its configuration\n"
4598         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4599
4600     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4601         my $arch = get_vm_arch($conf);
4602
4603         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4604             my $cpustr = print_cpu_device($conf, $arch, $i);
4605             qemu_deviceadd($vmid, $cpustr);
4606
4607             my $retry = 0;
4608             my $currentrunningvcpus = undef;
4609             while (1) {
4610                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4611                 last if scalar(@{$currentrunningvcpus}) == $i;
4612                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4613                 sleep 1;
4614                 $retry++;
4615             }
4616             #update conf after each succesfull cpu hotplug
4617             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4618             PVE::QemuConfig->write_config($vmid, $conf);
4619         }
4620     } else {
4621
4622         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4623             mon_cmd($vmid, "cpu-add", id => int($i));
4624         }
4625     }
4626 }
4627
4628 sub qemu_block_set_io_throttle {
4629     my ($vmid, $deviceid,
4630         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4631         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4632         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4633         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4634
4635     return if !check_running($vmid) ;
4636
4637     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4638         bps => int($bps),
4639         bps_rd => int($bps_rd),
4640         bps_wr => int($bps_wr),
4641         iops => int($iops),
4642         iops_rd => int($iops_rd),
4643         iops_wr => int($iops_wr),
4644         bps_max => int($bps_max),
4645         bps_rd_max => int($bps_rd_max),
4646         bps_wr_max => int($bps_wr_max),
4647         iops_max => int($iops_max),
4648         iops_rd_max => int($iops_rd_max),
4649         iops_wr_max => int($iops_wr_max),
4650         bps_max_length => int($bps_max_length),
4651         bps_rd_max_length => int($bps_rd_max_length),
4652         bps_wr_max_length => int($bps_wr_max_length),
4653         iops_max_length => int($iops_max_length),
4654         iops_rd_max_length => int($iops_rd_max_length),
4655         iops_wr_max_length => int($iops_wr_max_length),
4656     );
4657
4658 }
4659
4660 sub qemu_block_resize {
4661     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4662
4663     my $running = check_running($vmid);
4664
4665     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4666
4667     return if !$running;
4668
4669     my $padding = (1024 - $size % 1024) % 1024;
4670     $size = $size + $padding;
4671
4672     mon_cmd(
4673         $vmid,
4674         "block_resize",
4675         device => $deviceid,
4676         size => int($size),
4677         timeout => 60,
4678     );
4679 }
4680
4681 sub qemu_volume_snapshot {
4682     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4683
4684     my $running = check_running($vmid);
4685
4686     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4687         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4688     } else {
4689         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4690     }
4691 }
4692
4693 sub qemu_volume_snapshot_delete {
4694     my ($vmid, $storecfg, $volid, $snap) = @_;
4695
4696     my $running = check_running($vmid);
4697     my $attached_deviceid;
4698
4699     if ($running) {
4700         my $conf = PVE::QemuConfig->load_config($vmid);
4701         PVE::QemuConfig->foreach_volume($conf, sub {
4702             my ($ds, $drive) = @_;
4703             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4704         });
4705     }
4706
4707     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4708         mon_cmd(
4709             $vmid,
4710             'blockdev-snapshot-delete-internal-sync',
4711             device => $attached_deviceid,
4712             name => $snap,
4713         );
4714     } else {
4715         PVE::Storage::volume_snapshot_delete(
4716             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4717     }
4718 }
4719
4720 sub set_migration_caps {
4721     my ($vmid, $savevm) = @_;
4722
4723     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4724
4725     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4726     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4727
4728     my $cap_ref = [];
4729
4730     my $enabled_cap = {
4731         "auto-converge" => 1,
4732         "xbzrle" => 1,
4733         "x-rdma-pin-all" => 0,
4734         "zero-blocks" => 0,
4735         "compress" => 0,
4736         "dirty-bitmaps" => $dirty_bitmaps,
4737     };
4738
4739     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4740
4741     for my $supported_capability (@$supported_capabilities) {
4742         push @$cap_ref, {
4743             capability => $supported_capability->{capability},
4744             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4745         };
4746     }
4747
4748     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4749 }
4750
4751 sub foreach_volid {
4752     my ($conf, $func, @param) = @_;
4753
4754     my $volhash = {};
4755
4756     my $test_volid = sub {
4757         my ($key, $drive, $snapname, $pending) = @_;
4758
4759         my $volid = $drive->{file};
4760         return if !$volid;
4761
4762         $volhash->{$volid}->{cdrom} //= 1;
4763         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4764
4765         my $replicate = $drive->{replicate} // 1;
4766         $volhash->{$volid}->{replicate} //= 0;
4767         $volhash->{$volid}->{replicate} = 1 if $replicate;
4768
4769         $volhash->{$volid}->{shared} //= 0;
4770         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4771
4772         $volhash->{$volid}->{is_unused} //= 0;
4773         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4774
4775         $volhash->{$volid}->{is_attached} //= 0;
4776         $volhash->{$volid}->{is_attached} = 1
4777             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4778
4779         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4780             if defined($snapname);
4781
4782         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4783
4784         my $size = $drive->{size};
4785         $volhash->{$volid}->{size} //= $size if $size;
4786
4787         $volhash->{$volid}->{is_vmstate} //= 0;
4788         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4789
4790         $volhash->{$volid}->{is_tpmstate} //= 0;
4791         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4792
4793         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4794     };
4795
4796     my $include_opts = {
4797         extra_keys => ['vmstate'],
4798         include_unused => 1,
4799     };
4800
4801     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4802
4803     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4804         if defined($conf->{pending}) && $conf->{pending}->%*;
4805
4806     foreach my $snapname (keys %{$conf->{snapshots}}) {
4807         my $snap = $conf->{snapshots}->{$snapname};
4808         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4809     }
4810
4811     foreach my $volid (keys %$volhash) {
4812         &$func($volid, $volhash->{$volid}, @param);
4813     }
4814 }
4815
4816 my $fast_plug_option = {
4817     'description' => 1,
4818     'hookscript' => 1,
4819     'lock' => 1,
4820     'migrate_downtime' => 1,
4821     'migrate_speed' => 1,
4822     'name' => 1,
4823     'onboot' => 1,
4824     'protection' => 1,
4825     'shares' => 1,
4826     'startup' => 1,
4827     'tags' => 1,
4828     'vmstatestorage' => 1,
4829 };
4830
4831 for my $opt (keys %$confdesc_cloudinit) {
4832     $fast_plug_option->{$opt} = 1;
4833 };
4834
4835 # hotplug changes in [PENDING]
4836 # $selection hash can be used to only apply specified options, for
4837 # example: { cores => 1 } (only apply changed 'cores')
4838 # $errors ref is used to return error messages
4839 sub vmconfig_hotplug_pending {
4840     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4841
4842     my $defaults = load_defaults();
4843     my $arch = get_vm_arch($conf);
4844     my $machine_type = get_vm_machine($conf, undef, $arch);
4845
4846     # commit values which do not have any impact on running VM first
4847     # Note: those option cannot raise errors, we we do not care about
4848     # $selection and always apply them.
4849
4850     my $add_error = sub {
4851         my ($opt, $msg) = @_;
4852         $errors->{$opt} = "hotplug problem - $msg";
4853     };
4854
4855     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4856
4857     my $cloudinit_record_changed = sub {
4858         my ($conf, $opt, $old, $new) = @_;
4859         return if !$cloudinit_pending_properties->{$opt};
4860
4861         my $ci = ($conf->{cloudinit} //= {});
4862
4863         my $recorded = $ci->{$opt};
4864         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4865
4866         if (defined($new)) {
4867             if (defined($old)) {
4868                 # an existing value is being modified
4869                 if (defined($recorded)) {
4870                     # the value was already not in sync
4871                     if ($new eq $recorded) {
4872                         # a value is being reverted to the cloud-init state:
4873                         delete $ci->{$opt};
4874                         delete $added{$opt};
4875                     } else {
4876                         # the value was changed multiple times, do nothing
4877                     }
4878                 } elsif ($added{$opt}) {
4879                     # the value had been marked as added and is being changed, do nothing
4880                 } else {
4881                     # the value is new, record it:
4882                     $ci->{$opt} = $old;
4883                 }
4884             } else {
4885                 # a new value is being added
4886                 if (defined($recorded)) {
4887                     # it was already not in sync
4888                     if ($new eq $recorded) {
4889                         # a value is being reverted to the cloud-init state:
4890                         delete $ci->{$opt};
4891                         delete $added{$opt};
4892                     } else {
4893                         # the value had temporarily been removed, do nothing
4894                     }
4895                 } elsif ($added{$opt}) {
4896                     # the value had been marked as added already, do nothing
4897                 } else {
4898                     # the value is new, add it
4899                     $added{$opt} = 1;
4900                 }
4901             }
4902         } elsif (!defined($old)) {
4903             # a non-existent value is being removed? ignore...
4904         } else {
4905             # a value is being deleted
4906             if (defined($recorded)) {
4907                 # a value was already recorded, just keep it
4908             } elsif ($added{$opt}) {
4909                 # the value was marked as added, remove it
4910                 delete $added{$opt};
4911             } else {
4912                 # a previously unrecorded value is being removed, record the old value:
4913                 $ci->{$opt} = $old;
4914             }
4915         }
4916
4917         my $added = join(',', sort keys %added);
4918         $ci->{added} = $added if length($added);
4919     };
4920
4921     my $changes = 0;
4922     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4923         if ($fast_plug_option->{$opt}) {
4924             my $new = delete $conf->{pending}->{$opt};
4925             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4926             $conf->{$opt} = $new;
4927             $changes = 1;
4928         }
4929     }
4930
4931     if ($changes) {
4932         PVE::QemuConfig->write_config($vmid, $conf);
4933     }
4934
4935     my $ostype = $conf->{ostype};
4936     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4937     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4938     my $usb_hotplug = $hotplug_features->{usb}
4939         && min_version($version, 7, 1)
4940         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4941
4942     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4943     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4944
4945     foreach my $opt (sort keys %$pending_delete_hash) {
4946         next if $selection && !$selection->{$opt};
4947         my $force = $pending_delete_hash->{$opt}->{force};
4948         eval {
4949             if ($opt eq 'hotplug') {
4950                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4951             } elsif ($opt eq 'tablet') {
4952                 die "skip\n" if !$hotplug_features->{usb};
4953                 if ($defaults->{tablet}) {
4954                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4955                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4956                         if $arch eq 'aarch64';
4957                 } else {
4958                     vm_deviceunplug($vmid, $conf, 'tablet');
4959                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4960                 }
4961             } elsif ($opt =~ m/^usb(\d+)$/) {
4962                 my $index = $1;
4963                 die "skip\n" if !$usb_hotplug;
4964                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4965                 vm_deviceunplug($vmid, $conf, $opt);
4966             } elsif ($opt eq 'vcpus') {
4967                 die "skip\n" if !$hotplug_features->{cpu};
4968                 qemu_cpu_hotplug($vmid, $conf, undef);
4969             } elsif ($opt eq 'balloon') {
4970                 # enable balloon device is not hotpluggable
4971                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4972                 # here we reset the ballooning value to memory
4973                 my $balloon = get_current_memory($conf->{memory});
4974                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4975             } elsif ($fast_plug_option->{$opt}) {
4976                 # do nothing
4977             } elsif ($opt =~ m/^net(\d+)$/) {
4978                 die "skip\n" if !$hotplug_features->{network};
4979                 vm_deviceunplug($vmid, $conf, $opt);
4980                 if($have_sdn) {
4981                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4982                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4983                 }
4984             } elsif (is_valid_drivename($opt)) {
4985                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4986                 vm_deviceunplug($vmid, $conf, $opt);
4987                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4988             } elsif ($opt =~ m/^memory$/) {
4989                 die "skip\n" if !$hotplug_features->{memory};
4990                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4991             } elsif ($opt eq 'cpuunits') {
4992                 $cgroup->change_cpu_shares(undef);
4993             } elsif ($opt eq 'cpulimit') {
4994                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4995             } else {
4996                 die "skip\n";
4997             }
4998         };
4999         if (my $err = $@) {
5000             &$add_error($opt, $err) if $err ne "skip\n";
5001         } else {
5002             my $old = delete $conf->{$opt};
5003             $cloudinit_record_changed->($conf, $opt, $old, undef);
5004             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5005         }
5006     }
5007
5008     my $cloudinit_opt;
5009     foreach my $opt (keys %{$conf->{pending}}) {
5010         next if $selection && !$selection->{$opt};
5011         my $value = $conf->{pending}->{$opt};
5012         eval {
5013             if ($opt eq 'hotplug') {
5014                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5015                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5016             } elsif ($opt eq 'tablet') {
5017                 die "skip\n" if !$hotplug_features->{usb};
5018                 if ($value == 1) {
5019                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5020                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5021                         if $arch eq 'aarch64';
5022                 } elsif ($value == 0) {
5023                     vm_deviceunplug($vmid, $conf, 'tablet');
5024                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5025                 }
5026             } elsif ($opt =~ m/^usb(\d+)$/) {
5027                 my $index = $1;
5028                 die "skip\n" if !$usb_hotplug;
5029                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5030                 my $id = $opt;
5031                 if ($d->{host} =~ m/^spice$/i)  {
5032                     $id = "usbredirdev$index";
5033                 }
5034                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5035             } elsif ($opt eq 'vcpus') {
5036                 die "skip\n" if !$hotplug_features->{cpu};
5037                 qemu_cpu_hotplug($vmid, $conf, $value);
5038             } elsif ($opt eq 'balloon') {
5039                 # enable/disable balloning device is not hotpluggable
5040                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5041                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5042                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5043
5044                 # allow manual ballooning if shares is set to zero
5045                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5046                     my $memory = get_current_memory($conf->{memory});
5047                     my $balloon = $conf->{pending}->{balloon} || $memory;
5048                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5049                 }
5050             } elsif ($opt =~ m/^net(\d+)$/) {
5051                 # some changes can be done without hotplug
5052                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5053                                     $vmid, $opt, $value, $arch, $machine_type);
5054             } elsif (is_valid_drivename($opt)) {
5055                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5056                 # some changes can be done without hotplug
5057                 my $drive = parse_drive($opt, $value);
5058                 if (drive_is_cloudinit($drive)) {
5059                     $cloudinit_opt = [$opt, $drive];
5060                     # apply all the other changes first, then generate the cloudinit disk
5061                     die "skip\n";
5062                 }
5063                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5064                                      $vmid, $opt, $value, $arch, $machine_type);
5065             } elsif ($opt =~ m/^memory$/) { #dimms
5066                 die "skip\n" if !$hotplug_features->{memory};
5067                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5068             } elsif ($opt eq 'cpuunits') {
5069                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5070                 $cgroup->change_cpu_shares($new_cpuunits);
5071             } elsif ($opt eq 'cpulimit') {
5072                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5073                 $cgroup->change_cpu_quota($cpulimit, 100000);
5074             } elsif ($opt eq 'agent') {
5075                 vmconfig_update_agent($conf, $opt, $value);
5076             } else {
5077                 die "skip\n";  # skip non-hot-pluggable options
5078             }
5079         };
5080         if (my $err = $@) {
5081             &$add_error($opt, $err) if $err ne "skip\n";
5082         } else {
5083             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5084             $conf->{$opt} = $value;
5085             delete $conf->{pending}->{$opt};
5086         }
5087     }
5088
5089     if (defined($cloudinit_opt)) {
5090         my ($opt, $drive) = @$cloudinit_opt;
5091         my $value = $conf->{pending}->{$opt};
5092         eval {
5093             my $temp = {%$conf, $opt => $value};
5094             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5095             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5096                                  $vmid, $opt, $value, $arch, $machine_type);
5097         };
5098         if (my $err = $@) {
5099             &$add_error($opt, $err) if $err ne "skip\n";
5100         } else {
5101             $conf->{$opt} = $value;
5102             delete $conf->{pending}->{$opt};
5103         }
5104     }
5105
5106     # unplug xhci controller if no usb device is left
5107     if ($usb_hotplug) {
5108         my $has_usb = 0;
5109         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5110             next if !defined($conf->{"usb$i"});
5111             $has_usb = 1;
5112             last;
5113         }
5114         if (!$has_usb) {
5115             vm_deviceunplug($vmid, $conf, 'xhci');
5116         }
5117     }
5118
5119     PVE::QemuConfig->write_config($vmid, $conf);
5120
5121     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5122         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5123     }
5124 }
5125
5126 sub try_deallocate_drive {
5127     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5128
5129     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5130         my $volid = $drive->{file};
5131         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5132             my $sid = PVE::Storage::parse_volume_id($volid);
5133             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5134
5135             # check if the disk is really unused
5136             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5137                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5138             PVE::Storage::vdisk_free($storecfg, $volid);
5139             return 1;
5140         } else {
5141             # If vm is not owner of this disk remove from config
5142             return 1;
5143         }
5144     }
5145
5146     return;
5147 }
5148
5149 sub vmconfig_delete_or_detach_drive {
5150     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5151
5152     my $drive = parse_drive($opt, $conf->{$opt});
5153
5154     my $rpcenv = PVE::RPCEnvironment::get();
5155     my $authuser = $rpcenv->get_user();
5156
5157     if ($force) {
5158         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5159         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5160     } else {
5161         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5162     }
5163 }
5164
5165
5166
5167 sub vmconfig_apply_pending {
5168     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5169
5170     return if !scalar(keys %{$conf->{pending}});
5171
5172     my $add_apply_error = sub {
5173         my ($opt, $msg) = @_;
5174         my $err_msg = "unable to apply pending change $opt : $msg";
5175         $errors->{$opt} = $err_msg;
5176         warn $err_msg;
5177     };
5178
5179     # cold plug
5180
5181     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5182     foreach my $opt (sort keys %$pending_delete_hash) {
5183         my $force = $pending_delete_hash->{$opt}->{force};
5184         eval {
5185             if ($opt =~ m/^unused/) {
5186                 die "internal error";
5187             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5188                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5189             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5190                 if($have_sdn) {
5191                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5192                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5193                     warn if $@;
5194                 }
5195             }
5196         };
5197         if (my $err = $@) {
5198             $add_apply_error->($opt, $err);
5199         } else {
5200             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5201             delete $conf->{$opt};
5202         }
5203     }
5204
5205     PVE::QemuConfig->cleanup_pending($conf);
5206
5207     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5208
5209     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5210         next if $opt eq 'delete'; # just to be sure
5211         eval {
5212             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5213                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5214             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5215                 return if !$have_sdn; # return from eval if SDN is not available
5216
5217                 my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5218                 if ($conf->{$opt}) {
5219                     my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5220
5221                     if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (
5222                         safe_string_ne($old_net->{bridge}, $new_net->{bridge}) ||
5223                         safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})
5224                     )) {
5225                         PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5226                     }
5227                 }
5228                 #fixme: reuse ip if mac change && same bridge
5229                 PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5230             }
5231         };
5232         if (my $err = $@) {
5233             $add_apply_error->($opt, $err);
5234         } else {
5235
5236             if (is_valid_drivename($opt)) {
5237                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5238                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5239             }
5240
5241             $conf->{$opt} = delete $conf->{pending}->{$opt};
5242         }
5243     }
5244
5245     # write all changes at once to avoid unnecessary i/o
5246     PVE::QemuConfig->write_config($vmid, $conf);
5247     if ($generate_cloudinit) {
5248         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5249             # After successful generation and if there were changes to be applied, update the
5250             # config to drop the {cloudinit} entry.
5251             PVE::QemuConfig->write_config($vmid, $conf);
5252         }
5253     }
5254 }
5255
5256 sub vmconfig_update_net {
5257     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5258
5259     my $newnet = parse_net($value);
5260
5261     if ($conf->{$opt}) {
5262         my $oldnet = parse_net($conf->{$opt});
5263
5264         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5265             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5266             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5267             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5268             !($newnet->{bridge} && $oldnet->{bridge})
5269         ) { # bridge/nat mode change
5270
5271             # for non online change, we try to hot-unplug
5272             die "skip\n" if !$hotplug;
5273             vm_deviceunplug($vmid, $conf, $opt);
5274
5275             if ($have_sdn) {
5276                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5277             }
5278
5279         } else {
5280
5281             die "internal error" if $opt !~ m/net(\d+)/;
5282             my $iface = "tap${vmid}i$1";
5283
5284             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5285                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5286                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5287                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})
5288             ) {
5289                 PVE::Network::tap_unplug($iface);
5290
5291                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5292                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5293                     safe_num_ne($oldnet->{tag}, $newnet->{tag})
5294                 ) {
5295                     qemu_set_link_status($vmid, $opt, 0);
5296                 }
5297
5298                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5299                     if ($have_sdn) {
5300                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5301                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5302                     }
5303                 }
5304
5305                 if ($have_sdn) {
5306                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5307                 } else {
5308                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5309                 }
5310
5311                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5312                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5313                     safe_num_ne($oldnet->{tag}, $newnet->{tag})
5314                 ) {
5315                     qemu_set_link_status($vmid, $opt, 1);
5316                 }
5317
5318             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5319                 # Rate can be applied on its own but any change above needs to
5320                 # include the rate in tap_plug since OVS resets everything.
5321                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5322             }
5323
5324             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5325                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5326             }
5327
5328             return 1;
5329         }
5330     }
5331
5332     if ($hotplug) {
5333         if ($have_sdn) {
5334             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5335             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5336         }
5337         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5338     } else {
5339         die "skip\n";
5340     }
5341 }
5342
5343 sub vmconfig_update_agent {
5344     my ($conf, $opt, $value) = @_;
5345
5346     die "skip\n" if !$conf->{$opt};
5347
5348     my $hotplug_options = { fstrim_cloned_disks => 1 };
5349
5350     my $old_agent = parse_guest_agent($conf);
5351     my $agent = parse_guest_agent({$opt => $value});
5352
5353     for my $option (keys %$agent) { # added/changed options
5354         next if defined($hotplug_options->{$option});
5355         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5356     }
5357
5358     for my $option (keys %$old_agent) { # removed options
5359         next if defined($hotplug_options->{$option});
5360         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5361     }
5362
5363     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5364 }
5365
5366 sub vmconfig_update_disk {
5367     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5368
5369     my $drive = parse_drive($opt, $value);
5370
5371     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5372         my $media = $drive->{media} || 'disk';
5373         my $oldmedia = $old_drive->{media} || 'disk';
5374         die "unable to change media type\n" if $media ne $oldmedia;
5375
5376         if (!drive_is_cdrom($old_drive)) {
5377
5378             if ($drive->{file} ne $old_drive->{file}) {
5379
5380                 die "skip\n" if !$hotplug;
5381
5382                 # unplug and register as unused
5383                 vm_deviceunplug($vmid, $conf, $opt);
5384                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5385
5386             } else {
5387                 # update existing disk
5388
5389                 # skip non hotpluggable value
5390                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5391                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5392                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5393                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5394                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5395                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5396                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5397                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5398                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5399                     die "skip\n";
5400                 }
5401
5402                 # apply throttle
5403                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5404                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5405                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5406                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5407                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5408                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5409                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5410                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5411                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5412                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5413                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5414                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5415                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5416                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5417                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5418                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5419                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5420                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5421
5422                     qemu_block_set_io_throttle(
5423                         $vmid,"drive-$opt",
5424                         ($drive->{mbps} || 0)*1024*1024,
5425                         ($drive->{mbps_rd} || 0)*1024*1024,
5426                         ($drive->{mbps_wr} || 0)*1024*1024,
5427                          $drive->{iops} || 0,
5428                          $drive->{iops_rd} || 0,
5429                          $drive->{iops_wr} || 0,
5430                         ($drive->{mbps_max} || 0)*1024*1024,
5431                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5432                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5433                          $drive->{iops_max} || 0,
5434                          $drive->{iops_rd_max} || 0,
5435                          $drive->{iops_wr_max} || 0,
5436                          $drive->{bps_max_length} || 1,
5437                          $drive->{bps_rd_max_length} || 1,
5438                          $drive->{bps_wr_max_length} || 1,
5439                          $drive->{iops_max_length} || 1,
5440                          $drive->{iops_rd_max_length} || 1,
5441                          $drive->{iops_wr_max_length} || 1,
5442                     );
5443
5444                 }
5445
5446                 return 1;
5447             }
5448
5449         } else { # cdrom
5450
5451             if ($drive->{file} eq 'none') {
5452                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5453                 if (drive_is_cloudinit($old_drive)) {
5454                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5455                 }
5456             } else {
5457                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5458
5459                 # force eject if locked
5460                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5461
5462                 if ($path) {
5463                     mon_cmd($vmid, "blockdev-change-medium",
5464                         id => "$opt", filename => "$path");
5465                 }
5466             }
5467
5468             return 1;
5469         }
5470     }
5471
5472     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5473     # hotplug new disks
5474     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5475     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5476 }
5477
5478 sub vmconfig_update_cloudinit_drive {
5479     my ($storecfg, $conf, $vmid) = @_;
5480
5481     my $cloudinit_ds = undef;
5482     my $cloudinit_drive = undef;
5483
5484     PVE::QemuConfig->foreach_volume($conf, sub {
5485         my ($ds, $drive) = @_;
5486         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5487             $cloudinit_ds = $ds;
5488             $cloudinit_drive = $drive;
5489         }
5490     });
5491
5492     return if !$cloudinit_drive;
5493
5494     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5495         PVE::QemuConfig->write_config($vmid, $conf);
5496     }
5497
5498     my $running = PVE::QemuServer::check_running($vmid);
5499
5500     if ($running) {
5501         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5502         if ($path) {
5503             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5504             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5505         }
5506     }
5507 }
5508
5509 # called in locked context by incoming migration
5510 sub vm_migrate_get_nbd_disks {
5511     my ($storecfg, $conf, $replicated_volumes) = @_;
5512
5513     my $local_volumes = {};
5514     PVE::QemuConfig->foreach_volume($conf, sub {
5515         my ($ds, $drive) = @_;
5516
5517         return if drive_is_cdrom($drive);
5518         return if $ds eq 'tpmstate0';
5519
5520         my $volid = $drive->{file};
5521
5522         return if !$volid;
5523
5524         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5525
5526         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5527         return if $scfg->{shared};
5528
5529         my $format = qemu_img_format($scfg, $volname);
5530
5531         # replicated disks re-use existing state via bitmap
5532         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5533         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5534     });
5535     return $local_volumes;
5536 }
5537
5538 # called in locked context by incoming migration
5539 sub vm_migrate_alloc_nbd_disks {
5540     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5541
5542     my $nbd = {};
5543     foreach my $opt (sort keys %$source_volumes) {
5544         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5545
5546         if ($use_existing) {
5547             $nbd->{$opt}->{drivestr} = print_drive($drive);
5548             $nbd->{$opt}->{volid} = $volid;
5549             $nbd->{$opt}->{replicated} = 1;
5550             next;
5551         }
5552
5553         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5554
5555         # order of precedence, filtered by whether storage supports it:
5556         # 1. explicit requested format
5557         # 2. default format of storage
5558         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5559         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5560
5561         my $size = $drive->{size} / 1024;
5562         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5563         my $newdrive = $drive;
5564         $newdrive->{format} = $format;
5565         $newdrive->{file} = $newvolid;
5566         my $drivestr = print_drive($newdrive);
5567         $nbd->{$opt}->{drivestr} = $drivestr;
5568         $nbd->{$opt}->{volid} = $newvolid;
5569     }
5570
5571     return $nbd;
5572 }
5573
5574 # see vm_start_nolock for parameters, additionally:
5575 # migrate_opts:
5576 #   storagemap = parsed storage map for allocating NBD disks
5577 sub vm_start {
5578     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5579
5580     return PVE::QemuConfig->lock_config($vmid, sub {
5581         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5582
5583         die "you can't start a vm if it's a template\n"
5584             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5585
5586         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5587         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5588
5589         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5590
5591         if ($has_backup_lock && $running) {
5592             # a backup is currently running, attempt to start the guest in the
5593             # existing QEMU instance
5594             return vm_resume($vmid);
5595         }
5596
5597         PVE::QemuConfig->check_lock($conf)
5598             if !($params->{skiplock} || $has_suspended_lock);
5599
5600         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5601
5602         die "VM $vmid already running\n" if $running;
5603
5604         if (my $storagemap = $migrate_opts->{storagemap}) {
5605             my $replicated = $migrate_opts->{replicated_volumes};
5606             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5607             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5608
5609             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5610                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5611             }
5612         }
5613
5614         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5615     });
5616 }
5617
5618
5619 # params:
5620 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5621 #   skiplock => 0/1, skip checking for config lock
5622 #   skiptemplate => 0/1, skip checking whether VM is template
5623 #   forcemachine => to force QEMU machine (rollback/migration)
5624 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5625 #   timeout => in seconds
5626 #   paused => start VM in paused state (backup)
5627 #   resume => resume from hibernation
5628 #   live-restore-backing => {
5629 #      sata0 => {
5630 #          name => blockdev-name,
5631 #          blockdev => "arg to the -blockdev command instantiating device named 'name'",
5632 #      },
5633 #      virtio2 => ...
5634 #   }
5635 # migrate_opts:
5636 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5637 #   migratedfrom => source node
5638 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5639 #   network => CIDR of migration network
5640 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5641 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5642 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5643 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5644 #       contained in config
5645 sub vm_start_nolock {
5646     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5647
5648     my $statefile = $params->{statefile};
5649     my $resume = $params->{resume};
5650
5651     my $migratedfrom = $migrate_opts->{migratedfrom};
5652     my $migration_type = $migrate_opts->{type};
5653
5654     my $res = {};
5655
5656     # clean up leftover reboot request files
5657     eval { clear_reboot_request($vmid); };
5658     warn $@ if $@;
5659
5660     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5661         vmconfig_apply_pending($vmid, $conf, $storecfg);
5662         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5663     }
5664
5665     # don't regenerate the ISO if the VM is started as part of a live migration
5666     # this way we can reuse the old ISO with the correct config
5667     if (!$migratedfrom) {
5668         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5669             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5670             # $conf->{cloudinit}, so we could just not do this?
5671             # But we do it above, so for now let's be consistent.
5672             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5673         }
5674     }
5675
5676     # override offline migrated volumes, conf is out of date still
5677     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5678         for my $key (sort keys $offline_volumes->%*) {
5679             my $parsed = parse_drive($key, $conf->{$key});
5680             $parsed->{file} = $offline_volumes->{$key};
5681             $conf->{$key} = print_drive($parsed);
5682         }
5683     }
5684
5685     my $defaults = load_defaults();
5686
5687     # set environment variable useful inside network script
5688     # for remote migration the config is available on the target node!
5689     if (!$migrate_opts->{remote_node}) {
5690         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5691     }
5692
5693     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5694
5695     my $forcemachine = $params->{forcemachine};
5696     my $forcecpu = $params->{forcecpu};
5697     if ($resume) {
5698         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5699         $forcemachine = $conf->{runningmachine};
5700         $forcecpu = $conf->{runningcpu};
5701         print "Resuming suspended VM\n";
5702     }
5703
5704     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5705         $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
5706
5707     my $migration_ip;
5708     my $get_migration_ip = sub {
5709         my ($nodename) = @_;
5710
5711         return $migration_ip if defined($migration_ip);
5712
5713         my $cidr = $migrate_opts->{network};
5714
5715         if (!defined($cidr)) {
5716             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5717             $cidr = $dc_conf->{migration}->{network};
5718         }
5719
5720         if (defined($cidr)) {
5721             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5722
5723             die "could not get IP: no address configured on local " .
5724                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5725
5726             die "could not get IP: multiple addresses configured on local " .
5727                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5728
5729             $migration_ip = @$ips[0];
5730         }
5731
5732         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5733             if !defined($migration_ip);
5734
5735         return $migration_ip;
5736     };
5737
5738     if ($statefile) {
5739         if ($statefile eq 'tcp') {
5740             my $migrate = $res->{migrate} = { proto => 'tcp' };
5741             $migrate->{addr} = "localhost";
5742             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5743             my $nodename = nodename();
5744
5745             if (!defined($migration_type)) {
5746                 if (defined($datacenterconf->{migration}->{type})) {
5747                     $migration_type = $datacenterconf->{migration}->{type};
5748                 } else {
5749                     $migration_type = 'secure';
5750                 }
5751             }
5752
5753             if ($migration_type eq 'insecure') {
5754                 $migrate->{addr} = $get_migration_ip->($nodename);
5755                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5756             }
5757
5758             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5759             # via QMP later
5760             push @$cmd, '-incoming', 'defer';
5761             push @$cmd, '-S';
5762
5763         } elsif ($statefile eq 'unix') {
5764             # should be default for secure migrations as a ssh TCP forward
5765             # tunnel is not deterministic reliable ready and fails regurarly
5766             # to set up in time, so use UNIX socket forwards
5767             my $migrate = $res->{migrate} = { proto => 'unix' };
5768             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5769             unlink $migrate->{addr};
5770
5771             $migrate->{uri} = "unix:$migrate->{addr}";
5772             push @$cmd, '-incoming', $migrate->{uri};
5773             push @$cmd, '-S';
5774
5775         } elsif (-e $statefile) {
5776             push @$cmd, '-loadstate', $statefile;
5777         } else {
5778             my $statepath = PVE::Storage::path($storecfg, $statefile);
5779             push @$vollist, $statefile;
5780             push @$cmd, '-loadstate', $statepath;
5781         }
5782     } elsif ($params->{paused}) {
5783         push @$cmd, '-S';
5784     }
5785
5786     my $memory = get_current_memory($conf->{memory});
5787     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5788
5789     my $pci_reserve_list = [];
5790     for my $device (values $pci_devices->%*) {
5791         next if $device->{mdev}; # we don't reserve for mdev devices
5792         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5793     }
5794
5795     # reserve all PCI IDs before actually doing anything with them
5796     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5797
5798     eval {
5799         my $uuid;
5800         for my $id (sort keys %$pci_devices) {
5801             my $d = $pci_devices->{$id};
5802             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5803
5804             my $chosen_mdev;
5805             for my $dev ($d->{ids}->@*) {
5806                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5807                 if ($d->{mdev}) {
5808                     warn $@ if $@;
5809                     $chosen_mdev = $info;
5810                     last if $chosen_mdev; # if successful, we're done
5811                 } else {
5812                     die $@ if $@;
5813                 }
5814             }
5815
5816             next if !$d->{mdev};
5817             die "could not create mediated device\n" if !defined($chosen_mdev);
5818
5819             # nvidia grid needs the uuid of the mdev as qemu parameter
5820             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5821                 if (defined($conf->{smbios1})) {
5822                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5823                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5824                 }
5825                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5826             }
5827         }
5828         push @$cmd, '-uuid', $uuid if defined($uuid);
5829     };
5830     if (my $err = $@) {
5831         eval { cleanup_pci_devices($vmid, $conf) };
5832         warn $@ if $@;
5833         die $err;
5834     }
5835
5836     PVE::Storage::activate_volumes($storecfg, $vollist);
5837
5838
5839     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5840     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5841     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5842     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5843     # timeout should be more than enough here...
5844     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5845
5846     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5847
5848     my %run_params = (
5849         timeout => $statefile ? undef : $start_timeout,
5850         umask => 0077,
5851         noerr => 1,
5852     );
5853
5854     # when migrating, prefix QEMU output so other side can pick up any
5855     # errors that might occur and show the user
5856     if ($migratedfrom) {
5857         $run_params{quiet} = 1;
5858         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5859     }
5860
5861     my %systemd_properties = (
5862         Slice => 'qemu.slice',
5863         KillMode => 'process',
5864         SendSIGKILL => 0,
5865         TimeoutStopUSec => ULONG_MAX, # infinity
5866     );
5867
5868     if (PVE::CGroup::cgroup_mode() == 2) {
5869         $systemd_properties{CPUWeight} = $cpuunits;
5870     } else {
5871         $systemd_properties{CPUShares} = $cpuunits;
5872     }
5873
5874     if (my $cpulimit = $conf->{cpulimit}) {
5875         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5876     }
5877     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5878
5879     my $run_qemu = sub {
5880         PVE::Tools::run_fork sub {
5881             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5882
5883             my $tpmpid;
5884             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5885                 # start the TPM emulator so QEMU can connect on start
5886                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5887             }
5888
5889             my $exitcode = run_command($cmd, %run_params);
5890             if ($exitcode) {
5891                 if ($tpmpid) {
5892                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5893                     kill 'TERM', $tpmpid;
5894                 }
5895                 die "QEMU exited with code $exitcode\n";
5896             }
5897         };
5898     };
5899
5900     if ($conf->{hugepages}) {
5901
5902         my $code = sub {
5903             my $hotplug_features =
5904                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5905             my $hugepages_topology =
5906                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5907
5908             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5909
5910             PVE::QemuServer::Memory::hugepages_mount();
5911             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5912
5913             eval { $run_qemu->() };
5914             if (my $err = $@) {
5915                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5916                     if !$conf->{keephugepages};
5917                 die $err;
5918             }
5919
5920             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5921                 if !$conf->{keephugepages};
5922         };
5923         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5924
5925     } else {
5926         eval { $run_qemu->() };
5927     }
5928
5929     if (my $err = $@) {
5930         # deactivate volumes if start fails
5931         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5932         warn $@ if $@;
5933         eval { cleanup_pci_devices($vmid, $conf) };
5934         warn $@ if $@;
5935
5936         die "start failed: $err";
5937     }
5938
5939     # re-reserve all PCI IDs now that we can know the actual VM PID
5940     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5941     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5942     warn $@ if $@;
5943
5944     if (defined(my $migrate = $res->{migrate})) {
5945         if ($migrate->{proto} eq 'tcp') {
5946             my $nodename = nodename();
5947             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5948             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5949             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5950             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5951         }
5952         print "migration listens on $migrate->{uri}\n";
5953     } elsif ($statefile) {
5954         eval { mon_cmd($vmid, "cont"); };
5955         warn $@ if $@;
5956     }
5957
5958     #start nbd server for storage migration
5959     if (my $nbd = $migrate_opts->{nbd}) {
5960         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5961
5962         my $migrate_storage_uri;
5963         # nbd_protocol_version > 0 for unix socket support
5964         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5965             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5966             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5967             $migrate_storage_uri = "nbd:unix:$socket_path";
5968             $res->{migrate}->{unix_sockets} = [$socket_path];
5969         } else {
5970             my $nodename = nodename();
5971             my $localip = $get_migration_ip->($nodename);
5972             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5973             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5974
5975             mon_cmd($vmid, "nbd-server-start", addr => {
5976                 type => 'inet',
5977                 data => {
5978                     host => "${localip}",
5979                     port => "${storage_migrate_port}",
5980                 },
5981             });
5982             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5983             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5984         }
5985
5986         my $block_info = mon_cmd($vmid, "query-block");
5987         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5988
5989         foreach my $opt (sort keys %$nbd) {
5990             my $drivestr = $nbd->{$opt}->{drivestr};
5991             my $volid = $nbd->{$opt}->{volid};
5992
5993             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5994
5995             mon_cmd(
5996                 $vmid,
5997                 "block-export-add",
5998                 id => "drive-$opt",
5999                 'node-name' => $block_node,
6000                 writable => JSON::true,
6001                 type => "nbd",
6002                 name => "drive-$opt", # NBD export name
6003             );
6004
6005             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6006             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6007             print "re-using replicated volume: $opt - $volid\n"
6008                 if $nbd->{$opt}->{replicated};
6009
6010             $res->{drives}->{$opt} = $nbd->{$opt};
6011             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6012         }
6013     }
6014
6015     if ($migratedfrom) {
6016         eval {
6017             set_migration_caps($vmid);
6018         };
6019         warn $@ if $@;
6020
6021         if ($spice_port) {
6022             print "spice listens on port $spice_port\n";
6023             $res->{spice_port} = $spice_port;
6024             if ($migrate_opts->{spice_ticket}) {
6025                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6026                     $migrate_opts->{spice_ticket});
6027                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6028             }
6029         }
6030
6031     } else {
6032         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6033             if !$statefile && $conf->{balloon};
6034
6035         foreach my $opt (keys %$conf) {
6036             next if $opt !~  m/^net\d+$/;
6037             my $nicconf = parse_net($conf->{$opt});
6038             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6039         }
6040         add_nets_bridge_fdb($conf, $vmid);
6041     }
6042
6043    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6044         eval {
6045             mon_cmd(
6046                 $vmid,
6047                 'qom-set',
6048                 path => "machine/peripheral/balloon0",
6049                 property => "guest-stats-polling-interval",
6050                 value => 2
6051             );
6052         };
6053         log_warn("could not set polling interval for ballooning - $@") if $@;
6054     }
6055
6056     if ($resume) {
6057         print "Resumed VM, removing state\n";
6058         if (my $vmstate = $conf->{vmstate}) {
6059             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6060             PVE::Storage::vdisk_free($storecfg, $vmstate);
6061         }
6062         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6063         PVE::QemuConfig->write_config($vmid, $conf);
6064     }
6065
6066     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6067
6068     my ($current_machine, $is_deprecated) =
6069         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6070     if ($is_deprecated) {
6071         log_warn(
6072             "current machine version '$current_machine' is deprecated - see the documentation and ".
6073             "change to a newer one",
6074         );
6075     }
6076
6077     return $res;
6078 }
6079
6080 sub vm_commandline {
6081     my ($storecfg, $vmid, $snapname) = @_;
6082
6083     my $conf = PVE::QemuConfig->load_config($vmid);
6084
6085     my ($forcemachine, $forcecpu);
6086     if ($snapname) {
6087         my $snapshot = $conf->{snapshots}->{$snapname};
6088         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6089
6090         # check for machine or CPU overrides in snapshot
6091         $forcemachine = $snapshot->{runningmachine};
6092         $forcecpu = $snapshot->{runningcpu};
6093
6094         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6095
6096         $conf = $snapshot;
6097     }
6098
6099     my $defaults = load_defaults();
6100
6101     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6102
6103     return PVE::Tools::cmd2string($cmd);
6104 }
6105
6106 sub vm_reset {
6107     my ($vmid, $skiplock) = @_;
6108
6109     PVE::QemuConfig->lock_config($vmid, sub {
6110
6111         my $conf = PVE::QemuConfig->load_config($vmid);
6112
6113         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6114
6115         mon_cmd($vmid, "system_reset");
6116     });
6117 }
6118
6119 sub get_vm_volumes {
6120     my ($conf) = @_;
6121
6122     my $vollist = [];
6123     foreach_volid($conf, sub {
6124         my ($volid, $attr) = @_;
6125
6126         return if $volid =~ m|^/|;
6127
6128         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6129         return if !$sid;
6130
6131         push @$vollist, $volid;
6132     });
6133
6134     return $vollist;
6135 }
6136
6137 sub cleanup_pci_devices {
6138     my ($vmid, $conf) = @_;
6139
6140     foreach my $key (keys %$conf) {
6141         next if $key !~ m/^hostpci(\d+)$/;
6142         my $hostpciindex = $1;
6143         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6144         my $d = parse_hostpci($conf->{$key});
6145         if ($d->{mdev}) {
6146             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6147             # don't want to break ABI just for this two liner
6148             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6149
6150             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6151             # out when we do it first. so wait for up to 10 seconds and then try it manually
6152             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6153                 my $count = 0;
6154                 while (-e $dev_sysfs_dir && $count < 10) {
6155                     sleep 1;
6156                     $count++;
6157                 }
6158                 print "waited $count seconds for mediated device driver finishing clean up\n";
6159             }
6160
6161             if (-e $dev_sysfs_dir) {
6162                 print "actively clean up mediated device with UUID $uuid\n";
6163                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6164             }
6165         }
6166     }
6167     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6168 }
6169
6170 sub vm_stop_cleanup {
6171     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6172
6173     eval {
6174
6175         if (!$keepActive) {
6176             my $vollist = get_vm_volumes($conf);
6177             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6178
6179             if (my $tpmdrive = $conf->{tpmstate0}) {
6180                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6181                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6182                 if ($storeid) {
6183                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6184                 }
6185             }
6186         }
6187
6188         foreach my $ext (qw(mon qmp pid vnc qga)) {
6189             unlink "/var/run/qemu-server/${vmid}.$ext";
6190         }
6191
6192         if ($conf->{ivshmem}) {
6193             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6194             # just delete it for now, VMs which have this already open do not
6195             # are affected, but new VMs will get a separated one. If this
6196             # becomes an issue we either add some sort of ref-counting or just
6197             # add a "don't delete on stop" flag to the ivshmem format.
6198             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6199         }
6200
6201         cleanup_pci_devices($vmid, $conf);
6202
6203         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6204     };
6205     warn $@ if $@; # avoid errors - just warn
6206 }
6207
6208 # call only in locked context
6209 sub _do_vm_stop {
6210     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6211
6212     my $pid = check_running($vmid, $nocheck);
6213     return if !$pid;
6214
6215     my $conf;
6216     if (!$nocheck) {
6217         $conf = PVE::QemuConfig->load_config($vmid);
6218         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6219         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6220             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6221             $timeout = $opts->{down} if $opts->{down};
6222         }
6223         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6224     }
6225
6226     eval {
6227         if ($shutdown) {
6228             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6229                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6230             } else {
6231                 mon_cmd($vmid, "system_powerdown");
6232             }
6233         } else {
6234             mon_cmd($vmid, "quit");
6235         }
6236     };
6237     my $err = $@;
6238
6239     if (!$err) {
6240         $timeout = 60 if !defined($timeout);
6241
6242         my $count = 0;
6243         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6244             $count++;
6245             sleep 1;
6246         }
6247
6248         if ($count >= $timeout) {
6249             if ($force) {
6250                 warn "VM still running - terminating now with SIGTERM\n";
6251                 kill 15, $pid;
6252             } else {
6253                 die "VM quit/powerdown failed - got timeout\n";
6254             }
6255         } else {
6256             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6257             return;
6258         }
6259     } else {
6260        if (!check_running($vmid, $nocheck)) {
6261             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6262             return;
6263        }
6264        if ($force) {
6265             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6266             kill 15, $pid;
6267         } else {
6268             die "VM quit/powerdown failed\n";
6269         }
6270     }
6271
6272     # wait again
6273     $timeout = 10;
6274
6275     my $count = 0;
6276     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6277         $count++;
6278         sleep 1;
6279     }
6280
6281     if ($count >= $timeout) {
6282         warn "VM still running - terminating now with SIGKILL\n";
6283         kill 9, $pid;
6284         sleep 1;
6285     }
6286
6287     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6288 }
6289
6290 # Note: use $nocheck to skip tests if VM configuration file exists.
6291 # We need that when migration VMs to other nodes (files already moved)
6292 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6293 sub vm_stop {
6294     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6295
6296     $force = 1 if !defined($force) && !$shutdown;
6297
6298     if ($migratedfrom){
6299         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6300         kill 15, $pid if $pid;
6301         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6302         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6303         return;
6304     }
6305
6306     PVE::QemuConfig->lock_config($vmid, sub {
6307         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6308    });
6309 }
6310
6311 sub vm_reboot {
6312     my ($vmid, $timeout) = @_;
6313
6314     PVE::QemuConfig->lock_config($vmid, sub {
6315         eval {
6316
6317             # only reboot if running, as qmeventd starts it again on a stop event
6318             return if !check_running($vmid);
6319
6320             create_reboot_request($vmid);
6321
6322             my $storecfg = PVE::Storage::config();
6323             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6324
6325         };
6326         if (my $err = $@) {
6327             # avoid that the next normal shutdown will be confused for a reboot
6328             clear_reboot_request($vmid);
6329             die $err;
6330         }
6331    });
6332 }
6333
6334 # note: if using the statestorage parameter, the caller has to check privileges
6335 sub vm_suspend {
6336     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6337
6338     my $conf;
6339     my $path;
6340     my $storecfg;
6341     my $vmstate;
6342
6343     PVE::QemuConfig->lock_config($vmid, sub {
6344
6345         $conf = PVE::QemuConfig->load_config($vmid);
6346
6347         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6348         PVE::QemuConfig->check_lock($conf)
6349             if !($skiplock || $is_backing_up);
6350
6351         die "cannot suspend to disk during backup\n"
6352             if $is_backing_up && $includestate;
6353
6354         if ($includestate) {
6355             $conf->{lock} = 'suspending';
6356             my $date = strftime("%Y-%m-%d", localtime(time()));
6357             $storecfg = PVE::Storage::config();
6358             if (!$statestorage) {
6359                 $statestorage = find_vmstate_storage($conf, $storecfg);
6360                 # check permissions for the storage
6361                 my $rpcenv = PVE::RPCEnvironment::get();
6362                 if ($rpcenv->{type} ne 'cli') {
6363                     my $authuser = $rpcenv->get_user();
6364                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6365                 }
6366             }
6367
6368
6369             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6370                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6371             $path = PVE::Storage::path($storecfg, $vmstate);
6372             PVE::QemuConfig->write_config($vmid, $conf);
6373         } else {
6374             mon_cmd($vmid, "stop");
6375         }
6376     });
6377
6378     if ($includestate) {
6379         # save vm state
6380         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6381
6382         eval {
6383             set_migration_caps($vmid, 1);
6384             mon_cmd($vmid, "savevm-start", statefile => $path);
6385             for(;;) {
6386                 my $state = mon_cmd($vmid, "query-savevm");
6387                 if (!$state->{status}) {
6388                     die "savevm not active\n";
6389                 } elsif ($state->{status} eq 'active') {
6390                     sleep(1);
6391                     next;
6392                 } elsif ($state->{status} eq 'completed') {
6393                     print "State saved, quitting\n";
6394                     last;
6395                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6396                     die "query-savevm failed with error '$state->{error}'\n"
6397                 } else {
6398                     die "query-savevm returned status '$state->{status}'\n";
6399                 }
6400             }
6401         };
6402         my $err = $@;
6403
6404         PVE::QemuConfig->lock_config($vmid, sub {
6405             $conf = PVE::QemuConfig->load_config($vmid);
6406             if ($err) {
6407                 # cleanup, but leave suspending lock, to indicate something went wrong
6408                 eval {
6409                     mon_cmd($vmid, "savevm-end");
6410                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6411                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6412                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6413                     PVE::QemuConfig->write_config($vmid, $conf);
6414                 };
6415                 warn $@ if $@;
6416                 die $err;
6417             }
6418
6419             die "lock changed unexpectedly\n"
6420                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6421
6422             mon_cmd($vmid, "quit");
6423             $conf->{lock} = 'suspended';
6424             PVE::QemuConfig->write_config($vmid, $conf);
6425         });
6426     }
6427 }
6428
6429 # $nocheck is set when called as part of a migration - in this context the
6430 # location of the config file (source or target node) is not deterministic,
6431 # since migration cannot wait for pmxcfs to process the rename
6432 sub vm_resume {
6433     my ($vmid, $skiplock, $nocheck) = @_;
6434
6435     PVE::QemuConfig->lock_config($vmid, sub {
6436         my $res = mon_cmd($vmid, 'query-status');
6437         my $resume_cmd = 'cont';
6438         my $reset = 0;
6439         my $conf;
6440         if ($nocheck) {
6441             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6442             if ($@) {
6443                 my $vmlist = PVE::Cluster::get_vmlist();
6444                 if (exists($vmlist->{ids}->{$vmid})) {
6445                     my $node = $vmlist->{ids}->{$vmid}->{node};
6446                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6447                 }
6448                 if (!$conf) {
6449                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6450                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6451                 }
6452             }
6453         } else {
6454             $conf = PVE::QemuConfig->load_config($vmid);
6455         }
6456
6457         if ($res->{status}) {
6458             return if $res->{status} eq 'running'; # job done, go home
6459             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6460             $reset = 1 if $res->{status} eq 'shutdown';
6461         }
6462
6463         if (!$nocheck) {
6464             PVE::QemuConfig->check_lock($conf)
6465                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6466         }
6467
6468         if ($reset) {
6469             # required if a VM shuts down during a backup and we get a resume
6470             # request before the backup finishes for example
6471             mon_cmd($vmid, "system_reset");
6472         }
6473
6474         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6475
6476         mon_cmd($vmid, $resume_cmd);
6477     });
6478 }
6479
6480 sub vm_sendkey {
6481     my ($vmid, $skiplock, $key) = @_;
6482
6483     PVE::QemuConfig->lock_config($vmid, sub {
6484
6485         my $conf = PVE::QemuConfig->load_config($vmid);
6486
6487         # there is no qmp command, so we use the human monitor command
6488         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6489         die $res if $res ne '';
6490     });
6491 }
6492
6493 sub check_bridge_access {
6494     my ($rpcenv, $authuser, $conf) = @_;
6495
6496     return 1 if $authuser eq 'root@pam';
6497
6498     for my $opt (sort keys $conf->%*) {
6499         next if $opt !~ m/^net\d+$/;
6500         my $net = parse_net($conf->{$opt});
6501         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6502         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6503     }
6504     return 1;
6505 };
6506
6507 sub check_mapping_access {
6508     my ($rpcenv, $user, $conf) = @_;
6509
6510     for my $opt (keys $conf->%*) {
6511         if ($opt =~ m/^usb\d+$/) {
6512             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6513             if (my $host = $device->{host}) {
6514                 die "only root can set '$opt' config for real devices\n"
6515                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6516             } elsif ($device->{mapping}) {
6517                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6518             } else {
6519                 die "either 'host' or 'mapping' must be set.\n";
6520             }
6521         } elsif ($opt =~ m/^hostpci\d+$/) {
6522             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6523             if ($device->{host}) {
6524                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6525             } elsif ($device->{mapping}) {
6526                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6527             } else {
6528                 die "either 'host' or 'mapping' must be set.\n";
6529             }
6530        }
6531    }
6532 };
6533
6534 sub check_restore_permissions {
6535     my ($rpcenv, $user, $conf) = @_;
6536
6537     check_bridge_access($rpcenv, $user, $conf);
6538     check_mapping_access($rpcenv, $user, $conf);
6539 }
6540 # vzdump restore implementaion
6541
6542 sub tar_archive_read_firstfile {
6543     my $archive = shift;
6544
6545     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6546
6547     # try to detect archive type first
6548     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6549         die "unable to open file '$archive'\n";
6550     my $firstfile = <$fh>;
6551     kill 15, $pid;
6552     close $fh;
6553
6554     die "ERROR: archive contaions no data\n" if !$firstfile;
6555     chomp $firstfile;
6556
6557     return $firstfile;
6558 }
6559
6560 sub tar_restore_cleanup {
6561     my ($storecfg, $statfile) = @_;
6562
6563     print STDERR "starting cleanup\n";
6564
6565     if (my $fd = IO::File->new($statfile, "r")) {
6566         while (defined(my $line = <$fd>)) {
6567             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6568                 my $volid = $2;
6569                 eval {
6570                     if ($volid =~ m|^/|) {
6571                         unlink $volid || die 'unlink failed\n';
6572                     } else {
6573                         PVE::Storage::vdisk_free($storecfg, $volid);
6574                     }
6575                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6576                 };
6577                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6578             } else {
6579                 print STDERR "unable to parse line in statfile - $line";
6580             }
6581         }
6582         $fd->close();
6583     }
6584 }
6585
6586 sub restore_file_archive {
6587     my ($archive, $vmid, $user, $opts) = @_;
6588
6589     return restore_vma_archive($archive, $vmid, $user, $opts)
6590         if $archive eq '-';
6591
6592     my $info = PVE::Storage::archive_info($archive);
6593     my $format = $opts->{format} // $info->{format};
6594     my $comp = $info->{compression};
6595
6596     # try to detect archive format
6597     if ($format eq 'tar') {
6598         return restore_tar_archive($archive, $vmid, $user, $opts);
6599     } else {
6600         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6601     }
6602 }
6603
6604 # hepler to remove disks that will not be used after restore
6605 my $restore_cleanup_oldconf = sub {
6606     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6607
6608     my $kept_disks = {};
6609
6610     PVE::QemuConfig->foreach_volume($oldconf, sub {
6611         my ($ds, $drive) = @_;
6612
6613         return if drive_is_cdrom($drive, 1);
6614
6615         my $volid = $drive->{file};
6616         return if !$volid || $volid =~ m|^/|;
6617
6618         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6619         return if !$path || !$owner || ($owner != $vmid);
6620
6621         # Note: only delete disk we want to restore
6622         # other volumes will become unused
6623         if ($virtdev_hash->{$ds}) {
6624             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6625             if (my $err = $@) {
6626                 warn $err;
6627             }
6628         } else {
6629             $kept_disks->{$volid} = 1;
6630         }
6631     });
6632
6633     # after the restore we have no snapshots anymore
6634     for my $snapname (keys $oldconf->{snapshots}->%*) {
6635         my $snap = $oldconf->{snapshots}->{$snapname};
6636         if ($snap->{vmstate}) {
6637             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6638             if (my $err = $@) {
6639                 warn $err;
6640             }
6641         }
6642
6643         for my $volid (keys $kept_disks->%*) {
6644             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6645             warn $@ if $@;
6646         }
6647     }
6648 };
6649
6650 # Helper to parse vzdump backup device hints
6651 #
6652 # $rpcenv: Environment, used to ckeck storage permissions
6653 # $user: User ID, to check storage permissions
6654 # $storecfg: Storage configuration
6655 # $fh: the file handle for reading the configuration
6656 # $devinfo: should contain device sizes for all backu-up'ed devices
6657 # $options: backup options (pool, default storage)
6658 #
6659 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6660 my $parse_backup_hints = sub {
6661     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6662
6663     my $check_storage = sub { # assert if an image can be allocate
6664         my ($storeid, $scfg) = @_;
6665         die "Content type 'images' is not available on storage '$storeid'\n"
6666             if !$scfg->{content}->{images};
6667         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6668             if $user ne 'root@pam';
6669     };
6670
6671     my $virtdev_hash = {};
6672     while (defined(my $line = <$fh>)) {
6673         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6674             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6675             die "archive does not contain data for drive '$virtdev'\n"
6676                 if !$devinfo->{$devname};
6677
6678             if (defined($options->{storage})) {
6679                 $storeid = $options->{storage} || 'local';
6680             } elsif (!$storeid) {
6681                 $storeid = 'local';
6682             }
6683             $format = 'raw' if !$format;
6684             $devinfo->{$devname}->{devname} = $devname;
6685             $devinfo->{$devname}->{virtdev} = $virtdev;
6686             $devinfo->{$devname}->{format} = $format;
6687             $devinfo->{$devname}->{storeid} = $storeid;
6688
6689             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6690             $check_storage->($storeid, $scfg); # permission and content type check
6691
6692             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6693         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6694             my $virtdev = $1;
6695             my $drive = parse_drive($virtdev, $2);
6696
6697             if (drive_is_cloudinit($drive)) {
6698                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6699                 $storeid = $options->{storage} if defined ($options->{storage});
6700                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6701                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6702
6703                 $check_storage->($storeid, $scfg); # permission and content type check
6704
6705                 $virtdev_hash->{$virtdev} = {
6706                     format => $format,
6707                     storeid => $storeid,
6708                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6709                     is_cloudinit => 1,
6710                 };
6711             }
6712         }
6713     }
6714
6715     return $virtdev_hash;
6716 };
6717
6718 # Helper to allocate and activate all volumes required for a restore
6719 #
6720 # $storecfg: Storage configuration
6721 # $virtdev_hash: as returned by parse_backup_hints()
6722 #
6723 # Returns: { $virtdev => $volid }
6724 my $restore_allocate_devices = sub {
6725     my ($storecfg, $virtdev_hash, $vmid) = @_;
6726
6727     my $map = {};
6728     foreach my $virtdev (sort keys %$virtdev_hash) {
6729         my $d = $virtdev_hash->{$virtdev};
6730         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6731         my $storeid = $d->{storeid};
6732         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6733
6734         # test if requested format is supported
6735         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6736         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6737         $d->{format} = $defFormat if !$supported;
6738
6739         my $name;
6740         if ($d->{is_cloudinit}) {
6741             $name = "vm-$vmid-cloudinit";
6742             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6743             if ($scfg->{path}) {
6744                 $name .= ".$d->{format}";
6745             }
6746         }
6747
6748         my $volid = PVE::Storage::vdisk_alloc(
6749             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6750
6751         print STDERR "new volume ID is '$volid'\n";
6752         $d->{volid} = $volid;
6753
6754         PVE::Storage::activate_volumes($storecfg, [$volid]);
6755
6756         $map->{$virtdev} = $volid;
6757     }
6758
6759     return $map;
6760 };
6761
6762 sub restore_update_config_line {
6763     my ($cookie, $map, $line, $unique) = @_;
6764
6765     return '' if $line =~ m/^\#qmdump\#/;
6766     return '' if $line =~ m/^\#vzdump\#/;
6767     return '' if $line =~ m/^lock:/;
6768     return '' if $line =~ m/^unused\d+:/;
6769     return '' if $line =~ m/^parent:/;
6770
6771     my $res = '';
6772
6773     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6774     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6775         # try to convert old 1.X settings
6776         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6777         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6778             my ($model, $macaddr) = split(/\=/, $devconfig);
6779             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6780             my $net = {
6781                 model => $model,
6782                 bridge => "vmbr$ind",
6783                 macaddr => $macaddr,
6784             };
6785             my $netstr = print_net($net);
6786
6787             $res .= "net$cookie->{netcount}: $netstr\n";
6788             $cookie->{netcount}++;
6789         }
6790     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6791         my ($id, $netstr) = ($1, $2);
6792         my $net = parse_net($netstr);
6793         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6794         $netstr = print_net($net);
6795         $res .= "$id: $netstr\n";
6796     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6797         my $virtdev = $1;
6798         my $value = $3;
6799         my $di = parse_drive($virtdev, $value);
6800         if (defined($di->{backup}) && !$di->{backup}) {
6801             $res .= "#$line";
6802         } elsif ($map->{$virtdev}) {
6803             delete $di->{format}; # format can change on restore
6804             $di->{file} = $map->{$virtdev};
6805             $value = print_drive($di);
6806             $res .= "$virtdev: $value\n";
6807         } else {
6808             $res .= $line;
6809         }
6810     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6811         my $vmgenid = $1;
6812         if ($vmgenid ne '0') {
6813             # always generate a new vmgenid if there was a valid one setup
6814             $vmgenid = generate_uuid();
6815         }
6816         $res .= "vmgenid: $vmgenid\n";
6817     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6818         my ($uuid, $uuid_str);
6819         UUID::generate($uuid);
6820         UUID::unparse($uuid, $uuid_str);
6821         my $smbios1 = parse_smbios1($2);
6822         $smbios1->{uuid} = $uuid_str;
6823         $res .= $1.print_smbios1($smbios1)."\n";
6824     } else {
6825         $res .= $line;
6826     }
6827
6828     return $res;
6829 }
6830
6831 my $restore_deactivate_volumes = sub {
6832     my ($storecfg, $virtdev_hash) = @_;
6833
6834     my $vollist = [];
6835     for my $dev (values $virtdev_hash->%*) {
6836         push $vollist->@*, $dev->{volid} if $dev->{volid};
6837     }
6838
6839     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6840     print STDERR $@ if $@;
6841 };
6842
6843 my $restore_destroy_volumes = sub {
6844     my ($storecfg, $virtdev_hash) = @_;
6845
6846     for my $dev (values $virtdev_hash->%*) {
6847         my $volid = $dev->{volid} or next;
6848         eval {
6849             PVE::Storage::vdisk_free($storecfg, $volid);
6850             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6851         };
6852         print STDERR "unable to cleanup '$volid' - $@" if $@;
6853     }
6854 };
6855
6856 sub restore_merge_config {
6857     my ($filename, $backup_conf_raw, $override_conf) = @_;
6858
6859     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6860     for my $key (keys $override_conf->%*) {
6861         $backup_conf->{$key} = $override_conf->{$key};
6862     }
6863
6864     return $backup_conf;
6865 }
6866
6867 sub scan_volids {
6868     my ($cfg, $vmid) = @_;
6869
6870     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6871
6872     my $volid_hash = {};
6873     foreach my $storeid (keys %$info) {
6874         foreach my $item (@{$info->{$storeid}}) {
6875             next if !($item->{volid} && $item->{size});
6876             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6877             $volid_hash->{$item->{volid}} = $item;
6878         }
6879     }
6880
6881     return $volid_hash;
6882 }
6883
6884 sub update_disk_config {
6885     my ($vmid, $conf, $volid_hash) = @_;
6886
6887     my $changes;
6888     my $prefix = "VM $vmid";
6889
6890     # used and unused disks
6891     my $referenced = {};
6892
6893     # Note: it is allowed to define multiple storages with same path (alias), so
6894     # we need to check both 'volid' and real 'path' (two different volid can point
6895     # to the same path).
6896
6897     my $referencedpath = {};
6898
6899     # update size info
6900     PVE::QemuConfig->foreach_volume($conf, sub {
6901         my ($opt, $drive) = @_;
6902
6903         my $volid = $drive->{file};
6904         return if !$volid;
6905         my $volume = $volid_hash->{$volid};
6906
6907         # mark volid as "in-use" for next step
6908         $referenced->{$volid} = 1;
6909         if ($volume && (my $path = $volume->{path})) {
6910             $referencedpath->{$path} = 1;
6911         }
6912
6913         return if drive_is_cdrom($drive);
6914         return if !$volume;
6915
6916         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6917         if (defined($updated)) {
6918             $changes = 1;
6919             $conf->{$opt} = print_drive($updated);
6920             print "$prefix ($opt): $msg\n";
6921         }
6922     });
6923
6924     # remove 'unusedX' entry if volume is used
6925     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6926         my ($opt, $drive) = @_;
6927
6928         my $volid = $drive->{file};
6929         return if !$volid;
6930
6931         my $path;
6932         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6933         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6934             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6935             $changes = 1;
6936             delete $conf->{$opt};
6937         }
6938
6939         $referenced->{$volid} = 1;
6940         $referencedpath->{$path} = 1 if $path;
6941     });
6942
6943     foreach my $volid (sort keys %$volid_hash) {
6944         next if $volid =~ m/vm-$vmid-state-/;
6945         next if $referenced->{$volid};
6946         my $path = $volid_hash->{$volid}->{path};
6947         next if !$path; # just to be sure
6948         next if $referencedpath->{$path};
6949         $changes = 1;
6950         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6951         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6952         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6953     }
6954
6955     return $changes;
6956 }
6957
6958 sub rescan {
6959     my ($vmid, $nolock, $dryrun) = @_;
6960
6961     my $cfg = PVE::Storage::config();
6962
6963     print "rescan volumes...\n";
6964     my $volid_hash = scan_volids($cfg, $vmid);
6965
6966     my $updatefn =  sub {
6967         my ($vmid) = @_;
6968
6969         my $conf = PVE::QemuConfig->load_config($vmid);
6970
6971         PVE::QemuConfig->check_lock($conf);
6972
6973         my $vm_volids = {};
6974         foreach my $volid (keys %$volid_hash) {
6975             my $info = $volid_hash->{$volid};
6976             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6977         }
6978
6979         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6980
6981         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6982     };
6983
6984     if (defined($vmid)) {
6985         if ($nolock) {
6986             &$updatefn($vmid);
6987         } else {
6988             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6989         }
6990     } else {
6991         my $vmlist = config_list();
6992         foreach my $vmid (keys %$vmlist) {
6993             if ($nolock) {
6994                 &$updatefn($vmid);
6995             } else {
6996                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6997             }
6998         }
6999     }
7000 }
7001
7002 sub restore_proxmox_backup_archive {
7003     my ($archive, $vmid, $user, $options) = @_;
7004
7005     my $storecfg = PVE::Storage::config();
7006
7007     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
7008     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7009
7010     my $fingerprint = $scfg->{fingerprint};
7011     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7012
7013     my $repo = PVE::PBSClient::get_repository($scfg);
7014     my $namespace = $scfg->{namespace};
7015
7016     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7017     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7018     local $ENV{PBS_PASSWORD} = $password;
7019     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7020
7021     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7022         PVE::Storage::parse_volname($storecfg, $archive);
7023
7024     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7025
7026     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7027
7028     my $tmpdir = "/var/tmp/vzdumptmp$$";
7029     rmtree $tmpdir;
7030     mkpath $tmpdir;
7031
7032     my $conffile = PVE::QemuConfig->config_file($vmid);
7033      # disable interrupts (always do cleanups)
7034     local $SIG{INT} =
7035         local $SIG{TERM} =
7036         local $SIG{QUIT} =
7037         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7038
7039     # Note: $oldconf is undef if VM does not exists
7040     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7041     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7042     my $new_conf_raw = '';
7043
7044     my $rpcenv = PVE::RPCEnvironment::get();
7045     my $devinfo = {}; # info about drives included in backup
7046     my $virtdev_hash = {}; # info about allocated drives
7047
7048     eval {
7049         # enable interrupts
7050         local $SIG{INT} =
7051             local $SIG{TERM} =
7052             local $SIG{QUIT} =
7053             local $SIG{HUP} =
7054             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7055
7056         my $cfgfn = "$tmpdir/qemu-server.conf";
7057         my $firewall_config_fn = "$tmpdir/fw.conf";
7058         my $index_fn = "$tmpdir/index.json";
7059
7060         my $cmd = "restore";
7061
7062         my $param = [$pbs_backup_name, "index.json", $index_fn];
7063         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7064         my $index = PVE::Tools::file_get_contents($index_fn);
7065         $index = decode_json($index);
7066
7067         foreach my $info (@{$index->{files}}) {
7068             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7069                 my $devname = $1;
7070                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7071                     $devinfo->{$devname}->{size} = $1;
7072                 } else {
7073                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7074                 }
7075             }
7076         }
7077
7078         my $is_qemu_server_backup = scalar(
7079             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7080         );
7081         if (!$is_qemu_server_backup) {
7082             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7083         }
7084         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7085
7086         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7087         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7088
7089         if ($has_firewall_config) {
7090             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7091             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7092
7093             my $pve_firewall_dir = '/etc/pve/firewall';
7094             mkdir $pve_firewall_dir; # make sure the dir exists
7095             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7096         }
7097
7098         my $fh = IO::File->new($cfgfn, "r") ||
7099             die "unable to read qemu-server.conf - $!\n";
7100
7101         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7102
7103         # fixme: rate limit?
7104
7105         # create empty/temp config
7106         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7107
7108         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7109
7110         # allocate volumes
7111         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7112
7113         foreach my $virtdev (sort keys %$virtdev_hash) {
7114             my $d = $virtdev_hash->{$virtdev};
7115             next if $d->{is_cloudinit}; # no need to restore cloudinit
7116
7117             # this fails if storage is unavailable
7118             my $volid = $d->{volid};
7119             my $path = PVE::Storage::path($storecfg, $volid);
7120
7121             # for live-restore we only want to preload the efidisk and TPM state
7122             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7123
7124             my @ns_arg;
7125             if (defined(my $ns = $scfg->{namespace})) {
7126                 @ns_arg = ('--ns', $ns);
7127             }
7128
7129             my $pbs_restore_cmd = [
7130                 '/usr/bin/pbs-restore',
7131                 '--repository', $repo,
7132                 @ns_arg,
7133                 $pbs_backup_name,
7134                 "$d->{devname}.img.fidx",
7135                 $path,
7136                 '--verbose',
7137                 ];
7138
7139             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7140             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7141
7142             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7143                 push @$pbs_restore_cmd, '--skip-zero';
7144             }
7145
7146             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7147             print "restore proxmox backup image: $dbg_cmdstring\n";
7148             run_command($pbs_restore_cmd);
7149         }
7150
7151         $fh->seek(0, 0) || die "seek failed - $!\n";
7152
7153         my $cookie = { netcount => 0 };
7154         while (defined(my $line = <$fh>)) {
7155             $new_conf_raw .= restore_update_config_line(
7156                 $cookie,
7157                 $map,
7158                 $line,
7159                 $options->{unique},
7160             );
7161         }
7162
7163         $fh->close();
7164     };
7165     my $err = $@;
7166
7167     if ($err || !$options->{live}) {
7168         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7169     }
7170
7171     rmtree $tmpdir;
7172
7173     if ($err) {
7174         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7175         die $err;
7176     }
7177
7178     if ($options->{live}) {
7179         # keep lock during live-restore
7180         $new_conf_raw .= "\nlock: create";
7181     }
7182
7183     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7184     check_restore_permissions($rpcenv, $user, $new_conf);
7185     PVE::QemuConfig->write_config($vmid, $new_conf);
7186
7187     eval { rescan($vmid, 1); };
7188     warn $@ if $@;
7189
7190     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7191
7192     if ($options->{live}) {
7193         # enable interrupts
7194         local $SIG{INT} =
7195             local $SIG{TERM} =
7196             local $SIG{QUIT} =
7197             local $SIG{HUP} =
7198             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7199
7200         my $conf = PVE::QemuConfig->load_config($vmid);
7201         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7202
7203         # these special drives are already restored before start
7204         delete $devinfo->{'drive-efidisk0'};
7205         delete $devinfo->{'drive-tpmstate0-backup'};
7206
7207         my $pbs_opts = {
7208             repo => $repo,
7209             keyfile => $keyfile,
7210             snapshot => $pbs_backup_name,
7211             namespace => $namespace,
7212         };
7213         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7214
7215         PVE::QemuConfig->remove_lock($vmid, "create");
7216     }
7217 }
7218
7219 sub pbs_live_restore {
7220     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7221
7222     print "starting VM for live-restore\n";
7223     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7224
7225     my $live_restore_backing = {};
7226     for my $ds (keys %$restored_disks) {
7227         $ds =~ m/^drive-(.*)$/;
7228         my $confname = $1;
7229         my $pbs_conf = {};
7230         $pbs_conf = {
7231             repository => $opts->{repo},
7232             snapshot => $opts->{snapshot},
7233             archive => "$ds.img.fidx",
7234         };
7235         $pbs_conf->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7236         $pbs_conf->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7237
7238         my $drive = parse_drive($confname, $conf->{$confname});
7239         print "restoring '$ds' to '$drive->{file}'\n";
7240
7241         my $pbs_name = "drive-${confname}-pbs";
7242         $live_restore_backing->{$confname} = {
7243             name => $pbs_name,
7244             blockdev => print_pbs_blockdev($pbs_conf, $pbs_name),
7245         };
7246     }
7247
7248     my $drives_streamed = 0;
7249     eval {
7250         # make sure HA doesn't interrupt our restore by stopping the VM
7251         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7252             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7253         }
7254
7255         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7256         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7257         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
7258
7259         my $qmeventd_fd = register_qmeventd_handle($vmid);
7260
7261         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7262         # this will effectively collapse the backing image chain consisting of
7263         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7264         # removes itself once all backing images vanish with 'auto-remove=on')
7265         my $jobs = {};
7266         for my $ds (sort keys %$restored_disks) {
7267             my $job_id = "restore-$ds";
7268             mon_cmd($vmid, 'block-stream',
7269                 'job-id' => $job_id,
7270                 device => "$ds",
7271             );
7272             $jobs->{$job_id} = {};
7273         }
7274
7275         mon_cmd($vmid, 'cont');
7276         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7277
7278         print "restore-drive jobs finished successfully, removing all tracking block devices"
7279             ." to disconnect from Proxmox Backup Server\n";
7280
7281         for my $ds (sort keys %$restored_disks) {
7282             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7283         }
7284
7285         close($qmeventd_fd);
7286     };
7287
7288     my $err = $@;
7289
7290     if ($err) {
7291         warn "An error occurred during live-restore: $err\n";
7292         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7293         die "live-restore failed\n";
7294     }
7295 }
7296
7297 # Inspired by pbs live-restore, this restores with the disks being available as files.
7298 # Theoretically this can also be used to quick-start a full-clone vm if the
7299 # disks are all available as files.
7300 #
7301 # The mapping should provide a path by config entry, such as
7302 # `{ scsi0 => { format => <qcow2|raw|...>, path => "/path/to/file", sata1 => ... } }`
7303 #
7304 # This is used when doing a `create` call with the `--live-import` parameter,
7305 # where the disks get an `import-from=` property. The non-live part is
7306 # therefore already handled in the `$create_disks()` call happening in the
7307 # `create` api call
7308 sub live_import_from_files {
7309     my ($mapping, $vmid, $conf, $restore_options) = @_;
7310
7311     my $live_restore_backing = {};
7312     for my $dev (keys %$mapping) {
7313         die "disk not support for live-restoring: '$dev'\n"
7314             if !is_valid_drivename($dev) || $dev =~ /^(?:efidisk|tpmstate)/;
7315
7316         die "mapping contains disk '$dev' which does not exist in the config\n"
7317             if !exists($conf->{$dev});
7318
7319         my $info = $mapping->{$dev};
7320         my ($format, $path) = $info->@{qw(format path)};
7321         die "missing path for '$dev' mapping\n" if !$path;
7322         die "missing format for '$dev' mapping\n" if !$format;
7323         die "invalid format '$format' for '$dev' mapping\n"
7324             if !grep { $format eq $_ } qw(raw qcow2 vmdk);
7325
7326         $live_restore_backing->{$dev} = {
7327             name => "drive-$dev-restore",
7328             blockdev => "driver=$format,node-name=drive-$dev-restore"
7329             . ",read-only=on"
7330             . ",file.driver=file,file.filename=$path"
7331         };
7332     };
7333
7334     my $storecfg = PVE::Storage::config();
7335     eval {
7336
7337         # make sure HA doesn't interrupt our restore by stopping the VM
7338         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7339             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7340         }
7341
7342         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
7343
7344         # prevent shutdowns from qmeventd when the VM powers off from the inside
7345         my $qmeventd_fd = register_qmeventd_handle($vmid);
7346
7347         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7348         # this will effectively collapse the backing image chain consisting of
7349         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7350         # removes itself once all backing images vanish with 'auto-remove=on')
7351         my $jobs = {};
7352         for my $ds (sort keys %$live_restore_backing) {
7353             my $job_id = "restore-$ds";
7354             mon_cmd($vmid, 'block-stream',
7355                 'job-id' => $job_id,
7356                 device => "drive-$ds",
7357             );
7358             $jobs->{$job_id} = {};
7359         }
7360
7361         mon_cmd($vmid, 'cont');
7362         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7363
7364         print "restore-drive jobs finished successfully, removing all tracking block devices\n";
7365
7366         for my $ds (sort keys %$live_restore_backing) {
7367             mon_cmd($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore");
7368         }
7369
7370         close($qmeventd_fd);
7371     };
7372
7373     my $err = $@;
7374
7375     if ($err) {
7376         warn "An error occurred during live-restore: $err\n";
7377         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7378         die "live-restore failed\n";
7379     }
7380
7381     PVE::QemuConfig->remove_lock($vmid, "import");
7382 }
7383
7384 sub restore_vma_archive {
7385     my ($archive, $vmid, $user, $opts, $comp) = @_;
7386
7387     my $readfrom = $archive;
7388
7389     my $cfg = PVE::Storage::config();
7390     my $commands = [];
7391     my $bwlimit = $opts->{bwlimit};
7392
7393     my $dbg_cmdstring = '';
7394     my $add_pipe = sub {
7395         my ($cmd) = @_;
7396         push @$commands, $cmd;
7397         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7398         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7399         $readfrom = '-';
7400     };
7401
7402     my $input = undef;
7403     if ($archive eq '-') {
7404         $input = '<&STDIN';
7405     } else {
7406         # If we use a backup from a PVE defined storage we also consider that
7407         # storage's rate limit:
7408         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7409         if (defined($volid)) {
7410             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7411             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7412             if ($readlimit) {
7413                 print STDERR "applying read rate limit: $readlimit\n";
7414                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7415                 $add_pipe->($cstream);
7416             }
7417         }
7418     }
7419
7420     if ($comp) {
7421         my $info = PVE::Storage::decompressor_info('vma', $comp);
7422         my $cmd = $info->{decompressor};
7423         push @$cmd, $readfrom;
7424         $add_pipe->($cmd);
7425     }
7426
7427     my $tmpdir = "/var/tmp/vzdumptmp$$";
7428     rmtree $tmpdir;
7429
7430     # disable interrupts (always do cleanups)
7431     local $SIG{INT} =
7432         local $SIG{TERM} =
7433         local $SIG{QUIT} =
7434         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7435
7436     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7437     POSIX::mkfifo($mapfifo, 0600);
7438     my $fifofh;
7439     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7440
7441     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7442
7443     my $devinfo = {}; # info about drives included in backup
7444     my $virtdev_hash = {}; # info about allocated drives
7445
7446     my $rpcenv = PVE::RPCEnvironment::get();
7447
7448     my $conffile = PVE::QemuConfig->config_file($vmid);
7449
7450     # Note: $oldconf is undef if VM does not exist
7451     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7452     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7453     my $new_conf_raw = '';
7454
7455     my %storage_limits;
7456
7457     my $print_devmap = sub {
7458         my $cfgfn = "$tmpdir/qemu-server.conf";
7459
7460         # we can read the config - that is already extracted
7461         my $fh = IO::File->new($cfgfn, "r") ||
7462             die "unable to read qemu-server.conf - $!\n";
7463
7464         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7465         if (-f $fwcfgfn) {
7466             my $pve_firewall_dir = '/etc/pve/firewall';
7467             mkdir $pve_firewall_dir; # make sure the dir exists
7468             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7469         }
7470
7471         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7472
7473         foreach my $info (values %{$virtdev_hash}) {
7474             my $storeid = $info->{storeid};
7475             next if defined($storage_limits{$storeid});
7476
7477             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7478             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7479             $storage_limits{$storeid} = $limit * 1024;
7480         }
7481
7482         foreach my $devname (keys %$devinfo) {
7483             die "found no device mapping information for device '$devname'\n"
7484                 if !$devinfo->{$devname}->{virtdev};
7485         }
7486
7487         # create empty/temp config
7488         if ($oldconf) {
7489             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7490             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7491         }
7492
7493         # allocate volumes
7494         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7495
7496         # print restore information to $fifofh
7497         foreach my $virtdev (sort keys %$virtdev_hash) {
7498             my $d = $virtdev_hash->{$virtdev};
7499             next if $d->{is_cloudinit}; # no need to restore cloudinit
7500
7501             my $storeid = $d->{storeid};
7502             my $volid = $d->{volid};
7503
7504             my $map_opts = '';
7505             if (my $limit = $storage_limits{$storeid}) {
7506                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7507             }
7508
7509             my $write_zeros = 1;
7510             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7511                 $write_zeros = 0;
7512             }
7513
7514             my $path = PVE::Storage::path($cfg, $volid);
7515
7516             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7517
7518             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7519         }
7520
7521         $fh->seek(0, 0) || die "seek failed - $!\n";
7522
7523         my $cookie = { netcount => 0 };
7524         while (defined(my $line = <$fh>)) {
7525             $new_conf_raw .= restore_update_config_line(
7526                 $cookie,
7527                 $map,
7528                 $line,
7529                 $opts->{unique},
7530             );
7531         }
7532
7533         $fh->close();
7534     };
7535
7536     my $oldtimeout;
7537
7538     eval {
7539         # enable interrupts
7540         local $SIG{INT} =
7541             local $SIG{TERM} =
7542             local $SIG{QUIT} =
7543             local $SIG{HUP} =
7544             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7545         local $SIG{ALRM} = sub { die "got timeout\n"; };
7546
7547         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7548
7549         my $parser = sub {
7550             my $line = shift;
7551
7552             print "$line\n";
7553
7554             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7555                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7556                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7557             } elsif ($line =~ m/^CTIME: /) {
7558                 # we correctly received the vma config, so we can disable
7559                 # the timeout now for disk allocation
7560                 alarm($oldtimeout || 0);
7561                 $oldtimeout = undef;
7562                 &$print_devmap();
7563                 print $fifofh "done\n";
7564                 close($fifofh);
7565                 $fifofh = undef;
7566             }
7567         };
7568
7569         print "restore vma archive: $dbg_cmdstring\n";
7570         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7571     };
7572     my $err = $@;
7573
7574     alarm($oldtimeout) if $oldtimeout;
7575
7576     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7577
7578     close($fifofh) if $fifofh;
7579     unlink $mapfifo;
7580     rmtree $tmpdir;
7581
7582     if ($err) {
7583         $restore_destroy_volumes->($cfg, $virtdev_hash);
7584         die $err;
7585     }
7586
7587     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7588     check_restore_permissions($rpcenv, $user, $new_conf);
7589     PVE::QemuConfig->write_config($vmid, $new_conf);
7590
7591     eval { rescan($vmid, 1); };
7592     warn $@ if $@;
7593
7594     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7595 }
7596
7597 sub restore_tar_archive {
7598     my ($archive, $vmid, $user, $opts) = @_;
7599
7600     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7601         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7602         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7603     }
7604
7605     if ($archive ne '-') {
7606         my $firstfile = tar_archive_read_firstfile($archive);
7607         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7608             if $firstfile ne 'qemu-server.conf';
7609     }
7610
7611     my $storecfg = PVE::Storage::config();
7612
7613     # avoid zombie disks when restoring over an existing VM -> cleanup first
7614     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7615     # skiplock=1 because qmrestore has set the 'create' lock itself already
7616     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7617     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7618
7619     my $tocmd = "/usr/lib/qemu-server/qmextract";
7620
7621     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7622     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7623     $tocmd .= ' --prealloc' if $opts->{prealloc};
7624     $tocmd .= ' --info' if $opts->{info};
7625
7626     # tar option "xf" does not autodetect compression when read from STDIN,
7627     # so we pipe to zcat
7628     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7629         PVE::Tools::shellquote("--to-command=$tocmd");
7630
7631     my $tmpdir = "/var/tmp/vzdumptmp$$";
7632     mkpath $tmpdir;
7633
7634     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7635     local $ENV{VZDUMP_VMID} = $vmid;
7636     local $ENV{VZDUMP_USER} = $user;
7637
7638     my $conffile = PVE::QemuConfig->config_file($vmid);
7639     my $new_conf_raw = '';
7640
7641     # disable interrupts (always do cleanups)
7642     local $SIG{INT} =
7643         local $SIG{TERM} =
7644         local $SIG{QUIT} =
7645         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7646
7647     eval {
7648         # enable interrupts
7649         local $SIG{INT} =
7650             local $SIG{TERM} =
7651             local $SIG{QUIT} =
7652             local $SIG{HUP} =
7653             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7654
7655         if ($archive eq '-') {
7656             print "extracting archive from STDIN\n";
7657             run_command($cmd, input => "<&STDIN");
7658         } else {
7659             print "extracting archive '$archive'\n";
7660             run_command($cmd);
7661         }
7662
7663         return if $opts->{info};
7664
7665         # read new mapping
7666         my $map = {};
7667         my $statfile = "$tmpdir/qmrestore.stat";
7668         if (my $fd = IO::File->new($statfile, "r")) {
7669             while (defined (my $line = <$fd>)) {
7670                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7671                     $map->{$1} = $2 if $1;
7672                 } else {
7673                     print STDERR "unable to parse line in statfile - $line\n";
7674                 }
7675             }
7676             $fd->close();
7677         }
7678
7679         my $confsrc = "$tmpdir/qemu-server.conf";
7680
7681         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7682
7683         my $cookie = { netcount => 0 };
7684         while (defined (my $line = <$srcfd>)) {
7685             $new_conf_raw .= restore_update_config_line(
7686                 $cookie,
7687                 $map,
7688                 $line,
7689                 $opts->{unique},
7690             );
7691         }
7692
7693         $srcfd->close();
7694     };
7695     if (my $err = $@) {
7696         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7697         die $err;
7698     }
7699
7700     rmtree $tmpdir;
7701
7702     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7703
7704     PVE::Cluster::cfs_update(); # make sure we read new file
7705
7706     eval { rescan($vmid, 1); };
7707     warn $@ if $@;
7708 };
7709
7710 sub foreach_storage_used_by_vm {
7711     my ($conf, $func) = @_;
7712
7713     my $sidhash = {};
7714
7715     PVE::QemuConfig->foreach_volume($conf, sub {
7716         my ($ds, $drive) = @_;
7717         return if drive_is_cdrom($drive);
7718
7719         my $volid = $drive->{file};
7720
7721         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7722         $sidhash->{$sid} = $sid if $sid;
7723     });
7724
7725     foreach my $sid (sort keys %$sidhash) {
7726         &$func($sid);
7727     }
7728 }
7729
7730 my $qemu_snap_storage = {
7731     rbd => 1,
7732 };
7733 sub do_snapshots_with_qemu {
7734     my ($storecfg, $volid, $deviceid) = @_;
7735
7736     return if $deviceid =~ m/tpmstate0/;
7737
7738     my $storage_name = PVE::Storage::parse_volume_id($volid);
7739     my $scfg = $storecfg->{ids}->{$storage_name};
7740     die "could not find storage '$storage_name'\n" if !defined($scfg);
7741
7742     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7743         return 1;
7744     }
7745
7746     if ($volid =~ m/\.(qcow2|qed)$/){
7747         return 1;
7748     }
7749
7750     return;
7751 }
7752
7753 sub qga_check_running {
7754     my ($vmid, $nowarn) = @_;
7755
7756     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7757     if ($@) {
7758         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7759         return 0;
7760     }
7761     return 1;
7762 }
7763
7764 sub template_create {
7765     my ($vmid, $conf, $disk) = @_;
7766
7767     my $storecfg = PVE::Storage::config();
7768
7769     PVE::QemuConfig->foreach_volume($conf, sub {
7770         my ($ds, $drive) = @_;
7771
7772         return if drive_is_cdrom($drive);
7773         return if $disk && $ds ne $disk;
7774
7775         my $volid = $drive->{file};
7776         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7777
7778         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7779         $drive->{file} = $voliddst;
7780         $conf->{$ds} = print_drive($drive);
7781         PVE::QemuConfig->write_config($vmid, $conf);
7782     });
7783 }
7784
7785 sub convert_iscsi_path {
7786     my ($path) = @_;
7787
7788     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7789         my $portal = $1;
7790         my $target = $2;
7791         my $lun = $3;
7792
7793         my $initiator_name = get_initiator_name();
7794
7795         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7796                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7797     }
7798
7799     die "cannot convert iscsi path '$path', unkown format\n";
7800 }
7801
7802 sub qemu_img_convert {
7803     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7804
7805     my $storecfg = PVE::Storage::config();
7806     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7807     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7808
7809     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7810
7811     my $cachemode;
7812     my $src_path;
7813     my $src_is_iscsi = 0;
7814     my $src_format;
7815
7816     if ($src_storeid) {
7817         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7818         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7819         $src_format = qemu_img_format($src_scfg, $src_volname);
7820         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7821         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7822         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7823     } elsif (-f $src_volid || -b $src_volid) {
7824         $src_path = $src_volid;
7825         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7826             $src_format = $1;
7827         }
7828     }
7829
7830     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7831
7832     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7833     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7834     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7835     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7836
7837     my $cmd = [];
7838     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7839     push @$cmd, '-l', "snapshot.name=$snapname"
7840         if $snapname && $src_format && $src_format eq "qcow2";
7841     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7842     push @$cmd, '-T', $cachemode if defined($cachemode);
7843     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7844
7845     if ($src_is_iscsi) {
7846         push @$cmd, '--image-opts';
7847         $src_path = convert_iscsi_path($src_path);
7848     } elsif ($src_format) {
7849         push @$cmd, '-f', $src_format;
7850     }
7851
7852     if ($dst_is_iscsi) {
7853         push @$cmd, '--target-image-opts';
7854         $dst_path = convert_iscsi_path($dst_path);
7855     } else {
7856         push @$cmd, '-O', $dst_format;
7857     }
7858
7859     push @$cmd, $src_path;
7860
7861     if (!$dst_is_iscsi && $is_zero_initialized) {
7862         push @$cmd, "zeroinit:$dst_path";
7863     } else {
7864         push @$cmd, $dst_path;
7865     }
7866
7867     my $parser = sub {
7868         my $line = shift;
7869         if($line =~ m/\((\S+)\/100\%\)/){
7870             my $percent = $1;
7871             my $transferred = int($size * $percent / 100);
7872             my $total_h = render_bytes($size, 1);
7873             my $transferred_h = render_bytes($transferred, 1);
7874
7875             print "transferred $transferred_h of $total_h ($percent%)\n";
7876         }
7877
7878     };
7879
7880     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7881     my $err = $@;
7882     die "copy failed: $err" if $err;
7883 }
7884
7885 sub qemu_img_format {
7886     my ($scfg, $volname) = @_;
7887
7888     # FIXME: this entire function is kind of weird given that `parse_volname`
7889     # also already gives us a format?
7890     my $is_path_storage = $scfg->{path} || $scfg->{type} eq 'esxi';
7891
7892     if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7893         return $1;
7894     } else {
7895         return "raw";
7896     }
7897 }
7898
7899 sub qemu_drive_mirror {
7900     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7901
7902     $jobs = {} if !$jobs;
7903
7904     my $qemu_target;
7905     my $format;
7906     $jobs->{"drive-$drive"} = {};
7907
7908     if ($dst_volid =~ /^nbd:/) {
7909         $qemu_target = $dst_volid;
7910         $format = "nbd";
7911     } else {
7912         my $storecfg = PVE::Storage::config();
7913         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7914
7915         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7916
7917         $format = qemu_img_format($dst_scfg, $dst_volname);
7918
7919         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7920
7921         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7922     }
7923
7924     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7925     $opts->{format} = $format if $format;
7926
7927     if (defined($src_bitmap)) {
7928         $opts->{sync} = 'incremental';
7929         $opts->{bitmap} = $src_bitmap;
7930         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7931     }
7932
7933     if (defined($bwlimit)) {
7934         $opts->{speed} = $bwlimit * 1024;
7935         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7936     } else {
7937         print "drive mirror is starting for drive-$drive\n";
7938     }
7939
7940     # if a job already runs for this device we get an error, catch it for cleanup
7941     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7942     if (my $err = $@) {
7943         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7944         warn "$@\n" if $@;
7945         die "mirroring error: $err\n";
7946     }
7947
7948     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7949 }
7950
7951 # $completion can be either
7952 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7953 # 'cancel': wait until all jobs are ready, block-job-cancel them
7954 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7955 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7956 sub qemu_drive_mirror_monitor {
7957     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7958
7959     $completion //= 'complete';
7960     $op //= "mirror";
7961
7962     eval {
7963         my $err_complete = 0;
7964
7965         my $starttime = time ();
7966         while (1) {
7967             die "block job ('$op') timed out\n" if $err_complete > 300;
7968
7969             my $stats = mon_cmd($vmid, "query-block-jobs");
7970             my $ctime = time();
7971
7972             my $running_jobs = {};
7973             for my $stat (@$stats) {
7974                 next if $stat->{type} ne $op;
7975                 $running_jobs->{$stat->{device}} = $stat;
7976             }
7977
7978             my $readycounter = 0;
7979
7980             for my $job_id (sort keys %$jobs) {
7981                 my $job = $running_jobs->{$job_id};
7982
7983                 my $vanished = !defined($job);
7984                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7985                 if($complete || ($vanished && $completion eq 'auto')) {
7986                     print "$job_id: $op-job finished\n";
7987                     delete $jobs->{$job_id};
7988                     next;
7989                 }
7990
7991                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7992
7993                 my $busy = $job->{busy};
7994                 my $ready = $job->{ready};
7995                 if (my $total = $job->{len}) {
7996                     my $transferred = $job->{offset} || 0;
7997                     my $remaining = $total - $transferred;
7998                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7999
8000                     my $duration = $ctime - $starttime;
8001                     my $total_h = render_bytes($total, 1);
8002                     my $transferred_h = render_bytes($transferred, 1);
8003
8004                     my $status = sprintf(
8005                         "transferred $transferred_h of $total_h ($percent%%) in %s",
8006                         render_duration($duration),
8007                     );
8008
8009                     if ($ready) {
8010                         if ($busy) {
8011                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
8012                         } else {
8013                             $status .= ", ready";
8014                         }
8015                     }
8016                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
8017                     $jobs->{$job_id}->{ready} = $ready;
8018                 }
8019
8020                 $readycounter++ if $job->{ready};
8021             }
8022
8023             last if scalar(keys %$jobs) == 0;
8024
8025             if ($readycounter == scalar(keys %$jobs)) {
8026                 print "all '$op' jobs are ready\n";
8027
8028                 # do the complete later (or has already been done)
8029                 last if $completion eq 'skip' || $completion eq 'auto';
8030
8031                 if ($vmiddst && $vmiddst != $vmid) {
8032                     my $agent_running = $qga && qga_check_running($vmid);
8033                     if ($agent_running) {
8034                         print "freeze filesystem\n";
8035                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
8036                         warn $@ if $@;
8037                     } else {
8038                         print "suspend vm\n";
8039                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
8040                         warn $@ if $@;
8041                     }
8042
8043                     # if we clone a disk for a new target vm, we don't switch the disk
8044                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
8045
8046                     if ($agent_running) {
8047                         print "unfreeze filesystem\n";
8048                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
8049                         warn $@ if $@;
8050                     } else {
8051                         print "resume vm\n";
8052                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
8053                         warn $@ if $@;
8054                     }
8055
8056                     last;
8057                 } else {
8058
8059                     for my $job_id (sort keys %$jobs) {
8060                         # try to switch the disk if source and destination are on the same guest
8061                         print "$job_id: Completing block job_id...\n";
8062
8063                         my $op;
8064                         if ($completion eq 'complete') {
8065                             $op = 'block-job-complete';
8066                         } elsif ($completion eq 'cancel') {
8067                             $op = 'block-job-cancel';
8068                         } else {
8069                             die "invalid completion value: $completion\n";
8070                         }
8071                         eval { mon_cmd($vmid, $op, device => $job_id) };
8072                         if ($@ =~ m/cannot be completed/) {
8073                             print "$job_id: block job cannot be completed, trying again.\n";
8074                             $err_complete++;
8075                         }else {
8076                             print "$job_id: Completed successfully.\n";
8077                             $jobs->{$job_id}->{complete} = 1;
8078                         }
8079                     }
8080                 }
8081             }
8082             sleep 1;
8083         }
8084     };
8085     my $err = $@;
8086
8087     if ($err) {
8088         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
8089         die "block job ($op) error: $err";
8090     }
8091 }
8092
8093 sub qemu_blockjobs_cancel {
8094     my ($vmid, $jobs) = @_;
8095
8096     foreach my $job (keys %$jobs) {
8097         print "$job: Cancelling block job\n";
8098         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
8099         $jobs->{$job}->{cancel} = 1;
8100     }
8101
8102     while (1) {
8103         my $stats = mon_cmd($vmid, "query-block-jobs");
8104
8105         my $running_jobs = {};
8106         foreach my $stat (@$stats) {
8107             $running_jobs->{$stat->{device}} = $stat;
8108         }
8109
8110         foreach my $job (keys %$jobs) {
8111
8112             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8113                 print "$job: Done.\n";
8114                 delete $jobs->{$job};
8115             }
8116         }
8117
8118         last if scalar(keys %$jobs) == 0;
8119
8120         sleep 1;
8121     }
8122 }
8123
8124 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8125 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8126 my sub clone_disk_check_io_uring {
8127     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8128
8129     return if !$use_drive_mirror;
8130
8131     # Don't complain when not changing storage.
8132     # Assume if it works for the source, it'll work for the target too.
8133     return if $src_storeid eq $dst_storeid;
8134
8135     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8136     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8137
8138     my $cache_direct = drive_uses_cache_direct($src_drive);
8139
8140     my $src_uses_io_uring;
8141     if ($src_drive->{aio}) {
8142         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8143     } else {
8144         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8145     }
8146
8147     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8148         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8149 }
8150
8151 sub clone_disk {
8152     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8153
8154     my ($vmid, $running) = $source->@{qw(vmid running)};
8155     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8156
8157     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8158     my ($storage, $format) = $dest->@{qw(storage format)};
8159
8160     my $unused = defined($src_drivename) && $src_drivename =~ /^unused/;
8161     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused;
8162
8163     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8164         die "cloning from/to EFI disk requires EFI disk\n"
8165             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8166         die "cloning from/to TPM state requires TPM state\n"
8167             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8168
8169         # This would lead to two device nodes in QEMU pointing to the same backing image!
8170         die "cannot change drive name when cloning disk from/to the same VM\n"
8171             if $use_drive_mirror && $vmid == $newvmid;
8172     }
8173
8174     die "cannot move TPM state while VM is running\n"
8175         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8176
8177     my $newvolid;
8178
8179     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8180     print "$src_drivename " if $src_drivename;
8181     print "($drive->{file})\n";
8182
8183     if (!$full) {
8184         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8185         push @$newvollist, $newvolid;
8186     } else {
8187         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8188         my $storeid = $storage || $src_storeid;
8189
8190         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8191
8192         my $name = undef;
8193         my $size = undef;
8194         if (drive_is_cloudinit($drive)) {
8195             $name = "vm-$newvmid-cloudinit";
8196             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8197             if ($scfg->{path}) {
8198                 $name .= ".$dst_format";
8199             }
8200             $snapname = undef;
8201             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8202         } elsif ($dst_drivename eq 'efidisk0') {
8203             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8204         } elsif ($dst_drivename eq 'tpmstate0') {
8205             $dst_format = 'raw';
8206             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8207         } else {
8208             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8209
8210             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8211         }
8212         $newvolid = PVE::Storage::vdisk_alloc(
8213             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8214         );
8215         push @$newvollist, $newvolid;
8216
8217         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8218
8219         if (drive_is_cloudinit($drive)) {
8220             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8221             # if this is the case, we have to complete any block-jobs still there from
8222             # previous drive-mirrors
8223             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8224                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8225             }
8226             goto no_data_clone;
8227         }
8228
8229         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8230         if ($use_drive_mirror) {
8231             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8232                 $completion, $qga, $bwlimit);
8233         } else {
8234             if ($dst_drivename eq 'efidisk0') {
8235                 # the relevant data on the efidisk may be smaller than the source
8236                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8237                 # that is given by the OVMF_VARS.fd
8238                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8239                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8240
8241                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8242
8243                 # better for Ceph if block size is not too small, see bug #3324
8244                 my $bs = 1024*1024;
8245
8246                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8247
8248                 if ($src_format eq 'qcow2' && $snapname) {
8249                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8250                         if !min_version(kvm_user_version(), 6, 2);
8251                     push $cmd->@*, '-l', $snapname;
8252                 }
8253                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8254                 run_command($cmd);
8255             } else {
8256                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8257             }
8258         }
8259     }
8260
8261 no_data_clone:
8262     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8263
8264     my $disk = dclone($drive);
8265     delete $disk->{format};
8266     $disk->{file} = $newvolid;
8267     $disk->{size} = $size if defined($size) && !$unused;
8268
8269     return $disk;
8270 }
8271
8272 sub get_running_qemu_version {
8273     my ($vmid) = @_;
8274     my $res = mon_cmd($vmid, "query-version");
8275     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8276 }
8277
8278 sub qemu_use_old_bios_files {
8279     my ($machine_type) = @_;
8280
8281     return if !$machine_type;
8282
8283     my $use_old_bios_files = undef;
8284
8285     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8286         $machine_type = $1;
8287         $use_old_bios_files = 1;
8288     } else {
8289         my $version = extract_version($machine_type, kvm_user_version());
8290         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8291         # load new efi bios files on migration. So this hack is required to allow
8292         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8293         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8294         $use_old_bios_files = !min_version($version, 2, 4);
8295     }
8296
8297     return ($use_old_bios_files, $machine_type);
8298 }
8299
8300 sub get_efivars_size {
8301     my ($conf, $efidisk) = @_;
8302
8303     my $arch = get_vm_arch($conf);
8304     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8305     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8306     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8307     return -s $ovmf_vars;
8308 }
8309
8310 sub update_efidisk_size {
8311     my ($conf) = @_;
8312
8313     return if !defined($conf->{efidisk0});
8314
8315     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8316     $disk->{size} = get_efivars_size($conf);
8317     $conf->{efidisk0} = print_drive($disk);
8318
8319     return;
8320 }
8321
8322 sub update_tpmstate_size {
8323     my ($conf) = @_;
8324
8325     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8326     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8327     $conf->{tpmstate0} = print_drive($disk);
8328 }
8329
8330 sub create_efidisk($$$$$$$) {
8331     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8332
8333     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8334
8335     my $vars_size_b = -s $ovmf_vars;
8336     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8337     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8338     PVE::Storage::activate_volumes($storecfg, [$volid]);
8339
8340     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8341     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8342
8343     return ($volid, $size/1024);
8344 }
8345
8346 sub vm_iothreads_list {
8347     my ($vmid) = @_;
8348
8349     my $res = mon_cmd($vmid, 'query-iothreads');
8350
8351     my $iothreads = {};
8352     foreach my $iothread (@$res) {
8353         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8354     }
8355
8356     return $iothreads;
8357 }
8358
8359 sub scsihw_infos {
8360     my ($conf, $drive) = @_;
8361
8362     my $maxdev = 0;
8363
8364     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8365         $maxdev = 7;
8366     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8367         $maxdev = 1;
8368     } else {
8369         $maxdev = 256;
8370     }
8371
8372     my $controller = int($drive->{index} / $maxdev);
8373     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8374         ? "virtioscsi"
8375         : "scsihw";
8376
8377     return ($maxdev, $controller, $controller_prefix);
8378 }
8379
8380 sub resolve_dst_disk_format {
8381         my ($storecfg, $storeid, $src_volname, $format) = @_;
8382         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8383
8384         if (!$format) {
8385             # if no target format is specified, use the source disk format as hint
8386             if ($src_volname) {
8387                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8388                 $format = qemu_img_format($scfg, $src_volname);
8389             } else {
8390                 return $defFormat;
8391             }
8392         }
8393
8394         # test if requested format is supported - else use default
8395         my $supported = grep { $_ eq $format } @$validFormats;
8396         $format = $defFormat if !$supported;
8397         return $format;
8398 }
8399
8400 # NOTE: if this logic changes, please update docs & possibly gui logic
8401 sub find_vmstate_storage {
8402     my ($conf, $storecfg) = @_;
8403
8404     # first, return storage from conf if set
8405     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8406
8407     my ($target, $shared, $local);
8408
8409     foreach_storage_used_by_vm($conf, sub {
8410         my ($sid) = @_;
8411         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8412         my $dst = $scfg->{shared} ? \$shared : \$local;
8413         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8414     });
8415
8416     # second, use shared storage where VM has at least one disk
8417     # third, use local storage where VM has at least one disk
8418     # fall back to local storage
8419     $target = $shared // $local // 'local';
8420
8421     return $target;
8422 }
8423
8424 sub generate_uuid {
8425     my ($uuid, $uuid_str);
8426     UUID::generate($uuid);
8427     UUID::unparse($uuid, $uuid_str);
8428     return $uuid_str;
8429 }
8430
8431 sub generate_smbios1_uuid {
8432     return "uuid=".generate_uuid();
8433 }
8434
8435 sub nbd_stop {
8436     my ($vmid) = @_;
8437
8438     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8439 }
8440
8441 sub create_reboot_request {
8442     my ($vmid) = @_;
8443     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8444         or die "failed to create reboot trigger file: $!\n";
8445     close($fh);
8446 }
8447
8448 sub clear_reboot_request {
8449     my ($vmid) = @_;
8450     my $path = "/run/qemu-server/$vmid.reboot";
8451     my $res = 0;
8452
8453     $res = unlink($path);
8454     die "could not remove reboot request for $vmid: $!"
8455         if !$res && $! != POSIX::ENOENT;
8456
8457     return $res;
8458 }
8459
8460 sub bootorder_from_legacy {
8461     my ($conf, $bootcfg) = @_;
8462
8463     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8464     my $bootindex_hash = {};
8465     my $i = 1;
8466     foreach my $o (split(//, $boot)) {
8467         $bootindex_hash->{$o} = $i*100;
8468         $i++;
8469     }
8470
8471     my $bootorder = {};
8472
8473     PVE::QemuConfig->foreach_volume($conf, sub {
8474         my ($ds, $drive) = @_;
8475
8476         if (drive_is_cdrom ($drive, 1)) {
8477             if ($bootindex_hash->{d}) {
8478                 $bootorder->{$ds} = $bootindex_hash->{d};
8479                 $bootindex_hash->{d} += 1;
8480             }
8481         } elsif ($bootindex_hash->{c}) {
8482             $bootorder->{$ds} = $bootindex_hash->{c}
8483                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8484             $bootindex_hash->{c} += 1;
8485         }
8486     });
8487
8488     if ($bootindex_hash->{n}) {
8489         for (my $i = 0; $i < $MAX_NETS; $i++) {
8490             my $netname = "net$i";
8491             next if !$conf->{$netname};
8492             $bootorder->{$netname} = $bootindex_hash->{n};
8493             $bootindex_hash->{n} += 1;
8494         }
8495     }
8496
8497     return $bootorder;
8498 }
8499
8500 # Generate default device list for 'boot: order=' property. Matches legacy
8501 # default boot order, but with explicit device names. This is important, since
8502 # the fallback for when neither 'order' nor the old format is specified relies
8503 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8504 sub get_default_bootdevices {
8505     my ($conf) = @_;
8506
8507     my @ret = ();
8508
8509     # harddisk
8510     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8511     push @ret, $first if $first;
8512
8513     # cdrom
8514     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8515     push @ret, $first if $first;
8516
8517     # network
8518     for (my $i = 0; $i < $MAX_NETS; $i++) {
8519         my $netname = "net$i";
8520         next if !$conf->{$netname};
8521         push @ret, $netname;
8522         last;
8523     }
8524
8525     return \@ret;
8526 }
8527
8528 sub device_bootorder {
8529     my ($conf) = @_;
8530
8531     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8532
8533     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8534
8535     my $bootorder = {};
8536     if (!defined($boot) || $boot->{legacy}) {
8537         $bootorder = bootorder_from_legacy($conf, $boot);
8538     } elsif ($boot->{order}) {
8539         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8540         for my $dev (PVE::Tools::split_list($boot->{order})) {
8541             $bootorder->{$dev} = $i++;
8542         }
8543     }
8544
8545     return $bootorder;
8546 }
8547
8548 sub register_qmeventd_handle {
8549     my ($vmid) = @_;
8550
8551     my $fh;
8552     my $peer = "/var/run/qmeventd.sock";
8553     my $count = 0;
8554
8555     for (;;) {
8556         $count++;
8557         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8558         last if $fh;
8559         if ($! != EINTR && $! != EAGAIN) {
8560             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8561         }
8562         if ($count > 4) {
8563             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8564               . "after $count retries\n";
8565         }
8566         usleep(25000);
8567     }
8568
8569     # send handshake to mark VM as backing up
8570     print $fh to_json({vzdump => {vmid => "$vmid"}});
8571
8572     # return handle to be closed later when inhibit is no longer required
8573     return $fh;
8574 }
8575
8576 # bash completion helper
8577
8578 sub complete_backup_archives {
8579     my ($cmdname, $pname, $cvalue) = @_;
8580
8581     my $cfg = PVE::Storage::config();
8582
8583     my $storeid;
8584
8585     if ($cvalue =~ m/^([^:]+):/) {
8586         $storeid = $1;
8587     }
8588
8589     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8590
8591     my $res = [];
8592     foreach my $id (keys %$data) {
8593         foreach my $item (@{$data->{$id}}) {
8594             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8595             push @$res, $item->{volid} if defined($item->{volid});
8596         }
8597     }
8598
8599     return $res;
8600 }
8601
8602 my $complete_vmid_full = sub {
8603     my ($running) = @_;
8604
8605     my $idlist = vmstatus();
8606
8607     my $res = [];
8608
8609     foreach my $id (keys %$idlist) {
8610         my $d = $idlist->{$id};
8611         if (defined($running)) {
8612             next if $d->{template};
8613             next if $running && $d->{status} ne 'running';
8614             next if !$running && $d->{status} eq 'running';
8615         }
8616         push @$res, $id;
8617
8618     }
8619     return $res;
8620 };
8621
8622 sub complete_vmid {
8623     return &$complete_vmid_full();
8624 }
8625
8626 sub complete_vmid_stopped {
8627     return &$complete_vmid_full(0);
8628 }
8629
8630 sub complete_vmid_running {
8631     return &$complete_vmid_full(1);
8632 }
8633
8634 sub complete_storage {
8635
8636     my $cfg = PVE::Storage::config();
8637     my $ids = $cfg->{ids};
8638
8639     my $res = [];
8640     foreach my $sid (keys %$ids) {
8641         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8642         next if !$ids->{$sid}->{content}->{images};
8643         push @$res, $sid;
8644     }
8645
8646     return $res;
8647 }
8648
8649 sub complete_migration_storage {
8650     my ($cmd, $param, $current_value, $all_args) = @_;
8651
8652     my $targetnode = @$all_args[1];
8653
8654     my $cfg = PVE::Storage::config();
8655     my $ids = $cfg->{ids};
8656
8657     my $res = [];
8658     foreach my $sid (keys %$ids) {
8659         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8660         next if !$ids->{$sid}->{content}->{images};
8661         push @$res, $sid;
8662     }
8663
8664     return $res;
8665 }
8666
8667 sub vm_is_paused {
8668     my ($vmid, $include_suspended) = @_;
8669     my $qmpstatus = eval {
8670         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8671         mon_cmd($vmid, "query-status");
8672     };
8673     warn "$@\n" if $@;
8674     return $qmpstatus && (
8675         $qmpstatus->{status} eq "paused" ||
8676         $qmpstatus->{status} eq "prelaunch" ||
8677         ($include_suspended && $qmpstatus->{status} eq "suspended")
8678     );
8679 }
8680
8681 sub check_volume_storage_type {
8682     my ($storecfg, $vol) = @_;
8683
8684     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8685     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8686     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8687
8688     die "storage '$storeid' does not support content-type '$vtype'\n"
8689         if !$scfg->{content}->{$vtype};
8690
8691     return 1;
8692 }
8693
8694 sub add_nets_bridge_fdb {
8695     my ($conf, $vmid) = @_;
8696
8697     for my $opt (keys %$conf) {
8698         next if $opt !~ m/^net(\d+)$/;
8699         my $iface = "tap${vmid}i$1";
8700         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8701         my $net = parse_net($conf->{$opt}, 1) or next;
8702
8703         my $mac = $net->{macaddr};
8704         if (!$mac) {
8705             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8706                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8707             next;
8708         }
8709
8710         my $bridge = $net->{bridge};
8711         if (!$bridge) {
8712             log_warn("Interface '$iface' not attached to any bridge.");
8713             next;
8714         }
8715         if ($have_sdn) {
8716             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8717         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8718             PVE::Network::add_bridge_fdb($iface, $mac);
8719         }
8720     }
8721 }
8722
8723 sub del_nets_bridge_fdb {
8724     my ($conf, $vmid) = @_;
8725
8726     for my $opt (keys %$conf) {
8727         next if $opt !~ m/^net(\d+)$/;
8728         my $iface = "tap${vmid}i$1";
8729
8730         my $net = parse_net($conf->{$opt}) or next;
8731         my $mac = $net->{macaddr} or next;
8732
8733         my $bridge = $net->{bridge};
8734         if ($have_sdn) {
8735             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8736         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8737             PVE::Network::del_bridge_fdb($iface, $mac);
8738         }
8739     }
8740 }
8741
8742 sub create_ifaces_ipams_ips {
8743     my ($conf, $vmid) = @_;
8744
8745     return if !$have_sdn;
8746
8747     foreach my $opt (keys %$conf) {
8748         if ($opt =~ m/^net(\d+)$/) {
8749             my $value = $conf->{$opt};
8750             my $net = PVE::QemuServer::parse_net($value);
8751             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8752             warn $@ if $@;
8753         }
8754     }
8755 }
8756
8757 sub delete_ifaces_ipams_ips {
8758     my ($conf, $vmid) = @_;
8759
8760     return if !$have_sdn;
8761
8762     foreach my $opt (keys %$conf) {
8763         if ($opt =~ m/^net(\d+)$/) {
8764             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8765             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8766             warn $@ if $@;
8767         }
8768     }
8769 }
8770
8771 1;