PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     $have_sdn = 1;
  68 };
  69
  70 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  71 my $OVMF = {
  72     x86_64 => {
  73         '4m-no-smm' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  76         ],
  77         '4m-no-smm-ms' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  80         ],
  81         '4m' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  84         ],
  85         '4m-ms' => [
  86             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  88         ],
  89         default => [
  90             "$EDK2_FW_BASE/OVMF_CODE.fd",
  91             "$EDK2_FW_BASE/OVMF_VARS.fd",
  92         ],
  93     },
  94     aarch64 => {
  95         default => [
  96             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  97             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  98         ],
  99     },
 100 };
 101
 102 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 103
 104 # Note about locking: we use flock on the config file protect against concurent actions.
 105 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 106 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 107 # But you can ignore this kind of lock with the --skiplock flag.
 108
 109 cfs_register_file(
 110     '/qemu-server/',
 111     \&parse_vm_config,
 112     \&write_vm_config
 113 );
 114
 115 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 116     description => "Some command save/restore state from this location.",
 117     type => 'string',
 118     maxLength => 128,
 119     optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 123         description => "Specifies the QEMU machine type.",
 124         type => 'string',
 125         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 126         maxLength => 40,
 127         optional => 1,
 128 });
 129
 130 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0,
 167     },
 168     'freeze-fs-on-backup' => {
 169         description => "Freeze/thaw guest filesystems on backup for consistency.",
 170         type => 'boolean',
 171         optional => 1,
 172         default => 1,
 173     },
 174     type => {
 175         description => "Select the agent type",
 176         type => 'string',
 177         default => 'virtio',
 178         optional => 1,
 179         enum => [qw(virtio isa)],
 180     },
 181 };
 182
 183 my $vga_fmt = {
 184     type => {
 185         description => "Select the VGA type.",
 186         type => 'string',
 187         default => 'std',
 188         optional => 1,
 189         default_key => 1,
 190         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 191     },
 192     memory => {
 193         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 194         type => 'integer',
 195         optional => 1,
 196         minimum => 4,
 197         maximum => 512,
 198     },
 199 };
 200
 201 my $ivshmem_fmt = {
 202     size => {
 203         type => 'integer',
 204         minimum => 1,
 205         description => "The size of the file in MB.",
 206     },
 207     name => {
 208         type => 'string',
 209         pattern => '[a-zA-Z0-9\-]+',
 210         optional => 1,
 211         format_description => 'string',
 212         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 213     },
 214 };
 215
 216 my $audio_fmt = {
 217     device => {
 218         type => 'string',
 219         enum => [qw(ich9-intel-hda intel-hda AC97)],
 220         description =>  "Configure an audio device."
 221     },
 222     driver =>  {
 223         type => 'string',
 224         enum => ['spice', 'none'],
 225         default => 'spice',
 226         optional => 1,
 227         description => "Driver backend for the audio device."
 228     },
 229 };
 230
 231 my $spice_enhancements_fmt = {
 232     foldersharing => {
 233         type => 'boolean',
 234         optional => 1,
 235         default => '0',
 236         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 237     },
 238     videostreaming =>  {
 239         type => 'string',
 240         enum => ['off', 'all', 'filter'],
 241         default => 'off',
 242         optional => 1,
 243         description => "Enable video streaming. Uses compression for detected video streams."
 244     },
 245 };
 246
 247 my $rng_fmt = {
 248     source => {
 249         type => 'string',
 250         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 251         default_key => 1,
 252         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 253             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 254             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 255             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 256             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 257             ." a hardware RNG from the host.",
 258     },
 259     max_bytes => {
 260         type => 'integer',
 261         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 262             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 263             ." `0` to disable limiting (potentially dangerous!).",
 264         optional => 1,
 265
 266         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 267         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 268         # reading from /dev/urandom
 269         default => 1024,
 270     },
 271     period => {
 272         type => 'integer',
 273         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 274             ." the guest to retrieve another 'max_bytes' of entropy.",
 275         optional => 1,
 276         default => 1000,
 277     },
 278 };
 279
 280 my $meta_info_fmt = {
 281     'ctime' => {
 282         type => 'integer',
 283         description => "The guest creation timestamp as UNIX epoch time",
 284         minimum => 0,
 285         optional => 1,
 286     },
 287     'creation-qemu' => {
 288         type => 'string',
 289         description => "The QEMU (machine) version from the time this VM was created.",
 290         pattern => '\d+(\.\d+)+',
 291         optional => 1,
 292     },
 293 };
 294
 295 my $confdesc = {
 296     onboot => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Specifies whether a VM will be started during system bootup.",
 300         default => 0,
 301     },
 302     autostart => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Automatic restart after crash (currently ignored).",
 306         default => 0,
 307     },
 308     hotplug => {
 309         optional => 1,
 310         type => 'string', format => 'pve-hotplug-features',
 311         description => "Selectively enable hotplug features. This is a comma separated list of"
 312             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 313             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 314             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 315             ." windows > 7.",
 316         default => 'network,disk,usb',
 317     },
 318     reboot => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 322         default => 1,
 323     },
 324     lock => {
 325         optional => 1,
 326         type => 'string',
 327         description => "Lock/unlock the VM.",
 328         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 329     },
 330     cpulimit => {
 331         optional => 1,
 332         type => 'number',
 333         description => "Limit of CPU usage.",
 334         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 335             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 336         minimum => 0,
 337         maximum => 128,
 338         default => 0,
 339     },
 340     cpuunits => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 344         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 345             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 346             ." weights of all the other running VMs.",
 347         minimum => 1,
 348         maximum => 262144,
 349         default => 'cgroup v1: 1024, cgroup v2: 100',
 350     },
 351     memory => {
 352         optional => 1,
 353         type => 'string',
 354         description => "Memory properties.",
 355         format => $PVE::QemuServer::Memory::memory_fmt
 356     },
 357     balloon => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 361         minimum => 0,
 362     },
 363     shares => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 367             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 368             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 369         minimum => 0,
 370         maximum => 50000,
 371         default => 1000,
 372     },
 373     keyboard => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Keyboard layout for VNC server. This option is generally not required and"
 377         ." is often better handled from within the guest OS.",
 378         enum => PVE::Tools::kvmkeymaplist(),
 379         default => undef,
 380     },
 381     name => {
 382         optional => 1,
 383         type => 'string', format => 'dns-name',
 384         description => "Set a name for the VM. Only used on the configuration web interface.",
 385     },
 386     scsihw => {
 387         optional => 1,
 388         type => 'string',
 389         description => "SCSI controller model",
 390         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 391         default => 'lsi',
 392     },
 393     description => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Description for the VM. Shown in the web-interface VM's summary."
 397             ." This is saved as comment inside the configuration file.",
 398         maxLength => 1024 * 8,
 399     },
 400     ostype => {
 401         optional => 1,
 402         type => 'string',
 403         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 404         description => "Specify guest operating system.",
 405         verbose_description => <<EODESC,
 406 Specify guest operating system. This is used to enable special
 407 optimization/features for specific operating systems:
 408
 409 [horizontal]
 410 other;; unspecified OS
 411 wxp;; Microsoft Windows XP
 412 w2k;; Microsoft Windows 2000
 413 w2k3;; Microsoft Windows 2003
 414 w2k8;; Microsoft Windows 2008
 415 wvista;; Microsoft Windows Vista
 416 win7;; Microsoft Windows 7
 417 win8;; Microsoft Windows 8/2012/2012r2
 418 win10;; Microsoft Windows 10/2016/2019
 419 win11;; Microsoft Windows 11/2022
 420 l24;; Linux 2.4 Kernel
 421 l26;; Linux 2.6 - 6.X Kernel
 422 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 423 EODESC
 424     },
 425     boot => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-boot',
 428         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 429             ." key or 'legacy=' is deprecated.",
 430     },
 431     bootdisk => {
 432         optional => 1,
 433         type => 'string', format => 'pve-qm-bootdisk',
 434         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 435         pattern => '(ide|sata|scsi|virtio)\d+',
 436     },
 437     smp => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPUs. Please use option -sockets instead.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     sockets => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of CPU sockets.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     cores => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of cores per socket.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     numa => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable NUMA.",
 462         default => 0,
 463     },
 464     hugepages => {
 465         optional => 1,
 466         type => 'string',
 467         description => "Enable/disable hugepages memory.",
 468         enum => [qw(any 2 1024)],
 469     },
 470     keephugepages => {
 471         optional => 1,
 472         type => 'boolean',
 473         default => 0,
 474         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 475             ." after VM shutdown and can be used for subsequent starts.",
 476     },
 477     vcpus => {
 478         optional => 1,
 479         type => 'integer',
 480         description => "Number of hotplugged vcpus.",
 481         minimum => 1,
 482         default => 0,
 483     },
 484     acpi => {
 485         optional => 1,
 486         type => 'boolean',
 487         description => "Enable/disable ACPI.",
 488         default => 1,
 489     },
 490     agent => {
 491         optional => 1,
 492         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 493         type => 'string',
 494         format => $agent_fmt,
 495     },
 496     kvm => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable KVM hardware virtualization.",
 500         default => 1,
 501     },
 502     tdf => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable time drift fix.",
 506         default => 0,
 507     },
 508     localtime => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 512             ." the `ostype` indicates a Microsoft Windows OS.",
 513     },
 514     freeze => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 518     },
 519     vga => {
 520         optional => 1,
 521         type => 'string', format => $vga_fmt,
 522         description => "Configure the VGA hardware.",
 523         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 524             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 525             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 526             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 527             ." display server. For win* OS you can select how many independent displays you want,"
 528             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 529             ." using a serial device as terminal.",
 530     },
 531     watchdog => {
 532         optional => 1,
 533         type => 'string', format => 'pve-qm-watchdog',
 534         description => "Create a virtual hardware watchdog device.",
 535         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 536             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 537             ." else the watchdog will reset the guest (or execute the respective action specified)",
 538     },
 539     startdate => {
 540         optional => 1,
 541         type => 'string',
 542         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 543         description => "Set the initial date of the real time clock. Valid format for date are:"
 544             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 545         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 546         default => 'now',
 547     },
 548     startup =>  get_standard_option('pve-startup-order'),
 549     template => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Enable/disable Template.",
 553         default => 0,
 554     },
 555     args => {
 556         optional => 1,
 557         type => 'string',
 558         description => "Arbitrary arguments passed to kvm.",
 559         verbose_description => <<EODESCR,
 560 Arbitrary arguments passed to kvm, for example:
 561
 562 args: -no-reboot -smbios 'type=0,vendor=FOO'
 563
 564 NOTE: this option is for experts only.
 565 EODESCR
 566     },
 567     tablet => {
 568         optional => 1,
 569         type => 'boolean',
 570         default => 1,
 571         description => "Enable/disable the USB tablet device.",
 572         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 573             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 574             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 575             ." may consider disabling this to save some context switches. This is turned off by"
 576             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 577     },
 578     migrate_speed => {
 579         optional => 1,
 580         type => 'integer',
 581         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 582         minimum => 0,
 583         default => 0,
 584     },
 585     migrate_downtime => {
 586         optional => 1,
 587         type => 'number',
 588         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 589         minimum => 0,
 590         default => 0.1,
 591     },
 592     cdrom => {
 593         optional => 1,
 594         type => 'string', format => 'pve-qm-ide',
 595         typetext => '<volume>',
 596         description => "This is an alias for option -ide2",
 597     },
 598     cpu => {
 599         optional => 1,
 600         description => "Emulated CPU type.",
 601         type => 'string',
 602         format => 'pve-vm-cpu-conf',
 603     },
 604     parent => get_standard_option('pve-snapshot-name', {
 605         optional => 1,
 606         description => "Parent snapshot name. This is used internally, and should not be modified.",
 607     }),
 608     snaptime => {
 609         optional => 1,
 610         description => "Timestamp for snapshots.",
 611         type => 'integer',
 612         minimum => 0,
 613     },
 614     vmstate => {
 615         optional => 1,
 616         type => 'string', format => 'pve-volume-id',
 617         description => "Reference to a volume which stores the VM state. This is used internally"
 618             ." for snapshots.",
 619     },
 620     vmstatestorage => get_standard_option('pve-storage-id', {
 621         description => "Default storage for VM state volumes/files.",
 622         optional => 1,
 623     }),
 624     runningmachine => get_standard_option('pve-qemu-machine', {
 625         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 626             ." for snapshots.",
 627     }),
 628     runningcpu => {
 629         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 630             ." internally for snapshots.",
 631         optional => 1,
 632         type => 'string',
 633         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 634         format_description => 'QEMU -cpu parameter'
 635     },
 636     machine => get_standard_option('pve-qemu-machine'),
 637     arch => {
 638         description => "Virtual processor architecture. Defaults to the host.",
 639         optional => 1,
 640         type => 'string',
 641         enum => [qw(x86_64 aarch64)],
 642     },
 643     smbios1 => {
 644         description => "Specify SMBIOS type 1 fields.",
 645         type => 'string', format => 'pve-qm-smbios1',
 646         maxLength => 512,
 647         optional => 1,
 648     },
 649     protection => {
 650         optional => 1,
 651         type => 'boolean',
 652         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 653             ." remove disk operations.",
 654         default => 0,
 655     },
 656     bios => {
 657         optional => 1,
 658         type => 'string',
 659         enum => [ qw(seabios ovmf) ],
 660         description => "Select BIOS implementation.",
 661         default => 'seabios',
 662     },
 663     vmgenid => {
 664         type => 'string',
 665         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 666         format_description => 'UUID',
 667         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 668             ." to disable explicitly.",
 669         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 670             ." value identifier to the guest OS. This allows to notify the guest operating system"
 671             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 672             ." execution or creation from a template). The guest operating system notices the"
 673             ." change, and is then able to react as appropriate by marking its copies of"
 674             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 675             ."Note that auto-creation only works when done through API/CLI create or update methods"
 676             .", but not when manually editing the config file.",
 677         default => "1 (autogenerated)",
 678         optional => 1,
 679     },
 680     hookscript => {
 681         type => 'string',
 682         format => 'pve-volume-id',
 683         optional => 1,
 684         description => "Script that will be executed during various steps in the vms lifetime.",
 685     },
 686     ivshmem => {
 687         type => 'string',
 688         format => $ivshmem_fmt,
 689         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 690             ." the host.",
 691         optional => 1,
 692     },
 693     audio0 => {
 694         type => 'string',
 695         format => $audio_fmt,
 696         description => "Configure a audio device, useful in combination with QXL/Spice.",
 697         optional => 1
 698     },
 699     spice_enhancements => {
 700         type => 'string',
 701         format => $spice_enhancements_fmt,
 702         description => "Configure additional enhancements for SPICE.",
 703         optional => 1
 704     },
 705     tags => {
 706         type => 'string', format => 'pve-tag-list',
 707         description => 'Tags of the VM. This is only meta information.',
 708         optional => 1,
 709     },
 710     rng0 => {
 711         type => 'string',
 712         format => $rng_fmt,
 713         description => "Configure a VirtIO-based Random Number Generator.",
 714         optional => 1,
 715     },
 716     meta => {
 717         type => 'string',
 718         format => $meta_info_fmt,
 719         description => "Some (read-only) meta-information about this guest.",
 720         optional => 1,
 721     },
 722     affinity => {
 723         type => 'string', format => 'pve-cpuset',
 724         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 725         optional => 1,
 726     },
 727 };
 728
 729 my $cicustom_fmt = {
 730     meta => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all meta data passed to the VM via"
 734             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     network => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 742         format => 'pve-volume-id',
 743         format_description => 'volume',
 744     },
 745     user => {
 746         type => 'string',
 747         optional => 1,
 748         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 749         format => 'pve-volume-id',
 750         format_description => 'volume',
 751     },
 752     vendor => {
 753         type => 'string',
 754         optional => 1,
 755         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 756         format => 'pve-volume-id',
 757         format_description => 'volume',
 758     },
 759 };
 760 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 761
 762 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 763 my $confdesc_cloudinit = {
 764     citype => {
 765         optional => 1,
 766         type => 'string',
 767         description => 'Specifies the cloud-init configuration format. The default depends on the'
 768             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 769             .' and `configdrive2` for windows.',
 770         enum => ['configdrive2', 'nocloud', 'opennebula'],
 771     },
 772     ciuser => {
 773         optional => 1,
 774         type => 'string',
 775         description => "cloud-init: User name to change ssh keys and password for instead of the"
 776             ." image's configured default user.",
 777     },
 778     cipassword => {
 779         optional => 1,
 780         type => 'string',
 781         description => 'cloud-init: Password to assign the user. Using this is generally not'
 782             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 783             .' support hashed passwords.',
 784     },
 785     ciupgrade => {
 786         optional => 1,
 787         type => 'boolean',
 788         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 789         default => 1,
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.',
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.',
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_NETS = 32;
 842 my $MAX_SERIAL_PORTS = 4;
 843 my $MAX_PARALLEL_PORTS = 3;
 844
 845 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 846     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 847 }
 848
 849 my $nic_model_list = [
 850     'e1000',
 851     'e1000-82540em',
 852     'e1000-82544gc',
 853     'e1000-82545em',
 854     'e1000e',
 855     'i82551',
 856     'i82557b',
 857     'i82559er',
 858     'ne2k_isa',
 859     'ne2k_pci',
 860     'pcnet',
 861     'rtl8139',
 862     'virtio',
 863     'vmxnet3',
 864 ];
 865 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 866
 867 my $net_fmt_bridge_descr = <<__EOD__;
 868 Bridge to attach the network device to. The Proxmox VE standard bridge
 869 is called 'vmbr0'.
 870
 871 If you do not specify a bridge, we create a kvm user (NATed) network
 872 device, which provides DHCP and DNS services. The following addresses
 873 are used:
 874
 875  10.0.2.2   Gateway
 876  10.0.2.3   DNS Server
 877  10.0.2.4   SMB Server
 878
 879 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 880 __EOD__
 881
 882 my $net_fmt = {
 883     macaddr  => get_standard_option('mac-addr', {
 884         description => "MAC address. That address must be unique withing your network. This is"
 885             ." automatically generated if not specified.",
 886     }),
 887     model => {
 888         type => 'string',
 889         description => "Network Card Model. The 'virtio' model provides the best performance with"
 890             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 891             ." best to use 'e1000'.",
 892         enum => $nic_model_list,
 893         default_key => 1,
 894     },
 895     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 896     bridge => get_standard_option('pve-bridge-id', {
 897         description => $net_fmt_bridge_descr,
 898         optional => 1,
 899     }),
 900     queues => {
 901         type => 'integer',
 902         minimum => 0, maximum => 64,
 903         description => 'Number of packet queues to be used on the device.',
 904         optional => 1,
 905     },
 906     rate => {
 907         type => 'number',
 908         minimum => 0,
 909         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 910         optional => 1,
 911     },
 912     tag => {
 913         type => 'integer',
 914         minimum => 1, maximum => 4094,
 915         description => 'VLAN tag to apply to packets on this interface.',
 916         optional => 1,
 917     },
 918     trunks => {
 919         type => 'string',
 920         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 921         description => 'VLAN trunks to pass through this interface.',
 922         format_description => 'vlanid[;vlanid...]',
 923         optional => 1,
 924     },
 925     firewall => {
 926         type => 'boolean',
 927         description => 'Whether this interface should be protected by the firewall.',
 928         optional => 1,
 929     },
 930     link_down => {
 931         type => 'boolean',
 932         description => 'Whether this interface should be disconnected (like pulling the plug).',
 933         optional => 1,
 934     },
 935     mtu => {
 936         type => 'integer',
 937         minimum => 1, maximum => 65520,
 938         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 939         optional => 1,
 940     },
 941 };
 942
 943 my $netdesc = {
 944     optional => 1,
 945     type => 'string', format => $net_fmt,
 946     description => "Specify network devices.",
 947 };
 948
 949 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 950
 951 my $ipconfig_fmt = {
 952     ip => {
 953         type => 'string',
 954         format => 'pve-ipv4-config',
 955         format_description => 'IPv4Format/CIDR',
 956         description => 'IPv4 address in CIDR format.',
 957         optional => 1,
 958         default => 'dhcp',
 959     },
 960     gw => {
 961         type => 'string',
 962         format => 'ipv4',
 963         format_description => 'GatewayIPv4',
 964         description => 'Default gateway for IPv4 traffic.',
 965         optional => 1,
 966         requires => 'ip',
 967     },
 968     ip6 => {
 969         type => 'string',
 970         format => 'pve-ipv6-config',
 971         format_description => 'IPv6Format/CIDR',
 972         description => 'IPv6 address in CIDR format.',
 973         optional => 1,
 974         default => 'dhcp',
 975     },
 976     gw6 => {
 977         type => 'string',
 978         format => 'ipv6',
 979         format_description => 'GatewayIPv6',
 980         description => 'Default gateway for IPv6 traffic.',
 981         optional => 1,
 982         requires => 'ip6',
 983     },
 984 };
 985 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 986 my $ipconfigdesc = {
 987     optional => 1,
 988     type => 'string', format => 'pve-qm-ipconfig',
 989     description => <<'EODESCR',
 990 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 991
 992 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 993
 994 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 995 gateway should be provided.
 996 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 997 cloud-init 19.4 or newer.
 998
 999 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1000 dhcp on IPv4.
1001 EODESCR
1002 };
1003 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1004
1005 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1006     $confdesc->{"net$i"} = $netdesc;
1007     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1008 }
1009
1010 foreach my $key (keys %$confdesc_cloudinit) {
1011     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1012 }
1013
1014 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1015 sub pve_verify_cpuset {
1016     my ($set_text, $noerr) = @_;
1017
1018     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1019
1020     if ($@) {
1021         return if $noerr;
1022         die "unable to parse cpuset option\n";
1023     }
1024
1025     return PVE::CpuSet->new($members)->short_string();
1026 }
1027
1028 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1029 sub verify_volume_id_or_qm_path {
1030     my ($volid, $noerr) = @_;
1031
1032     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1033
1034     return verify_volume_id_or_absolute_path($volid, $noerr);
1035 }
1036
1037 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1038 sub verify_volume_id_or_absolute_path {
1039     my ($volid, $noerr) = @_;
1040
1041     return $volid if $volid =~ m|^/|;
1042
1043     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1044     if ($@) {
1045         return if $noerr;
1046         die $@;
1047     }
1048     return $volid;
1049 }
1050
1051 my $serialdesc = {
1052         optional => 1,
1053         type => 'string',
1054         pattern => '(/dev/.+|socket)',
1055         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1056         verbose_description =>  <<EODESCR,
1057 Create a serial device inside the VM (n is 0 to 3), and pass through a
1058 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1059 host side (use 'qm terminal' to open a terminal connection).
1060
1061 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1062 use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 my $paralleldesc= {
1069         optional => 1,
1070         type => 'string',
1071         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1072         description =>  "Map host parallel devices (n is 0 to 2).",
1073         verbose_description =>  <<EODESCR,
1074 Map host parallel devices (n is 0 to 2).
1075
1076 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1077 machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082
1083 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1084     $confdesc->{"parallel$i"} = $paralleldesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1088     $confdesc->{"serial$i"} = $serialdesc;
1089 }
1090
1091 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1092     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1093 }
1094
1095 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1096     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1097 }
1098
1099 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1100     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1101 }
1102
1103 my $boot_fmt = {
1104     legacy => {
1105         optional => 1,
1106         default_key => 1,
1107         type => 'string',
1108         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1109                      . " Deprecated, use 'order=' instead.",
1110         pattern => '[acdn]{1,4}',
1111         format_description => "[acdn]{1,4}",
1112
1113         # note: this is also the fallback if boot: is not given at all
1114         default => 'cdn',
1115     },
1116     order => {
1117         optional => 1,
1118         type => 'string',
1119         format => 'pve-qm-bootdev-list',
1120         format_description => "device[;device...]",
1121         description => <<EODESC,
1122 The guest will attempt to boot from devices in the order they appear here.
1123
1124 Disks, optical drives and passed-through storage USB devices will be directly
1125 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1126 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1127
1128 Note that only devices in this list will be marked as bootable and thus loaded
1129 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1130 (e.g. software-raid), you need to specify all of them here.
1131
1132 Overrides the deprecated 'legacy=[acdn]*' value when given.
1133 EODESC
1134     },
1135 };
1136 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1137
1138 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1139 sub verify_bootdev {
1140     my ($dev, $noerr) = @_;
1141
1142     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1143     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1144
1145     my $check = sub {
1146         my ($base) = @_;
1147         return 0 if $dev !~ m/^$base\d+$/;
1148         return 0 if !$confdesc->{$dev};
1149         return 1;
1150     };
1151
1152     return $dev if $check->("net");
1153     return $dev if $check->("usb");
1154     return $dev if $check->("hostpci");
1155
1156     return if $noerr;
1157     die "invalid boot device '$dev'\n";
1158 }
1159
1160 sub print_bootorder {
1161     my ($devs) = @_;
1162     return "" if !@$devs;
1163     my $data = { order => join(';', @$devs) };
1164     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1165 }
1166
1167 my $kvm_api_version = 0;
1168
1169 sub kvm_version {
1170     return $kvm_api_version if $kvm_api_version;
1171
1172     open my $fh, '<', '/dev/kvm' or return;
1173
1174     # 0xae00 => KVM_GET_API_VERSION
1175     $kvm_api_version = ioctl($fh, 0xae00, 0);
1176     close($fh);
1177
1178     return $kvm_api_version;
1179 }
1180
1181 my $kvm_user_version = {};
1182 my $kvm_mtime = {};
1183
1184 sub kvm_user_version {
1185     my ($binary) = @_;
1186
1187     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1188     my $st = stat($binary);
1189
1190     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1191     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1192         $cachedmtime == $st->mtime;
1193
1194     $kvm_user_version->{$binary} = 'unknown';
1195     $kvm_mtime->{$binary} = $st->mtime;
1196
1197     my $code = sub {
1198         my $line = shift;
1199         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1200             $kvm_user_version->{$binary} = $2;
1201         }
1202     };
1203
1204     eval { run_command([$binary, '--version'], outfunc => $code); };
1205     warn $@ if $@;
1206
1207     return $kvm_user_version->{$binary};
1208
1209 }
1210 my sub extract_version {
1211     my ($machine_type, $version) = @_;
1212     $version = kvm_user_version() if !defined($version);
1213     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1214 }
1215
1216 sub kernel_has_vhost_net {
1217     return -c '/dev/vhost-net';
1218 }
1219
1220 sub option_exists {
1221     my $key = shift;
1222     return defined($confdesc->{$key});
1223 }
1224
1225 my $cdrom_path;
1226 sub get_cdrom_path {
1227
1228     return $cdrom_path if defined($cdrom_path);
1229
1230     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1231
1232     if (!defined($cdrom_path)) {
1233         log_warn("no physical CD-ROM available, ignoring");
1234         $cdrom_path = '';
1235     }
1236
1237     return $cdrom_path;
1238 }
1239
1240 sub get_iso_path {
1241     my ($storecfg, $vmid, $cdrom) = @_;
1242
1243     if ($cdrom eq 'cdrom') {
1244         return get_cdrom_path();
1245     } elsif ($cdrom eq 'none') {
1246         return '';
1247     } elsif ($cdrom =~ m|^/|) {
1248         return $cdrom;
1249     } else {
1250         return PVE::Storage::path($storecfg, $cdrom);
1251     }
1252 }
1253
1254 # try to convert old style file names to volume IDs
1255 sub filename_to_volume_id {
1256     my ($vmid, $file, $media) = @_;
1257
1258      if (!($file eq 'none' || $file eq 'cdrom' ||
1259           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1260
1261         return if $file =~ m|/|;
1262
1263         if ($media && $media eq 'cdrom') {
1264             $file = "local:iso/$file";
1265         } else {
1266             $file = "local:$vmid/$file";
1267         }
1268     }
1269
1270     return $file;
1271 }
1272
1273 sub verify_media_type {
1274     my ($opt, $vtype, $media) = @_;
1275
1276     return if !$media;
1277
1278     my $etype;
1279     if ($media eq 'disk') {
1280         $etype = 'images';
1281     } elsif ($media eq 'cdrom') {
1282         $etype = 'iso';
1283     } else {
1284         die "internal error";
1285     }
1286
1287     return if ($vtype eq $etype);
1288
1289     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1290 }
1291
1292 sub cleanup_drive_path {
1293     my ($opt, $storecfg, $drive) = @_;
1294
1295     # try to convert filesystem paths to volume IDs
1296
1297     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1298         ($drive->{file} !~ m|^/dev/.+|) &&
1299         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1300         ($drive->{file} !~ m/^\d+$/)) {
1301         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1302         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1303             if !$vtype;
1304         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1305         verify_media_type($opt, $vtype, $drive->{media});
1306         $drive->{file} = $volid;
1307     }
1308
1309     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1310 }
1311
1312 sub parse_hotplug_features {
1313     my ($data) = @_;
1314
1315     my $res = {};
1316
1317     return $res if $data eq '0';
1318
1319     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1320
1321     foreach my $feature (PVE::Tools::split_list($data)) {
1322         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1323             $res->{$1} = 1;
1324         } else {
1325             die "invalid hotplug feature '$feature'\n";
1326         }
1327     }
1328     return $res;
1329 }
1330
1331 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1332 sub pve_verify_hotplug_features {
1333     my ($value, $noerr) = @_;
1334
1335     return $value if parse_hotplug_features($value);
1336
1337     return if $noerr;
1338
1339     die "unable to parse hotplug option\n";
1340 }
1341
1342 sub scsi_inquiry {
1343     my($fh, $noerr) = @_;
1344
1345     my $SG_IO = 0x2285;
1346     my $SG_GET_VERSION_NUM = 0x2282;
1347
1348     my $versionbuf = "\x00" x 8;
1349     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1350     if (!$ret) {
1351         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1352         return;
1353     }
1354     my $version = unpack("I", $versionbuf);
1355     if ($version < 30000) {
1356         die "scsi generic interface too old\n"  if !$noerr;
1357         return;
1358     }
1359
1360     my $buf = "\x00" x 36;
1361     my $sensebuf = "\x00" x 8;
1362     my $cmd = pack("C x3 C x1", 0x12, 36);
1363
1364     # see /usr/include/scsi/sg.h
1365     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1366
1367     my $packet = pack(
1368         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1369     );
1370
1371     $ret = ioctl($fh, $SG_IO, $packet);
1372     if (!$ret) {
1373         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1374         return;
1375     }
1376
1377     my @res = unpack($sg_io_hdr_t, $packet);
1378     if ($res[17] || $res[18]) {
1379         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1380         return;
1381     }
1382
1383     my $res = {};
1384     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1385
1386     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1387     $res->{type} &= 0x1F;
1388
1389     return $res;
1390 }
1391
1392 sub path_is_scsi {
1393     my ($path) = @_;
1394
1395     my $fh = IO::File->new("+<$path") || return;
1396     my $res = scsi_inquiry($fh, 1);
1397     close($fh);
1398
1399     return $res;
1400 }
1401
1402 sub print_tabletdevice_full {
1403     my ($conf, $arch) = @_;
1404
1405     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1406
1407     # we use uhci for old VMs because tablet driver was buggy in older qemu
1408     my $usbbus;
1409     if ($q35 || $arch eq 'aarch64') {
1410         $usbbus = 'ehci';
1411     } else {
1412         $usbbus = 'uhci';
1413     }
1414
1415     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1416 }
1417
1418 sub print_keyboarddevice_full {
1419     my ($conf, $arch) = @_;
1420
1421     return if $arch ne 'aarch64';
1422
1423     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1424 }
1425
1426 my sub get_drive_id {
1427     my ($drive) = @_;
1428     return "$drive->{interface}$drive->{index}";
1429 }
1430
1431 sub print_drivedevice_full {
1432     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1433
1434     my $device = '';
1435     my $maxdev = 0;
1436
1437     my $drive_id = get_drive_id($drive);
1438     if ($drive->{interface} eq 'virtio') {
1439         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1440         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1441         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1442     } elsif ($drive->{interface} eq 'scsi') {
1443
1444         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1445         my $unit = $drive->{index} % $maxdev;
1446         my $devicetype = 'hd';
1447         my $path = '';
1448         if (drive_is_cdrom($drive)) {
1449             $devicetype = 'cd';
1450         } else {
1451             if ($drive->{file} =~ m|^/|) {
1452                 $path = $drive->{file};
1453                 if (my $info = path_is_scsi($path)) {
1454                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1455                         $devicetype = 'block';
1456                     } elsif ($info->{type} == 1) { # tape
1457                         $devicetype = 'generic';
1458                     }
1459                 }
1460             } else {
1461                  $path = PVE::Storage::path($storecfg, $drive->{file});
1462             }
1463
1464             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1465             my $version = extract_version($machine_type, kvm_user_version());
1466             if ($path =~ m/^iscsi\:\/\// &&
1467                !min_version($version, 4, 1)) {
1468                 $devicetype = 'generic';
1469             }
1470         }
1471
1472         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1473             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1474         } else {
1475             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1476                 .",lun=$drive->{index}";
1477         }
1478         $device .= ",drive=drive-$drive_id,id=$drive_id";
1479
1480         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1481             $device .= ",rotation_rate=1";
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484
1485     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1486         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1487         my $controller = int($drive->{index} / $maxdev);
1488         my $unit = $drive->{index} % $maxdev;
1489
1490         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1491         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1492         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1493         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1494         # were before. Move odd ones up by 2 where they don't clash.
1495         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1496             $controller += 2 * ($unit % 2);
1497             $unit = 0;
1498         }
1499
1500         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1501
1502         $device = "ide-$devicetype";
1503         if ($drive->{interface} eq 'ide') {
1504             $device .= ",bus=ide.$controller,unit=$unit";
1505         } else {
1506             $device .= ",bus=ahci$controller.$unit";
1507         }
1508         $device .= ",drive=drive-$drive_id,id=$drive_id";
1509
1510         if ($devicetype eq 'hd') {
1511             if (my $model = $drive->{model}) {
1512                 $model = URI::Escape::uri_unescape($model);
1513                 $device .= ",model=$model";
1514             }
1515             if ($drive->{ssd}) {
1516                 $device .= ",rotation_rate=1";
1517             }
1518         }
1519         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1520     } elsif ($drive->{interface} eq 'usb') {
1521         die "implement me";
1522         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1523     } else {
1524         die "unsupported interface type";
1525     }
1526
1527     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1528
1529     if (my $serial = $drive->{serial}) {
1530         $serial = URI::Escape::uri_unescape($serial);
1531         $device .= ",serial=$serial";
1532     }
1533
1534
1535     return $device;
1536 }
1537
1538 sub get_initiator_name {
1539     my $initiator;
1540
1541     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1542     while (defined(my $line = <$fh>)) {
1543         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1544         $initiator = $1;
1545         last;
1546     }
1547     $fh->close();
1548
1549     return $initiator;
1550 }
1551
1552 my sub storage_allows_io_uring_default {
1553     my ($scfg, $cache_direct) = @_;
1554
1555     # io_uring with cache mode writeback or writethrough on krbd will hang...
1556     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1557
1558     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1559     # sometimes, just plain disable...
1560     return if $scfg && $scfg->{type} eq 'lvm';
1561
1562     # io_uring causes problems when used with CIFS since kernel 5.15
1563     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1564     return if $scfg && $scfg->{type} eq 'cifs';
1565
1566     return 1;
1567 }
1568
1569 my sub drive_uses_cache_direct {
1570     my ($drive, $scfg) = @_;
1571
1572     my $cache_direct = 0;
1573
1574     if (my $cache = $drive->{cache}) {
1575         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1576     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1577         $cache_direct = 1;
1578     }
1579
1580     return $cache_direct;
1581 }
1582
1583 sub print_drive_commandline_full {
1584     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1585
1586     my $path;
1587     my $volid = $drive->{file};
1588     my $format = $drive->{format};
1589     my $drive_id = get_drive_id($drive);
1590
1591     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1592     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1593
1594     if (drive_is_cdrom($drive)) {
1595         $path = get_iso_path($storecfg, $vmid, $volid);
1596         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1597     } else {
1598         if ($storeid) {
1599             $path = PVE::Storage::path($storecfg, $volid);
1600             $format //= qemu_img_format($scfg, $volname);
1601         } else {
1602             $path = $volid;
1603             $format //= "raw";
1604         }
1605    }
1606
1607    my $is_rbd = $path =~ m/^rbd:/;
1608
1609     my $opts = '';
1610     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1611     foreach my $o (@qemu_drive_options) {
1612         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1613     }
1614
1615     # snapshot only accepts on|off
1616     if (defined($drive->{snapshot})) {
1617         my $v = $drive->{snapshot} ? 'on' : 'off';
1618         $opts .= ",snapshot=$v";
1619     }
1620
1621     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1622         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1623     }
1624
1625     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1626         my ($dir, $qmpname) = @$type;
1627         if (my $v = $drive->{"mbps$dir"}) {
1628             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1629         }
1630         if (my $v = $drive->{"mbps${dir}_max"}) {
1631             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1632         }
1633         if (my $v = $drive->{"bps${dir}_max_length"}) {
1634             $opts .= ",throttling.bps$qmpname-max-length=$v";
1635         }
1636         if (my $v = $drive->{"iops${dir}"}) {
1637             $opts .= ",throttling.iops$qmpname=$v";
1638         }
1639         if (my $v = $drive->{"iops${dir}_max"}) {
1640             $opts .= ",throttling.iops$qmpname-max=$v";
1641         }
1642         if (my $v = $drive->{"iops${dir}_max_length"}) {
1643             $opts .= ",throttling.iops$qmpname-max-length=$v";
1644         }
1645     }
1646
1647     if ($pbs_name) {
1648         $format = "rbd" if $is_rbd;
1649         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1650             if !$format;
1651         $opts .= ",format=alloc-track,file.driver=$format";
1652     } elsif ($format) {
1653         $opts .= ",format=$format";
1654     }
1655
1656     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1657
1658     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1659
1660     if (!$drive->{aio}) {
1661         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1662             # io_uring supports all cache modes
1663             $opts .= ",aio=io_uring";
1664         } else {
1665             # aio native works only with O_DIRECT
1666             if($cache_direct) {
1667                 $opts .= ",aio=native";
1668             } else {
1669                 $opts .= ",aio=threads";
1670             }
1671         }
1672     }
1673
1674     if (!drive_is_cdrom($drive)) {
1675         my $detectzeroes;
1676         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1677             $detectzeroes = 'off';
1678         } elsif ($drive->{discard}) {
1679             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1680         } else {
1681             # This used to be our default with discard not being specified:
1682             $detectzeroes = 'on';
1683         }
1684
1685         # note: 'detect-zeroes' works per blockdev and we want it to persist
1686         # after the alloc-track is removed, so put it on 'file' directly
1687         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1688         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1689     }
1690
1691     if ($pbs_name) {
1692         $opts .= ",backing=$pbs_name";
1693         $opts .= ",auto-remove=on";
1694     }
1695
1696     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1697     my $file_param = "file";
1698     if ($pbs_name) {
1699         # non-rbd drivers require the underlying file to be a seperate block
1700         # node, so add a second .file indirection
1701         $file_param .= ".file" if !$is_rbd;
1702         $file_param .= ".filename";
1703     }
1704     my $pathinfo = $path ? "$file_param=$path," : '';
1705
1706     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1707 }
1708
1709 sub print_pbs_blockdev {
1710     my ($pbs_conf, $pbs_name) = @_;
1711     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1712     $blockdev .= ",repository=$pbs_conf->{repository}";
1713     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1714     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1715     $blockdev .= ",archive=$pbs_conf->{archive}";
1716     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1717     return $blockdev;
1718 }
1719
1720 sub print_netdevice_full {
1721     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1722
1723     my $device = $net->{model};
1724     if ($net->{model} eq 'virtio') {
1725          $device = 'virtio-net-pci';
1726      };
1727
1728     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1729     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1730     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1731         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1732         # and out of each queue plus one config interrupt and control vector queue
1733         my $vectors = $net->{queues} * 2 + 2;
1734         $tmpstr .= ",vectors=$vectors,mq=on";
1735         if (min_version($machine_version, 7, 1)) {
1736             $tmpstr .= ",packed=on";
1737         }
1738     }
1739
1740     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1741         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1742     }
1743
1744     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1745
1746     if (my $mtu = $net->{mtu}) {
1747         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1748             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1749             if ($mtu == 1) {
1750                  $mtu = $bridge_mtu;
1751             } elsif ($mtu < 576) {
1752                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1753             } elsif ($mtu > $bridge_mtu) {
1754                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1755             }
1756             $tmpstr .= ",host_mtu=$mtu";
1757         } else {
1758             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1759         }
1760     }
1761
1762     if ($use_old_bios_files) {
1763         my $romfile;
1764         if ($device eq 'virtio-net-pci') {
1765             $romfile = 'pxe-virtio.rom';
1766         } elsif ($device eq 'e1000') {
1767             $romfile = 'pxe-e1000.rom';
1768         } elsif ($device eq 'e1000e') {
1769             $romfile = 'pxe-e1000e.rom';
1770         } elsif ($device eq 'ne2k') {
1771             $romfile = 'pxe-ne2k_pci.rom';
1772         } elsif ($device eq 'pcnet') {
1773             $romfile = 'pxe-pcnet.rom';
1774         } elsif ($device eq 'rtl8139') {
1775             $romfile = 'pxe-rtl8139.rom';
1776         }
1777         $tmpstr .= ",romfile=$romfile" if $romfile;
1778     }
1779
1780     return $tmpstr;
1781 }
1782
1783 sub print_netdev_full {
1784     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1785
1786     my $i = '';
1787     if ($netid =~ m/^net(\d+)$/) {
1788         $i = int($1);
1789     }
1790
1791     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1792
1793     my $ifname = "tap${vmid}i$i";
1794
1795     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1796     die "interface name '$ifname' is too long (max 15 character)\n"
1797         if length($ifname) >= 16;
1798
1799     my $vhostparam = '';
1800     if (is_native($arch)) {
1801         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1802     }
1803
1804     my $vmname = $conf->{name} || "vm$vmid";
1805
1806     my $netdev = "";
1807     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1808
1809     if ($net->{bridge}) {
1810         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1811             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1812     } else {
1813         $netdev = "type=user,id=$netid,hostname=$vmname";
1814     }
1815
1816     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1817
1818     return $netdev;
1819 }
1820
1821 my $vga_map = {
1822     'cirrus' => 'cirrus-vga',
1823     'std' => 'VGA',
1824     'vmware' => 'vmware-svga',
1825     'virtio' => 'virtio-vga',
1826     'virtio-gl' => 'virtio-vga-gl',
1827 };
1828
1829 sub print_vga_device {
1830     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1831
1832     my $type = $vga_map->{$vga->{type}};
1833     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1834         $type = 'virtio-gpu';
1835     }
1836     my $vgamem_mb = $vga->{memory};
1837
1838     my $max_outputs = '';
1839     if ($qxlnum) {
1840         $type = $id ? 'qxl' : 'qxl-vga';
1841
1842         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1843             # set max outputs so linux can have up to 4 qxl displays with one device
1844             if (min_version($machine_version, 4, 1)) {
1845                 $max_outputs = ",max_outputs=4";
1846             }
1847         }
1848     }
1849
1850     die "no devicetype for $vga->{type}\n" if !$type;
1851
1852     my $memory = "";
1853     if ($vgamem_mb) {
1854         if ($vga->{type} =~ /^virtio/) {
1855             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1856             $memory = ",max_hostmem=$bytes";
1857         } elsif ($qxlnum) {
1858             # from https://www.spice-space.org/multiple-monitors.html
1859             $memory = ",vgamem_mb=$vga->{memory}";
1860             my $ram = $vgamem_mb * 4;
1861             my $vram = $vgamem_mb * 2;
1862             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1863         } else {
1864             $memory = ",vgamem_mb=$vga->{memory}";
1865         }
1866     } elsif ($qxlnum && $id) {
1867         $memory = ",ram_size=67108864,vram_size=33554432";
1868     }
1869
1870     my $edidoff = "";
1871     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1872         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1873     }
1874
1875     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1876     my $vgaid = "vga" . ($id // '');
1877     my $pciaddr;
1878     if ($q35 && $vgaid eq 'vga') {
1879         # the first display uses pcie.0 bus on q35 machines
1880         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1881     } else {
1882         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1883     }
1884
1885     if ($vga->{type} eq 'virtio-gl') {
1886         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1887         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1888             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1889
1890         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1891             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1892     }
1893
1894     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1895 }
1896
1897 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1898 sub parse_net {
1899     my ($data, $disable_mac_autogen) = @_;
1900
1901     my $res = eval { parse_property_string($net_fmt, $data) };
1902     if ($@) {
1903         warn $@;
1904         return;
1905     }
1906     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1907         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1908         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1909     }
1910     return $res;
1911 }
1912
1913 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1914 sub parse_ipconfig {
1915     my ($data) = @_;
1916
1917     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1918     if ($@) {
1919         warn $@;
1920         return;
1921     }
1922
1923     if ($res->{gw} && !$res->{ip}) {
1924         warn 'gateway specified without specifying an IP address';
1925         return;
1926     }
1927     if ($res->{gw6} && !$res->{ip6}) {
1928         warn 'IPv6 gateway specified without specifying an IPv6 address';
1929         return;
1930     }
1931     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1932         warn 'gateway specified together with DHCP';
1933         return;
1934     }
1935     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1936         # gw6 + auto/dhcp
1937         warn "IPv6 gateway specified together with $res->{ip6} address";
1938         return;
1939     }
1940
1941     if (!$res->{ip} && !$res->{ip6}) {
1942         return { ip => 'dhcp', ip6 => 'dhcp' };
1943     }
1944
1945     return $res;
1946 }
1947
1948 sub print_net {
1949     my $net = shift;
1950
1951     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1952 }
1953
1954 sub add_random_macs {
1955     my ($settings) = @_;
1956
1957     foreach my $opt (keys %$settings) {
1958         next if $opt !~ m/^net(\d+)$/;
1959         my $net = parse_net($settings->{$opt});
1960         next if !$net;
1961         $settings->{$opt} = print_net($net);
1962     }
1963 }
1964
1965 sub vm_is_volid_owner {
1966     my ($storecfg, $vmid, $volid) = @_;
1967
1968     if ($volid !~  m|^/|) {
1969         my ($path, $owner);
1970         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1971         if ($owner && ($owner == $vmid)) {
1972             return 1;
1973         }
1974     }
1975
1976     return;
1977 }
1978
1979 sub vmconfig_register_unused_drive {
1980     my ($storecfg, $vmid, $conf, $drive) = @_;
1981
1982     if (drive_is_cloudinit($drive)) {
1983         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1984         warn $@ if $@;
1985         delete $conf->{cloudinit};
1986     } elsif (!drive_is_cdrom($drive)) {
1987         my $volid = $drive->{file};
1988         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1989             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1990         }
1991     }
1992 }
1993
1994 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1995 my $smbios1_fmt = {
1996     uuid => {
1997         type => 'string',
1998         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1999         format_description => 'UUID',
2000         description => "Set SMBIOS1 UUID.",
2001         optional => 1,
2002     },
2003     version => {
2004         type => 'string',
2005         pattern => '[A-Za-z0-9+\/]+={0,2}',
2006         format_description => 'Base64 encoded string',
2007         description => "Set SMBIOS1 version.",
2008         optional => 1,
2009     },
2010     serial => {
2011         type => 'string',
2012         pattern => '[A-Za-z0-9+\/]+={0,2}',
2013         format_description => 'Base64 encoded string',
2014         description => "Set SMBIOS1 serial number.",
2015         optional => 1,
2016     },
2017     manufacturer => {
2018         type => 'string',
2019         pattern => '[A-Za-z0-9+\/]+={0,2}',
2020         format_description => 'Base64 encoded string',
2021         description => "Set SMBIOS1 manufacturer.",
2022         optional => 1,
2023     },
2024     product => {
2025         type => 'string',
2026         pattern => '[A-Za-z0-9+\/]+={0,2}',
2027         format_description => 'Base64 encoded string',
2028         description => "Set SMBIOS1 product ID.",
2029         optional => 1,
2030     },
2031     sku => {
2032         type => 'string',
2033         pattern => '[A-Za-z0-9+\/]+={0,2}',
2034         format_description => 'Base64 encoded string',
2035         description => "Set SMBIOS1 SKU string.",
2036         optional => 1,
2037     },
2038     family => {
2039         type => 'string',
2040         pattern => '[A-Za-z0-9+\/]+={0,2}',
2041         format_description => 'Base64 encoded string',
2042         description => "Set SMBIOS1 family string.",
2043         optional => 1,
2044     },
2045     base64 => {
2046         type => 'boolean',
2047         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2048         optional => 1,
2049     },
2050 };
2051
2052 sub parse_smbios1 {
2053     my ($data) = @_;
2054
2055     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2056     warn $@ if $@;
2057     return $res;
2058 }
2059
2060 sub print_smbios1 {
2061     my ($smbios1) = @_;
2062     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2063 }
2064
2065 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2066
2067 sub parse_watchdog {
2068     my ($value) = @_;
2069
2070     return if !$value;
2071
2072     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2073     warn $@ if $@;
2074     return $res;
2075 }
2076
2077 sub parse_guest_agent {
2078     my ($conf) = @_;
2079
2080     return {} if !defined($conf->{agent});
2081
2082     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2083     warn $@ if $@;
2084
2085     # if the agent is disabled ignore the other potentially set properties
2086     return {} if !$res->{enabled};
2087     return $res;
2088 }
2089
2090 sub get_qga_key {
2091     my ($conf, $key) = @_;
2092     return undef if !defined($conf->{agent});
2093
2094     my $agent = parse_guest_agent($conf);
2095     return $agent->{$key};
2096 }
2097
2098 sub parse_vga {
2099     my ($value) = @_;
2100
2101     return {} if !$value;
2102     my $res = eval { parse_property_string($vga_fmt, $value) };
2103     warn $@ if $@;
2104     return $res;
2105 }
2106
2107 sub parse_rng {
2108     my ($value) = @_;
2109
2110     return if !$value;
2111
2112     my $res = eval { parse_property_string($rng_fmt, $value) };
2113     warn $@ if $@;
2114     return $res;
2115 }
2116
2117 sub parse_meta_info {
2118     my ($value) = @_;
2119
2120     return if !$value;
2121
2122     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub new_meta_info_string {
2128     my () = @_; # for now do not allow to override any value
2129
2130     return PVE::JSONSchema::print_property_string(
2131         {
2132             'creation-qemu' => kvm_user_version(),
2133             ctime => "". int(time()),
2134         },
2135         $meta_info_fmt
2136     );
2137 }
2138
2139 sub qemu_created_version_fixups {
2140     my ($conf, $forcemachine, $kvmver) = @_;
2141
2142     my $meta = parse_meta_info($conf->{meta}) // {};
2143     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2144
2145     # check if we need to apply some handling for VMs that always use the latest machine version but
2146     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2147     # would be required (we do not want to pin machine version for non-windows OS type)
2148     if (
2149         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2150         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2151         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2152         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2153     ) {
2154         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2155         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2156             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2157             # and thus with the predictable interface naming of systemd
2158             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2159         }
2160     }
2161     return;
2162 }
2163
2164 # add JSON properties for create and set function
2165 sub json_config_properties {
2166     my ($prop, $with_disk_alloc) = @_;
2167
2168     my $skip_json_config_opts = {
2169         parent => 1,
2170         snaptime => 1,
2171         vmstate => 1,
2172         runningmachine => 1,
2173         runningcpu => 1,
2174         meta => 1,
2175     };
2176
2177     foreach my $opt (keys %$confdesc) {
2178         next if $skip_json_config_opts->{$opt};
2179
2180         if ($with_disk_alloc && is_valid_drivename($opt)) {
2181             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2182         } else {
2183             $prop->{$opt} = $confdesc->{$opt};
2184         }
2185     }
2186
2187     return $prop;
2188 }
2189
2190 # Properties that we can read from an OVF file
2191 sub json_ovf_properties {
2192     my $prop = {};
2193
2194     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2195         $prop->{$device} = {
2196             type => 'string',
2197             format => 'pve-volume-id-or-absolute-path',
2198             description => "Disk image that gets imported to $device",
2199             optional => 1,
2200         };
2201     }
2202
2203     $prop->{cores} = {
2204         type => 'integer',
2205         description => "The number of CPU cores.",
2206         optional => 1,
2207     };
2208     $prop->{memory} = {
2209         type => 'integer',
2210         description => "Amount of RAM for the VM in MB.",
2211         optional => 1,
2212     };
2213     $prop->{name} = {
2214         type => 'string',
2215         description => "Name of the VM.",
2216         optional => 1,
2217     };
2218
2219     return $prop;
2220 }
2221
2222 # return copy of $confdesc_cloudinit to generate documentation
2223 sub cloudinit_config_properties {
2224
2225     return dclone($confdesc_cloudinit);
2226 }
2227
2228 sub cloudinit_pending_properties {
2229     my $p = {
2230         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2231         name => 1,
2232     };
2233     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2234     return $p;
2235 }
2236
2237 sub check_type {
2238     my ($key, $value) = @_;
2239
2240     die "unknown setting '$key'\n" if !$confdesc->{$key};
2241
2242     my $type = $confdesc->{$key}->{type};
2243
2244     if (!defined($value)) {
2245         die "got undefined value\n";
2246     }
2247
2248     if ($value =~ m/[\n\r]/) {
2249         die "property contains a line feed\n";
2250     }
2251
2252     if ($type eq 'boolean') {
2253         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2254         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2255         die "type check ('boolean') failed - got '$value'\n";
2256     } elsif ($type eq 'integer') {
2257         return int($1) if $value =~ m/^(\d+)$/;
2258         die "type check ('integer') failed - got '$value'\n";
2259     } elsif ($type eq 'number') {
2260         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2261         die "type check ('number') failed - got '$value'\n";
2262     } elsif ($type eq 'string') {
2263         if (my $fmt = $confdesc->{$key}->{format}) {
2264             PVE::JSONSchema::check_format($fmt, $value);
2265             return $value;
2266         }
2267         $value =~ s/^\"(.*)\"$/$1/;
2268         return $value;
2269     } else {
2270         die "internal error"
2271     }
2272 }
2273
2274 sub destroy_vm {
2275     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2276
2277     my $conf = PVE::QemuConfig->load_config($vmid);
2278
2279     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2280         PVE::QemuConfig->check_lock($conf);
2281     }
2282
2283     if ($conf->{template}) {
2284         # check if any base image is still used by a linked clone
2285         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2286                 my ($ds, $drive) = @_;
2287                 return if drive_is_cdrom($drive);
2288
2289                 my $volid = $drive->{file};
2290                 return if !$volid || $volid =~ m|^/|;
2291
2292                 die "base volume '$volid' is still in use by linked cloned\n"
2293                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2294
2295         });
2296     }
2297
2298     my $volids = {};
2299     my $remove_owned_drive = sub {
2300         my ($ds, $drive) = @_;
2301         return if drive_is_cdrom($drive, 1);
2302
2303         my $volid = $drive->{file};
2304         return if !$volid || $volid =~ m|^/|;
2305         return if $volids->{$volid};
2306
2307         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2308         return if !$path || !$owner || ($owner != $vmid);
2309
2310         $volids->{$volid} = 1;
2311         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2312         warn "Could not remove disk '$volid', check manually: $@" if $@;
2313     };
2314
2315     # only remove disks owned by this VM (referenced in the config)
2316     my $include_opts = {
2317         include_unused => 1,
2318         extra_keys => ['vmstate'],
2319     };
2320     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2321
2322     for my $snap (values %{$conf->{snapshots}}) {
2323         next if !defined($snap->{vmstate});
2324         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2325         next if !defined($drive);
2326         $remove_owned_drive->('vmstate', $drive);
2327     }
2328
2329     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2330
2331     if ($purge_unreferenced) { # also remove unreferenced disk
2332         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2333         PVE::Storage::foreach_volid($vmdisks, sub {
2334             my ($volid, $sid, $volname, $d) = @_;
2335             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2336             warn $@ if $@;
2337         });
2338     }
2339
2340     if (defined $replacement_conf) {
2341         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2342     } else {
2343         PVE::QemuConfig->destroy_config($vmid);
2344     }
2345 }
2346
2347 sub parse_vm_config {
2348     my ($filename, $raw, $strict) = @_;
2349
2350     return if !defined($raw);
2351
2352     my $res = {
2353         digest => Digest::SHA::sha1_hex($raw),
2354         snapshots => {},
2355         pending => {},
2356         cloudinit => {},
2357     };
2358
2359     my $handle_error = sub {
2360         my ($msg) = @_;
2361
2362         if ($strict) {
2363             die $msg;
2364         } else {
2365             warn $msg;
2366         }
2367     };
2368
2369     $filename =~ m|/qemu-server/(\d+)\.conf$|
2370         || die "got strange filename '$filename'";
2371
2372     my $vmid = $1;
2373
2374     my $conf = $res;
2375     my $descr;
2376     my $finish_description = sub {
2377         if (defined($descr)) {
2378             $descr =~ s/\s+$//;
2379             $conf->{description} = $descr;
2380         }
2381         $descr = undef;
2382     };
2383     my $section = '';
2384
2385     my @lines = split(/\n/, $raw);
2386     foreach my $line (@lines) {
2387         next if $line =~ m/^\s*$/;
2388
2389         if ($line =~ m/^\[PENDING\]\s*$/i) {
2390             $section = 'pending';
2391             $finish_description->();
2392             $conf = $res->{$section} = {};
2393             next;
2394         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2395             $section = 'cloudinit';
2396             $finish_description->();
2397             $conf = $res->{$section} = {};
2398             next;
2399
2400         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2401             $section = $1;
2402             $finish_description->();
2403             $conf = $res->{snapshots}->{$section} = {};
2404             next;
2405         }
2406
2407         if ($line =~ m/^\#(.*)$/) {
2408             $descr = '' if !defined($descr);
2409             $descr .= PVE::Tools::decode_text($1) . "\n";
2410             next;
2411         }
2412
2413         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($2);
2416         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2417             $conf->{snapstate} = $1;
2418         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2419             my $key = $1;
2420             my $value = $2;
2421             $conf->{$key} = $value;
2422         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2423             my $value = $1;
2424             if ($section eq 'pending') {
2425                 $conf->{delete} = $value; # we parse this later
2426             } else {
2427                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2428             }
2429         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2430             my $key = $1;
2431             my $value = $2;
2432             if ($section eq 'cloudinit') {
2433                 # ignore validation only used for informative purpose
2434                 $conf->{$key} = $value;
2435                 next;
2436             }
2437             eval { $value = check_type($key, $value); };
2438             if ($@) {
2439                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2440             } else {
2441                 $key = 'ide2' if $key eq 'cdrom';
2442                 my $fmt = $confdesc->{$key}->{format};
2443                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2444                     my $v = parse_drive($key, $value);
2445                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2446                         $v->{file} = $volid;
2447                         $value = print_drive($v);
2448                     } else {
2449                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2450                         next;
2451                     }
2452                 }
2453
2454                 $conf->{$key} = $value;
2455             }
2456         } else {
2457             $handle_error->("vm $vmid - unable to parse config: $line\n");
2458         }
2459     }
2460
2461     $finish_description->();
2462     delete $res->{snapstate}; # just to be sure
2463
2464     return $res;
2465 }
2466
2467 sub write_vm_config {
2468     my ($filename, $conf) = @_;
2469
2470     delete $conf->{snapstate}; # just to be sure
2471
2472     if ($conf->{cdrom}) {
2473         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2474         $conf->{ide2} = $conf->{cdrom};
2475         delete $conf->{cdrom};
2476     }
2477
2478     # we do not use 'smp' any longer
2479     if ($conf->{sockets}) {
2480         delete $conf->{smp};
2481     } elsif ($conf->{smp}) {
2482         $conf->{sockets} = $conf->{smp};
2483         delete $conf->{cores};
2484         delete $conf->{smp};
2485     }
2486
2487     my $used_volids = {};
2488
2489     my $cleanup_config = sub {
2490         my ($cref, $pending, $snapname) = @_;
2491
2492         foreach my $key (keys %$cref) {
2493             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2494                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2495             my $value = $cref->{$key};
2496             if ($key eq 'delete') {
2497                 die "propertry 'delete' is only allowed in [PENDING]\n"
2498                     if !$pending;
2499                 # fixme: check syntax?
2500                 next;
2501             }
2502             eval { $value = check_type($key, $value); };
2503             die "unable to parse value of '$key' - $@" if $@;
2504
2505             $cref->{$key} = $value;
2506
2507             if (!$snapname && is_valid_drivename($key)) {
2508                 my $drive = parse_drive($key, $value);
2509                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2510             }
2511         }
2512     };
2513
2514     &$cleanup_config($conf);
2515
2516     &$cleanup_config($conf->{pending}, 1);
2517
2518     foreach my $snapname (keys %{$conf->{snapshots}}) {
2519         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2520         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2521     }
2522
2523     # remove 'unusedX' settings if we re-add a volume
2524     foreach my $key (keys %$conf) {
2525         my $value = $conf->{$key};
2526         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2527             delete $conf->{$key};
2528         }
2529     }
2530
2531     my $generate_raw_config = sub {
2532         my ($conf, $pending) = @_;
2533
2534         my $raw = '';
2535
2536         # add description as comment to top of file
2537         if (defined(my $descr = $conf->{description})) {
2538             if ($descr) {
2539                 foreach my $cl (split(/\n/, $descr)) {
2540                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2541                 }
2542             } else {
2543                 $raw .= "#\n" if $pending;
2544             }
2545         }
2546
2547         foreach my $key (sort keys %$conf) {
2548             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2549             $raw .= "$key: $conf->{$key}\n";
2550         }
2551         return $raw;
2552     };
2553
2554     my $raw = &$generate_raw_config($conf);
2555
2556     if (scalar(keys %{$conf->{pending}})){
2557         $raw .= "\n[PENDING]\n";
2558         $raw .= &$generate_raw_config($conf->{pending}, 1);
2559     }
2560
2561     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2562         $raw .= "\n[special:cloudinit]\n";
2563         $raw .= &$generate_raw_config($conf->{cloudinit});
2564     }
2565
2566     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2567         $raw .= "\n[$snapname]\n";
2568         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2569     }
2570
2571     return $raw;
2572 }
2573
2574 sub load_defaults {
2575
2576     my $res = {};
2577
2578     # we use static defaults from our JSON schema configuration
2579     foreach my $key (keys %$confdesc) {
2580         if (defined(my $default = $confdesc->{$key}->{default})) {
2581             $res->{$key} = $default;
2582         }
2583     }
2584
2585     return $res;
2586 }
2587
2588 sub config_list {
2589     my $vmlist = PVE::Cluster::get_vmlist();
2590     my $res = {};
2591     return $res if !$vmlist || !$vmlist->{ids};
2592     my $ids = $vmlist->{ids};
2593     my $nodename = nodename();
2594
2595     foreach my $vmid (keys %$ids) {
2596         my $d = $ids->{$vmid};
2597         next if !$d->{node} || $d->{node} ne $nodename;
2598         next if !$d->{type} || $d->{type} ne 'qemu';
2599         $res->{$vmid}->{exists} = 1;
2600     }
2601     return $res;
2602 }
2603
2604 # test if VM uses local resources (to prevent migration)
2605 sub check_local_resources {
2606     my ($conf, $noerr) = @_;
2607
2608     my @loc_res = ();
2609     my $mapped_res = [];
2610
2611     my $nodelist = PVE::Cluster::get_nodelist();
2612     my $pci_map = PVE::Mapping::PCI::config();
2613     my $usb_map = PVE::Mapping::USB::config();
2614
2615     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2616
2617     my $add_missing_mapping = sub {
2618         my ($type, $key, $id) = @_;
2619         for my $node (@$nodelist) {
2620             my $entry;
2621             if ($type eq 'pci') {
2622                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2623             } elsif ($type eq 'usb') {
2624                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2625             }
2626             if (!scalar($entry->@*)) {
2627                 push @{$missing_mappings_by_node->{$node}}, $key;
2628             }
2629         }
2630     };
2631
2632     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2633     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2634
2635     push @loc_res, "ivshmem" if $conf->{ivshmem};
2636
2637     foreach my $k (keys %$conf) {
2638         if ($k =~ m/^usb/) {
2639             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2640             next if $entry->{host} =~ m/^spice$/i;
2641             if ($entry->{mapping}) {
2642                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2643                 push @$mapped_res, $k;
2644             }
2645         }
2646         if ($k =~ m/^hostpci/) {
2647             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2648             if ($entry->{mapping}) {
2649                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2650                 push @$mapped_res, $k;
2651             }
2652         }
2653         # sockets are safe: they will recreated be on the target side post-migrate
2654         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2655         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2656     }
2657
2658     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2659
2660     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2661 }
2662
2663 # check if used storages are available on all nodes (use by migrate)
2664 sub check_storage_availability {
2665     my ($storecfg, $conf, $node) = @_;
2666
2667     PVE::QemuConfig->foreach_volume($conf, sub {
2668         my ($ds, $drive) = @_;
2669
2670         my $volid = $drive->{file};
2671         return if !$volid;
2672
2673         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2674         return if !$sid;
2675
2676         # check if storage is available on both nodes
2677         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2678         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2679
2680         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2681
2682         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2683             if !$scfg->{content}->{$vtype};
2684    });
2685 }
2686
2687 # list nodes where all VM images are available (used by has_feature API)
2688 sub shared_nodes {
2689     my ($conf, $storecfg) = @_;
2690
2691     my $nodelist = PVE::Cluster::get_nodelist();
2692     my $nodehash = { map { $_ => 1 } @$nodelist };
2693     my $nodename = nodename();
2694
2695     PVE::QemuConfig->foreach_volume($conf, sub {
2696         my ($ds, $drive) = @_;
2697
2698         my $volid = $drive->{file};
2699         return if !$volid;
2700
2701         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2702         if ($storeid) {
2703             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2704             if ($scfg->{disable}) {
2705                 $nodehash = {};
2706             } elsif (my $avail = $scfg->{nodes}) {
2707                 foreach my $node (keys %$nodehash) {
2708                     delete $nodehash->{$node} if !$avail->{$node};
2709                 }
2710             } elsif (!$scfg->{shared}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     delete $nodehash->{$node} if $node ne $nodename
2713                 }
2714             }
2715         }
2716     });
2717
2718     return $nodehash
2719 }
2720
2721 sub check_local_storage_availability {
2722     my ($conf, $storecfg) = @_;
2723
2724     my $nodelist = PVE::Cluster::get_nodelist();
2725     my $nodehash = { map { $_ => {} } @$nodelist };
2726
2727     PVE::QemuConfig->foreach_volume($conf, sub {
2728         my ($ds, $drive) = @_;
2729
2730         my $volid = $drive->{file};
2731         return if !$volid;
2732
2733         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2734         if ($storeid) {
2735             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2736
2737             if ($scfg->{disable}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2740                 }
2741             } elsif (my $avail = $scfg->{nodes}) {
2742                 foreach my $node (keys %$nodehash) {
2743                     if (!$avail->{$node}) {
2744                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2745                     }
2746                 }
2747             }
2748         }
2749     });
2750
2751     foreach my $node (values %$nodehash) {
2752         if (my $unavail = $node->{unavailable_storages}) {
2753             $node->{unavailable_storages} = [ sort keys %$unavail ];
2754         }
2755     }
2756
2757     return $nodehash
2758 }
2759
2760 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2761 sub check_running {
2762     my ($vmid, $nocheck, $node) = @_;
2763
2764     # $nocheck is set when called during a migration, in which case the config
2765     # file might still or already reside on the *other* node
2766     # - because rename has already happened, and current node is source
2767     # - because rename hasn't happened yet, and current node is target
2768     # - because rename has happened, current node is target, but hasn't yet
2769     # processed it yet
2770     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2771     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2772 }
2773
2774 sub vzlist {
2775
2776     my $vzlist = config_list();
2777
2778     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2779
2780     while (defined(my $de = $fd->read)) {
2781         next if $de !~ m/^(\d+)\.pid$/;
2782         my $vmid = $1;
2783         next if !defined($vzlist->{$vmid});
2784         if (my $pid = check_running($vmid)) {
2785             $vzlist->{$vmid}->{pid} = $pid;
2786         }
2787     }
2788
2789     return $vzlist;
2790 }
2791
2792 our $vmstatus_return_properties = {
2793     vmid => get_standard_option('pve-vmid'),
2794     status => {
2795         description => "QEMU process status.",
2796         type => 'string',
2797         enum => ['stopped', 'running'],
2798     },
2799     maxmem => {
2800         description => "Maximum memory in bytes.",
2801         type => 'integer',
2802         optional => 1,
2803         renderer => 'bytes',
2804     },
2805     maxdisk => {
2806         description => "Root disk size in bytes.",
2807         type => 'integer',
2808         optional => 1,
2809         renderer => 'bytes',
2810     },
2811     name => {
2812         description => "VM name.",
2813         type => 'string',
2814         optional => 1,
2815     },
2816     qmpstatus => {
2817         description => "VM run state from the 'query-status' QMP monitor command.",
2818         type => 'string',
2819         optional => 1,
2820     },
2821     pid => {
2822         description => "PID of running qemu process.",
2823         type => 'integer',
2824         optional => 1,
2825     },
2826     uptime => {
2827         description => "Uptime.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'duration',
2831     },
2832     cpus => {
2833         description => "Maximum usable CPUs.",
2834         type => 'number',
2835         optional => 1,
2836     },
2837     lock => {
2838         description => "The current config lock, if any.",
2839         type => 'string',
2840         optional => 1,
2841     },
2842     tags => {
2843         description  => "The current configured tags, if any",
2844         type => 'string',
2845         optional => 1,
2846     },
2847     'running-machine' => {
2848         description => "The currently running machine type (if running).",
2849         type => 'string',
2850         optional => 1,
2851     },
2852     'running-qemu' => {
2853         description => "The currently running QEMU version (if running).",
2854         type => 'string',
2855         optional => 1,
2856     },
2857 };
2858
2859 my $last_proc_pid_stat;
2860
2861 # get VM status information
2862 # This must be fast and should not block ($full == false)
2863 # We only query KVM using QMP if $full == true (this can be slow)
2864 sub vmstatus {
2865     my ($opt_vmid, $full) = @_;
2866
2867     my $res = {};
2868
2869     my $storecfg = PVE::Storage::config();
2870
2871     my $list = vzlist();
2872     my $defaults = load_defaults();
2873
2874     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2875
2876     my $cpucount = $cpuinfo->{cpus} || 1;
2877
2878     foreach my $vmid (keys %$list) {
2879         next if $opt_vmid && ($vmid ne $opt_vmid);
2880
2881         my $conf = PVE::QemuConfig->load_config($vmid);
2882
2883         my $d = { vmid => int($vmid) };
2884         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2885
2886         # fixme: better status?
2887         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2888
2889         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2890         if (defined($size)) {
2891             $d->{disk} = 0; # no info available
2892             $d->{maxdisk} = $size;
2893         } else {
2894             $d->{disk} = 0;
2895             $d->{maxdisk} = 0;
2896         }
2897
2898         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2899             * ($conf->{cores} || $defaults->{cores});
2900         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2901         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2902
2903         $d->{name} = $conf->{name} || "VM $vmid";
2904         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2905
2906         if ($conf->{balloon}) {
2907             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2908             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2909                 : $defaults->{shares};
2910         }
2911
2912         $d->{uptime} = 0;
2913         $d->{cpu} = 0;
2914         $d->{mem} = 0;
2915
2916         $d->{netout} = 0;
2917         $d->{netin} = 0;
2918
2919         $d->{diskread} = 0;
2920         $d->{diskwrite} = 0;
2921
2922         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2923
2924         $d->{serial} = 1 if conf_has_serial($conf);
2925         $d->{lock} = $conf->{lock} if $conf->{lock};
2926         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2927
2928         $res->{$vmid} = $d;
2929     }
2930
2931     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2932     foreach my $dev (keys %$netdev) {
2933         next if $dev !~ m/^tap([1-9]\d*)i/;
2934         my $vmid = $1;
2935         my $d = $res->{$vmid};
2936         next if !$d;
2937
2938         $d->{netout} += $netdev->{$dev}->{receive};
2939         $d->{netin} += $netdev->{$dev}->{transmit};
2940
2941         if ($full) {
2942             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2943             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2944         }
2945
2946     }
2947
2948     my $ctime = gettimeofday;
2949
2950     foreach my $vmid (keys %$list) {
2951
2952         my $d = $res->{$vmid};
2953         my $pid = $d->{pid};
2954         next if !$pid;
2955
2956         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2957         next if !$pstat; # not running
2958
2959         my $used = $pstat->{utime} + $pstat->{stime};
2960
2961         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2962
2963         if ($pstat->{vsize}) {
2964             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2965         }
2966
2967         my $old = $last_proc_pid_stat->{$pid};
2968         if (!$old) {
2969             $last_proc_pid_stat->{$pid} = {
2970                 time => $ctime,
2971                 used => $used,
2972                 cpu => 0,
2973             };
2974             next;
2975         }
2976
2977         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2978
2979         if ($dtime > 1000) {
2980             my $dutime = $used -  $old->{used};
2981
2982             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2983             $last_proc_pid_stat->{$pid} = {
2984                 time => $ctime,
2985                 used => $used,
2986                 cpu => $d->{cpu},
2987             };
2988         } else {
2989             $d->{cpu} = $old->{cpu};
2990         }
2991     }
2992
2993     return $res if !$full;
2994
2995     my $qmpclient = PVE::QMPClient->new();
2996
2997     my $ballooncb = sub {
2998         my ($vmid, $resp) = @_;
2999
3000         my $info = $resp->{'return'};
3001         return if !$info->{max_mem};
3002
3003         my $d = $res->{$vmid};
3004
3005         # use memory assigned to VM
3006         $d->{maxmem} = $info->{max_mem};
3007         $d->{balloon} = $info->{actual};
3008
3009         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3010             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3011             $d->{freemem} = $info->{free_mem};
3012         }
3013
3014         $d->{ballooninfo} = $info;
3015     };
3016
3017     my $blockstatscb = sub {
3018         my ($vmid, $resp) = @_;
3019         my $data = $resp->{'return'} || [];
3020         my $totalrdbytes = 0;
3021         my $totalwrbytes = 0;
3022
3023         for my $blockstat (@$data) {
3024             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3025             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3026
3027             $blockstat->{device} =~ s/drive-//;
3028             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3029         }
3030         $res->{$vmid}->{diskread} = $totalrdbytes;
3031         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3032     };
3033
3034     my $machinecb = sub {
3035         my ($vmid, $resp) = @_;
3036         my $data = $resp->{'return'} || [];
3037
3038         $res->{$vmid}->{'running-machine'} =
3039             PVE::QemuServer::Machine::current_from_query_machines($data);
3040     };
3041
3042     my $versioncb = sub {
3043         my ($vmid, $resp) = @_;
3044         my $data = $resp->{'return'} // {};
3045         my $version = 'unknown';
3046
3047         if (my $v = $data->{qemu}) {
3048             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3049         }
3050
3051         $res->{$vmid}->{'running-qemu'} = $version;
3052     };
3053
3054     my $statuscb = sub {
3055         my ($vmid, $resp) = @_;
3056
3057         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3058         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3059         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3060         # this fails if ballon driver is not loaded, so this must be
3061         # the last commnand (following command are aborted if this fails).
3062         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3063
3064         my $status = 'unknown';
3065         if (!defined($status = $resp->{'return'}->{status})) {
3066             warn "unable to get VM status\n";
3067             return;
3068         }
3069
3070         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3071     };
3072
3073     foreach my $vmid (keys %$list) {
3074         next if $opt_vmid && ($vmid ne $opt_vmid);
3075         next if !$res->{$vmid}->{pid}; # not running
3076         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3077     }
3078
3079     $qmpclient->queue_execute(undef, 2);
3080
3081     foreach my $vmid (keys %$list) {
3082         next if $opt_vmid && ($vmid ne $opt_vmid);
3083         next if !$res->{$vmid}->{pid}; #not running
3084
3085         # we can't use the $qmpclient since it might have already aborted on
3086         # 'query-balloon', but this might also fail for older versions...
3087         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3088         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3089     }
3090
3091     foreach my $vmid (keys %$list) {
3092         next if $opt_vmid && ($vmid ne $opt_vmid);
3093         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3094     }
3095
3096     return $res;
3097 }
3098
3099 sub conf_has_serial {
3100     my ($conf) = @_;
3101
3102     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3103         if ($conf->{"serial$i"}) {
3104             return 1;
3105         }
3106     }
3107
3108     return 0;
3109 }
3110
3111 sub conf_has_audio {
3112     my ($conf, $id) = @_;
3113
3114     $id //= 0;
3115     my $audio = $conf->{"audio$id"};
3116     return if !defined($audio);
3117
3118     my $audioproperties = parse_property_string($audio_fmt, $audio);
3119     my $audiodriver = $audioproperties->{driver} // 'spice';
3120
3121     return {
3122         dev => $audioproperties->{device},
3123         dev_id => "audiodev$id",
3124         backend => $audiodriver,
3125         backend_id => "$audiodriver-backend${id}",
3126     };
3127 }
3128
3129 sub audio_devs {
3130     my ($audio, $audiopciaddr, $machine_version) = @_;
3131
3132     my $devs = [];
3133
3134     my $id = $audio->{dev_id};
3135     my $audiodev = "";
3136     if (min_version($machine_version, 4, 2)) {
3137         $audiodev = ",audiodev=$audio->{backend_id}";
3138     }
3139
3140     if ($audio->{dev} eq 'AC97') {
3141         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3142     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3143         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3144         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3145         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3146     } else {
3147         die "unkown audio device '$audio->{dev}', implement me!";
3148     }
3149
3150     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3151
3152     return $devs;
3153 }
3154
3155 sub get_tpm_paths {
3156     my ($vmid) = @_;
3157     return {
3158         socket => "/var/run/qemu-server/$vmid.swtpm",
3159         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3160     };
3161 }
3162
3163 sub add_tpm_device {
3164     my ($vmid, $devices, $conf) = @_;
3165
3166     return if !$conf->{tpmstate0};
3167
3168     my $paths = get_tpm_paths($vmid);
3169
3170     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3171     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3172     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3173 }
3174
3175 sub start_swtpm {
3176     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3177
3178     return if !$tpmdrive;
3179
3180     my $state;
3181     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3182     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3183     if ($storeid) {
3184         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3185     } else {
3186         $state = $tpm->{file};
3187     }
3188
3189     my $paths = get_tpm_paths($vmid);
3190
3191     # during migration, we will get state from remote
3192     #
3193     if (!$migration) {
3194         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3195         my $setup_cmd = [
3196             "swtpm_setup",
3197             "--tpmstate",
3198             "file://$state",
3199             "--createek",
3200             "--create-ek-cert",
3201             "--create-platform-cert",
3202             "--lock-nvram",
3203             "--config",
3204             "/etc/swtpm_setup.conf", # do not use XDG configs
3205             "--runas",
3206             "0", # force creation as root, error if not possible
3207             "--not-overwrite", # ignore existing state, do not modify
3208         ];
3209
3210         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3211         # TPM 2.0 supports ECC crypto, use if possible
3212         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3213
3214         run_command($setup_cmd, outfunc => sub {
3215             print "swtpm_setup: $1\n";
3216         });
3217     }
3218
3219     # Used to distinguish different invocations in the log.
3220     my $log_prefix = "[id=" . int(time()) . "] ";
3221
3222     my $emulator_cmd = [
3223         "swtpm",
3224         "socket",
3225         "--tpmstate",
3226         "backend-uri=file://$state,mode=0600",
3227         "--ctrl",
3228         "type=unixio,path=$paths->{socket},mode=0600",
3229         "--pid",
3230         "file=$paths->{pid}",
3231         "--terminate", # terminate on QEMU disconnect
3232         "--daemon",
3233         "--log",
3234         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3235     ];
3236     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3237     run_command($emulator_cmd, outfunc => sub { print $1; });
3238
3239     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3240     while (! -e $paths->{pid}) {
3241         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3242         usleep(50_000);
3243     }
3244
3245     # return untainted PID of swtpm daemon so it can be killed on error
3246     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3247     return $1;
3248 }
3249
3250 sub vga_conf_has_spice {
3251     my ($vga) = @_;
3252
3253     my $vgaconf = parse_vga($vga);
3254     my $vgatype = $vgaconf->{type};
3255     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3256
3257     return $1 || 1;
3258 }
3259
3260 sub is_native($) {
3261     my ($arch) = @_;
3262     return get_host_arch() eq $arch;
3263 }
3264
3265 sub get_vm_arch {
3266     my ($conf) = @_;
3267     return $conf->{arch} // get_host_arch();
3268 }
3269
3270 my $default_machines = {
3271     x86_64 => 'pc',
3272     aarch64 => 'virt',
3273 };
3274
3275 sub get_installed_machine_version {
3276     my ($kvmversion) = @_;
3277     $kvmversion = kvm_user_version() if !defined($kvmversion);
3278     $kvmversion =~ m/^(\d+\.\d+)/;
3279     return $1;
3280 }
3281
3282 sub windows_get_pinned_machine_version {
3283     my ($machine, $base_version, $kvmversion) = @_;
3284
3285     my $pin_version = $base_version;
3286     if (!defined($base_version) ||
3287         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3288     ) {
3289         $pin_version = get_installed_machine_version($kvmversion);
3290     }
3291     if (!$machine || $machine eq 'pc') {
3292         $machine = "pc-i440fx-$pin_version";
3293     } elsif ($machine eq 'q35') {
3294         $machine = "pc-q35-$pin_version";
3295     } elsif ($machine eq 'virt') {
3296         $machine = "virt-$pin_version";
3297     } else {
3298         warn "unknown machine type '$machine', not touching that!\n";
3299     }
3300
3301     return $machine;
3302 }
3303
3304 sub get_vm_machine {
3305     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3306
3307     my $machine = $forcemachine || $conf->{machine};
3308
3309     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3310         $kvmversion //= kvm_user_version();
3311         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3312         # layout which confuses windows quite a bit and may result in various regressions..
3313         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3314         if (windows_version($conf->{ostype})) {
3315             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3316         }
3317         $arch //= 'x86_64';
3318         $machine ||= $default_machines->{$arch};
3319         if ($add_pve_version) {
3320             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3321             $machine .= "+pve$pvever";
3322         }
3323     }
3324
3325     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3326         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3327         $machine = $1 if $is_pxe;
3328
3329         # for version-pinned machines that do not include a pve-version (e.g.
3330         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3331         $machine .= '+pve0';
3332
3333         $machine .= '.pxe' if $is_pxe;
3334     }
3335
3336     return $machine;
3337 }
3338
3339 sub get_ovmf_files($$$) {
3340     my ($arch, $efidisk, $smm) = @_;
3341
3342     my $types = $OVMF->{$arch}
3343         or die "no OVMF images known for architecture '$arch'\n";
3344
3345     my $type = 'default';
3346     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3347         $type = $smm ? "4m" : "4m-no-smm";
3348         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3349     }
3350
3351     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3352     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3353     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3354
3355     return ($ovmf_code, $ovmf_vars);
3356 }
3357
3358 my $Arch2Qemu = {
3359     aarch64 => '/usr/bin/qemu-system-aarch64',
3360     x86_64 => '/usr/bin/qemu-system-x86_64',
3361 };
3362 sub get_command_for_arch($) {
3363     my ($arch) = @_;
3364     return '/usr/bin/kvm' if is_native($arch);
3365
3366     my $cmd = $Arch2Qemu->{$arch}
3367         or die "don't know how to emulate architecture '$arch'\n";
3368     return $cmd;
3369 }
3370
3371 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3372 # to use in a QEMU command line (-cpu element), first array_intersect the result
3373 # of query_supported_ with query_understood_. This is necessary because:
3374 #
3375 # a) query_understood_ returns flags the host cannot use and
3376 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3377 #    flags, but CPU settings - with most of them being flags. Those settings
3378 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3379 #
3380 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3381 # expensive. If you need the value returned from this, you can get it much
3382 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3383 # $accel being 'kvm' or 'tcg'.
3384 #
3385 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3386 # changes, automatically populating pmxcfs.
3387 #
3388 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3389 # since kvm and tcg machines support different flags
3390 #
3391 sub query_supported_cpu_flags {
3392     my ($arch) = @_;
3393
3394     $arch //= get_host_arch();
3395     my $default_machine = $default_machines->{$arch};
3396
3397     my $flags = {};
3398
3399     # FIXME: Once this is merged, the code below should work for ARM as well:
3400     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3401     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3402         $arch eq "aarch64";
3403
3404     my $kvm_supported = defined(kvm_version());
3405     my $qemu_cmd = get_command_for_arch($arch);
3406     my $fakevmid = -1;
3407     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3408
3409     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3410     my $query_supported_run_qemu = sub {
3411         my ($kvm) = @_;
3412
3413         my $flags = {};
3414         my $cmd = [
3415             $qemu_cmd,
3416             '-machine', $default_machine,
3417             '-display', 'none',
3418             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3419             '-mon', 'chardev=qmp,mode=control',
3420             '-pidfile', $pidfile,
3421             '-S', '-daemonize'
3422         ];
3423
3424         if (!$kvm) {
3425             push @$cmd, '-accel', 'tcg';
3426         }
3427
3428         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3429         die "QEMU flag querying VM exited with code " . $rc if $rc;
3430
3431         eval {
3432             my $cmd_result = mon_cmd(
3433                 $fakevmid,
3434                 'query-cpu-model-expansion',
3435                 type => 'full',
3436                 model => { name => 'host' }
3437             );
3438
3439             my $props = $cmd_result->{model}->{props};
3440             foreach my $prop (keys %$props) {
3441                 next if $props->{$prop} ne '1';
3442                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3443                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3444                 # We only keep those with underscores, to match /proc/cpuinfo
3445                 $prop =~ s/\.|-/_/g;
3446                 $flags->{$prop} = 1;
3447             }
3448         };
3449         my $err = $@;
3450
3451         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3452         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3453
3454         die $err if $err;
3455
3456         return [ sort keys %$flags ];
3457     };
3458
3459     # We need to query QEMU twice, since KVM and TCG have different supported flags
3460     PVE::QemuConfig->lock_config($fakevmid, sub {
3461         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3462         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3463
3464         if ($kvm_supported) {
3465             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3466             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3467         }
3468     });
3469
3470     return $flags;
3471 }
3472
3473 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3474 my $understood_cpu_flag_dir = "/usr/share/kvm";
3475 sub query_understood_cpu_flags {
3476     my $arch = get_host_arch();
3477     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3478
3479     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3480         if ! -e $filepath;
3481
3482     my $raw = file_get_contents($filepath);
3483     $raw =~ s/^\s+|\s+$//g;
3484     my @flags = split(/\s+/, $raw);
3485
3486     return \@flags;
3487 }
3488
3489 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3490 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3491 my sub should_disable_smm {
3492     my ($conf, $vga, $machine) = @_;
3493
3494     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3495
3496     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3497         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3498 }
3499
3500 my sub print_ovmf_drive_commandlines {
3501     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3502
3503     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3504
3505     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3506
3507     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3508     if ($d) {
3509         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3510         my ($path, $format) = $d->@{'file', 'format'};
3511         if ($storeid) {
3512             $path = PVE::Storage::path($storecfg, $d->{file});
3513             if (!defined($format)) {
3514                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3515                 $format = qemu_img_format($scfg, $volname);
3516             }
3517         } elsif (!defined($format)) {
3518             die "efidisk format must be specified\n";
3519         }
3520         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3521         if ($path =~ m/^rbd:/) {
3522             $var_drive_str .= ',cache=writeback';
3523             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3524         }
3525         $var_drive_str .= ",format=$format,file=$path";
3526
3527         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3528         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3529     } else {
3530         log_warn("no efidisk configured! Using temporary efivars disk.");
3531         my $path = "/tmp/$vmid-ovmf.fd";
3532         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3533         $var_drive_str .= ",format=raw,file=$path";
3534         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3535     }
3536
3537     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3538 }
3539
3540 sub config_to_command {
3541     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3542         $pbs_backing) = @_;
3543
3544     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3545     my $devices = [];
3546     my $bridges = {};
3547     my $ostype = $conf->{ostype};
3548     my $winversion = windows_version($ostype);
3549     my $kvm = $conf->{kvm};
3550     my $nodename = nodename();
3551
3552     my $arch = get_vm_arch($conf);
3553     my $kvm_binary = get_command_for_arch($arch);
3554     my $kvmver = kvm_user_version($kvm_binary);
3555
3556     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3557         $kvmver //= "undefined";
3558         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3559     }
3560
3561     my $add_pve_version = min_version($kvmver, 4, 1);
3562
3563     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3564     my $machine_version = extract_version($machine_type, $kvmver);
3565     $kvm //= 1 if is_native($arch);
3566
3567     $machine_version =~ m/(\d+)\.(\d+)/;
3568     my ($machine_major, $machine_minor) = ($1, $2);
3569
3570     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3571         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3572     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3573         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3574             ." please upgrade node '$nodename'\n"
3575     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3576         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3577         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3578             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3579             ." node '$nodename'\n";
3580     }
3581
3582     # if a specific +pve version is required for a feature, use $version_guard
3583     # instead of min_version to allow machines to be run with the minimum
3584     # required version
3585     my $required_pve_version = 0;
3586     my $version_guard = sub {
3587         my ($major, $minor, $pve) = @_;
3588         return 0 if !min_version($machine_version, $major, $minor, $pve);
3589         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3590         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3591         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3592         return 1;
3593     };
3594
3595     if ($kvm && !defined kvm_version()) {
3596         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3597             ." or enable in BIOS.\n";
3598     }
3599
3600     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3601     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3602     my $use_old_bios_files = undef;
3603     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3604
3605     my $cmd = [];
3606     if ($conf->{affinity}) {
3607         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3608     }
3609
3610     push @$cmd, $kvm_binary;
3611
3612     push @$cmd, '-id', $vmid;
3613
3614     my $vmname = $conf->{name} || "vm$vmid";
3615
3616     push @$cmd, '-name', "$vmname,debug-threads=on";
3617
3618     push @$cmd, '-no-shutdown';
3619
3620     my $use_virtio = 0;
3621
3622     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3623     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3624     push @$cmd, '-mon', "chardev=qmp,mode=control";
3625
3626     if (min_version($machine_version, 2, 12)) {
3627         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3628         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3629     }
3630
3631     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3632
3633     push @$cmd, '-daemonize';
3634
3635     if ($conf->{smbios1}) {
3636         my $smbios_conf = parse_smbios1($conf->{smbios1});
3637         if ($smbios_conf->{base64}) {
3638             # Do not pass base64 flag to qemu
3639             delete $smbios_conf->{base64};
3640             my $smbios_string = "";
3641             foreach my $key (keys %$smbios_conf) {
3642                 my $value;
3643                 if ($key eq "uuid") {
3644                     $value = $smbios_conf->{uuid}
3645                 } else {
3646                     $value = decode_base64($smbios_conf->{$key});
3647                 }
3648                 # qemu accepts any binary data, only commas need escaping by double comma
3649                 $value =~ s/,/,,/g;
3650                 $smbios_string .= "," . $key . "=" . $value if $value;
3651             }
3652             push @$cmd, '-smbios', "type=1" . $smbios_string;
3653         } else {
3654             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3655         }
3656     }
3657
3658     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3659         my ($code_drive_str, $var_drive_str) =
3660             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3661         push $cmd->@*, '-drive', $code_drive_str;
3662         push $cmd->@*, '-drive', $var_drive_str;
3663     }
3664
3665     if ($q35) { # tell QEMU to load q35 config early
3666         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3667         if (min_version($machine_version, 4, 0)) {
3668             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3669         } else {
3670             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3671         }
3672     }
3673
3674     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3675         push @$cmd, $fixups->@*;
3676     }
3677
3678     if ($conf->{vmgenid}) {
3679         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3680     }
3681
3682     # add usb controllers
3683     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3684         $conf, $bridges, $arch, $machine_type, $machine_version);
3685     push @$devices, @usbcontrollers if @usbcontrollers;
3686     my $vga = parse_vga($conf->{vga});
3687
3688     my $qxlnum = vga_conf_has_spice($conf->{vga});
3689     $vga->{type} = 'qxl' if $qxlnum;
3690
3691     if (!$vga->{type}) {
3692         if ($arch eq 'aarch64') {
3693             $vga->{type} = 'virtio';
3694         } elsif (min_version($machine_version, 2, 9)) {
3695             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3696         } else {
3697             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3698         }
3699     }
3700
3701     # enable absolute mouse coordinates (needed by vnc)
3702     my $tablet = $conf->{tablet};
3703     if (!defined($tablet)) {
3704         $tablet = $defaults->{tablet};
3705         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3706         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3707     }
3708
3709     if ($tablet) {
3710         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3711         my $kbd = print_keyboarddevice_full($conf, $arch);
3712         push @$devices, '-device', $kbd if defined($kbd);
3713     }
3714
3715     my $bootorder = device_bootorder($conf);
3716
3717     # host pci device passthrough
3718     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3719         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3720
3721     # usb devices
3722     my $usb_dev_features = {};
3723     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3724
3725     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3726         $conf, $usb_dev_features, $bootorder, $machine_version);
3727     push @$devices, @usbdevices if @usbdevices;
3728
3729     # serial devices
3730     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3731         my $path = $conf->{"serial$i"} or next;
3732         if ($path eq 'socket') {
3733             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3734             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3735             # On aarch64, serial0 is the UART device. QEMU only allows
3736             # connecting UART devices via the '-serial' command line, as
3737             # the device has a fixed slot on the hardware...
3738             if ($arch eq 'aarch64' && $i == 0) {
3739                 push @$devices, '-serial', "chardev:serial$i";
3740             } else {
3741                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3742             }
3743         } else {
3744             die "no such serial device\n" if ! -c $path;
3745             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3746             push @$devices, '-device', "isa-serial,chardev=serial$i";
3747         }
3748     }
3749
3750     # parallel devices
3751     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3752         if (my $path = $conf->{"parallel$i"}) {
3753             die "no such parallel device\n" if ! -c $path;
3754             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3755             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3756             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3757         }
3758     }
3759
3760     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3761         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3762         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3763         push @$devices, @$audio_devs;
3764     }
3765
3766     # Add a TPM only if the VM is not a template,
3767     # to support backing up template VMs even if the TPM disk is write-protected.
3768     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3769
3770     my $sockets = 1;
3771     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3772     $sockets = $conf->{sockets} if  $conf->{sockets};
3773
3774     my $cores = $conf->{cores} || 1;
3775
3776     my $maxcpus = $sockets * $cores;
3777
3778     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3779
3780     my $allowed_vcpus = $cpuinfo->{cpus};
3781
3782     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3783
3784     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3785         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3786         for (my $i = 2; $i <= $vcpus; $i++)  {
3787             my $cpustr = print_cpu_device($conf,$i);
3788             push @$cmd, '-device', $cpustr;
3789         }
3790
3791     } else {
3792
3793         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3794     }
3795     push @$cmd, '-nodefaults';
3796
3797     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3798
3799     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3800
3801     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3802
3803     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3804         push @$devices, '-device', print_vga_device(
3805             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3806
3807         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3808
3809         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3810         push @$cmd,  '-vnc', "unix:$socket,password=on";
3811     } else {
3812         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3813         push @$cmd, '-nographic';
3814     }
3815
3816     # time drift fix
3817     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3818     my $useLocaltime = $conf->{localtime};
3819
3820     if ($winversion >= 5) { # windows
3821         $useLocaltime = 1 if !defined($conf->{localtime});
3822
3823         # use time drift fix when acpi is enabled
3824         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3825             $tdf = 1 if !defined($conf->{tdf});
3826         }
3827     }
3828
3829     if ($winversion >= 6) {
3830         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3831         push @$machineFlags, 'hpet=off';
3832     }
3833
3834     push @$rtcFlags, 'driftfix=slew' if $tdf;
3835
3836     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3837         push @$rtcFlags, "base=$conf->{startdate}";
3838     } elsif ($useLocaltime) {
3839         push @$rtcFlags, 'base=localtime';
3840     }
3841
3842     if ($forcecpu) {
3843         push @$cmd, '-cpu', $forcecpu;
3844     } else {
3845         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3846     }
3847
3848     PVE::QemuServer::Memory::config(
3849         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3850
3851     push @$cmd, '-S' if $conf->{freeze};
3852
3853     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3854
3855     my $guest_agent = parse_guest_agent($conf);
3856
3857     if ($guest_agent->{enabled}) {
3858         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3859         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3860
3861         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3862             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3863             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3864             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3865         } elsif ($guest_agent->{type} eq 'isa') {
3866             push @$devices, '-device', "isa-serial,chardev=qga0";
3867         }
3868     }
3869
3870     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3871     if ($rng && $version_guard->(4, 1, 2)) {
3872         check_rng_source($rng->{source});
3873
3874         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3875         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3876         my $limiter_str = "";
3877         if ($max_bytes) {
3878             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3879         }
3880
3881         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3882         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3883         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3884     }
3885
3886     my $spice_port;
3887
3888     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3889         if ($qxlnum > 1) {
3890             if ($winversion){
3891                 for (my $i = 1; $i < $qxlnum; $i++){
3892                     push @$devices, '-device', print_vga_device(
3893                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3894                 }
3895             } else {
3896                 # assume other OS works like Linux
3897                 my ($ram, $vram) = ("134217728", "67108864");
3898                 if ($vga->{memory}) {
3899                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3900                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3901                 }
3902                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3903                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3904             }
3905         }
3906
3907         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3908
3909         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3910         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3911         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3912
3913         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3914         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3915         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3916
3917         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3918         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3919
3920         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3921         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3922         if ($spice_enhancement->{foldersharing}) {
3923             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3924             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3925         }
3926
3927         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3928         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3929             if $spice_enhancement->{videostreaming};
3930
3931         push @$devices, '-spice', "$spice_opts";
3932     }
3933
3934     # enable balloon by default, unless explicitly disabled
3935     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3936         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3937         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3938         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3939         push @$devices, '-device', $ballooncmd;
3940     }
3941
3942     if ($conf->{watchdog}) {
3943         my $wdopts = parse_watchdog($conf->{watchdog});
3944         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3945         my $watchdog = $wdopts->{model} || 'i6300esb';
3946         push @$devices, '-device', "$watchdog$pciaddr";
3947         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3948     }
3949
3950     my $vollist = [];
3951     my $scsicontroller = {};
3952     my $ahcicontroller = {};
3953     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3954
3955     # Add iscsi initiator name if available
3956     if (my $initiator = get_initiator_name()) {
3957         push @$devices, '-iscsi', "initiator-name=$initiator";
3958     }
3959
3960     PVE::QemuConfig->foreach_volume($conf, sub {
3961         my ($ds, $drive) = @_;
3962
3963         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3964             check_volume_storage_type($storecfg, $drive->{file});
3965             push @$vollist, $drive->{file};
3966         }
3967
3968         # ignore efidisk here, already added in bios/fw handling code above
3969         return if $drive->{interface} eq 'efidisk';
3970         # similar for TPM
3971         return if $drive->{interface} eq 'tpmstate';
3972
3973         $use_virtio = 1 if $ds =~ m/^virtio/;
3974
3975         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3976
3977         if ($drive->{interface} eq 'virtio'){
3978            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3979         }
3980
3981         if ($drive->{interface} eq 'scsi') {
3982
3983             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3984
3985             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3986                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3987
3988             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3989             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3990
3991             my $iothread = '';
3992             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3993                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3994                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3995             } elsif ($drive->{iothread}) {
3996                 log_warn(
3997                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3998                 );
3999             }
4000
4001             my $queues = '';
4002             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4003                 $queues = ",num_queues=$drive->{queues}";
4004             }
4005
4006             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4007                 if !$scsicontroller->{$controller};
4008             $scsicontroller->{$controller}=1;
4009         }
4010
4011         if ($drive->{interface} eq 'sata') {
4012             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4013             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4014             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4015                 if !$ahcicontroller->{$controller};
4016             $ahcicontroller->{$controller}=1;
4017         }
4018
4019         my $pbs_conf = $pbs_backing->{$ds};
4020         my $pbs_name = undef;
4021         if ($pbs_conf) {
4022             $pbs_name = "drive-$ds-pbs";
4023             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4024         }
4025
4026         my $drive_cmd = print_drive_commandline_full(
4027             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4028
4029         # extra protection for templates, but SATA and IDE don't support it..
4030         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4031
4032         push @$devices, '-drive',$drive_cmd;
4033         push @$devices, '-device', print_drivedevice_full(
4034             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4035     });
4036
4037     for (my $i = 0; $i < $MAX_NETS; $i++) {
4038         my $netname = "net$i";
4039
4040         next if !$conf->{$netname};
4041         my $d = parse_net($conf->{$netname});
4042         next if !$d;
4043         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4044
4045         $use_virtio = 1 if $d->{model} eq 'virtio';
4046
4047         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4048
4049         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4050         push @$devices, '-netdev', $netdevfull;
4051
4052         my $netdevicefull = print_netdevice_full(
4053             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4054
4055         push @$devices, '-device', $netdevicefull;
4056     }
4057
4058     if ($conf->{ivshmem}) {
4059         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4060
4061         my $bus;
4062         if ($q35) {
4063             $bus = print_pcie_addr("ivshmem");
4064         } else {
4065             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4066         }
4067
4068         my $ivshmem_name = $ivshmem->{name} // $vmid;
4069         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4070
4071         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4072         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4073             .",size=$ivshmem->{size}M";
4074     }
4075
4076     # pci.4 is nested in pci.1
4077     $bridges->{1} = 1 if $bridges->{4};
4078
4079     if (!$q35) { # add pci bridges
4080         if (min_version($machine_version, 2, 3)) {
4081            $bridges->{1} = 1;
4082            $bridges->{2} = 1;
4083         }
4084         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4085     }
4086
4087     for my $k (sort {$b cmp $a} keys %$bridges) {
4088         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4089
4090         my $k_name = $k;
4091         if ($k == 2 && $legacy_igd) {
4092             $k_name = "$k-igd";
4093         }
4094         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4095         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4096
4097         if ($q35) { # add after -readconfig pve-q35.cfg
4098             splice @$devices, 2, 0, '-device', $devstr;
4099         } else {
4100             unshift @$devices, '-device', $devstr if $k > 0;
4101         }
4102     }
4103
4104     if (!$kvm) {
4105         push @$machineFlags, 'accel=tcg';
4106     }
4107
4108     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4109
4110     my $machine_type_min = $machine_type;
4111     if ($add_pve_version) {
4112         $machine_type_min =~ s/\+pve\d+$//;
4113         $machine_type_min .= "+pve$required_pve_version";
4114     }
4115     push @$machineFlags, "type=${machine_type_min}";
4116
4117     push @$cmd, @$devices;
4118     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4119     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4120     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4121
4122     if (my $vmstate = $conf->{vmstate}) {
4123         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4124         push @$vollist, $vmstate;
4125         push @$cmd, '-loadstate', $statepath;
4126         print "activating and using '$vmstate' as vmstate\n";
4127     }
4128
4129     if (PVE::QemuConfig->is_template($conf)) {
4130         # needed to workaround base volumes being read-only
4131         push @$cmd, '-snapshot';
4132     }
4133
4134     # add custom args
4135     if ($conf->{args}) {
4136         my $aa = PVE::Tools::split_args($conf->{args});
4137         push @$cmd, @$aa;
4138     }
4139
4140     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4141 }
4142
4143 sub check_rng_source {
4144     my ($source) = @_;
4145
4146     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4147     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4148         if ! -e $source;
4149
4150     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4151     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4152         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4153         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4154         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4155             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4156             ." to the host.\n";
4157     }
4158 }
4159
4160 sub spice_port {
4161     my ($vmid) = @_;
4162
4163     my $res = mon_cmd($vmid, 'query-spice');
4164
4165     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4166 }
4167
4168 sub vm_devices_list {
4169     my ($vmid) = @_;
4170
4171     my $res = mon_cmd($vmid, 'query-pci');
4172     my $devices_to_check = [];
4173     my $devices = {};
4174     foreach my $pcibus (@$res) {
4175         push @$devices_to_check, @{$pcibus->{devices}},
4176     }
4177
4178     while (@$devices_to_check) {
4179         my $to_check = [];
4180         for my $d (@$devices_to_check) {
4181             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4182             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4183
4184             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4185             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4186         }
4187         $devices_to_check = $to_check;
4188     }
4189
4190     my $resblock = mon_cmd($vmid, 'query-block');
4191     foreach my $block (@$resblock) {
4192         if($block->{device} =~ m/^drive-(\S+)/){
4193                 $devices->{$1} = 1;
4194         }
4195     }
4196
4197     my $resmice = mon_cmd($vmid, 'query-mice');
4198     foreach my $mice (@$resmice) {
4199         if ($mice->{name} eq 'QEMU HID Tablet') {
4200             $devices->{tablet} = 1;
4201             last;
4202         }
4203     }
4204
4205     # for usb devices there is no query-usb
4206     # but we can iterate over the entries in
4207     # qom-list path=/machine/peripheral
4208     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4209     foreach my $per (@$resperipheral) {
4210         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4211             $devices->{$per->{name}} = 1;
4212         }
4213     }
4214
4215     return $devices;
4216 }
4217
4218 sub vm_deviceplug {
4219     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4220
4221     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4222
4223     my $devices_list = vm_devices_list($vmid);
4224     return 1 if defined($devices_list->{$deviceid});
4225
4226     # add PCI bridge if we need it for the device
4227     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4228
4229     if ($deviceid eq 'tablet') {
4230         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4231     } elsif ($deviceid eq 'keyboard') {
4232         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4233     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4234         my $id = $1;
4235         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4236         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4237     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4238         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4239     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4240         qemu_iothread_add($vmid, $deviceid, $device);
4241
4242         qemu_driveadd($storecfg, $vmid, $device);
4243         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4244
4245         qemu_deviceadd($vmid, $devicefull);
4246         eval { qemu_deviceaddverify($vmid, $deviceid); };
4247         if (my $err = $@) {
4248             eval { qemu_drivedel($vmid, $deviceid); };
4249             warn $@ if $@;
4250             die $err;
4251         }
4252     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4253         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4254         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4255         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4256
4257         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4258
4259         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4260             qemu_iothread_add($vmid, $deviceid, $device);
4261             $devicefull .= ",iothread=iothread-$deviceid";
4262         }
4263
4264         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4265             $devicefull .= ",num_queues=$device->{queues}";
4266         }
4267
4268         qemu_deviceadd($vmid, $devicefull);
4269         qemu_deviceaddverify($vmid, $deviceid);
4270     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4271         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4272         qemu_driveadd($storecfg, $vmid, $device);
4273
4274         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4275         eval { qemu_deviceadd($vmid, $devicefull); };
4276         if (my $err = $@) {
4277             eval { qemu_drivedel($vmid, $deviceid); };
4278             warn $@ if $@;
4279             die $err;
4280         }
4281     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4282         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4283
4284         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4285         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4286         my $use_old_bios_files = undef;
4287         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4288
4289         my $netdevicefull = print_netdevice_full(
4290             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4291         qemu_deviceadd($vmid, $netdevicefull);
4292         eval {
4293             qemu_deviceaddverify($vmid, $deviceid);
4294             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4295         };
4296         if (my $err = $@) {
4297             eval { qemu_netdevdel($vmid, $deviceid); };
4298             warn $@ if $@;
4299             die $err;
4300         }
4301     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4302         my $bridgeid = $2;
4303         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4304         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4305
4306         qemu_deviceadd($vmid, $devicefull);
4307         qemu_deviceaddverify($vmid, $deviceid);
4308     } else {
4309         die "can't hotplug device '$deviceid'\n";
4310     }
4311
4312     return 1;
4313 }
4314
4315 # fixme: this should raise exceptions on error!
4316 sub vm_deviceunplug {
4317     my ($vmid, $conf, $deviceid) = @_;
4318
4319     my $devices_list = vm_devices_list($vmid);
4320     return 1 if !defined($devices_list->{$deviceid});
4321
4322     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4323     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4324
4325     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4326         qemu_devicedel($vmid, $deviceid);
4327     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_devicedelverify($vmid, $deviceid);
4330     } elsif ($deviceid =~ m/^usb\d+$/) {
4331         qemu_devicedel($vmid, $deviceid);
4332         qemu_devicedelverify($vmid, $deviceid);
4333     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4334         my $device = parse_drive($deviceid, $conf->{$deviceid});
4335
4336         qemu_devicedel($vmid, $deviceid);
4337         qemu_devicedelverify($vmid, $deviceid);
4338         qemu_drivedel($vmid, $deviceid);
4339         qemu_iothread_del($vmid, $deviceid, $device);
4340     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4341         qemu_devicedel($vmid, $deviceid);
4342         qemu_devicedelverify($vmid, $deviceid);
4343     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4344         my $device = parse_drive($deviceid, $conf->{$deviceid});
4345
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348         qemu_drivedel($vmid, $deviceid);
4349         qemu_deletescsihw($conf, $vmid, $deviceid);
4350
4351         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4352             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4353     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356         qemu_netdevdel($vmid, $deviceid);
4357     } else {
4358         die "can't unplug device '$deviceid'\n";
4359     }
4360
4361     return 1;
4362 }
4363
4364 sub qemu_spice_usbredir_chardev_add {
4365     my ($vmid, $id) = @_;
4366
4367     mon_cmd($vmid, "chardev-add" , (
4368         id => $id,
4369         backend => {
4370             type => 'spicevmc',
4371             data => {
4372                 type => "usbredir",
4373             },
4374         },
4375     ));
4376 }
4377
4378 sub qemu_iothread_add {
4379     my ($vmid, $deviceid, $device) = @_;
4380
4381     if ($device->{iothread}) {
4382         my $iothreads = vm_iothreads_list($vmid);
4383         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4384     }
4385 }
4386
4387 sub qemu_iothread_del {
4388     my ($vmid, $deviceid, $device) = @_;
4389
4390     if ($device->{iothread}) {
4391         my $iothreads = vm_iothreads_list($vmid);
4392         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4393     }
4394 }
4395
4396 sub qemu_driveadd {
4397     my ($storecfg, $vmid, $device) = @_;
4398
4399     my $kvmver = get_running_qemu_version($vmid);
4400     my $io_uring = min_version($kvmver, 6, 0);
4401     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4402     $drive =~ s/\\/\\\\/g;
4403     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4404
4405     # If the command succeeds qemu prints: "OK"
4406     return 1 if $ret =~ m/OK/s;
4407
4408     die "adding drive failed: $ret\n";
4409 }
4410
4411 sub qemu_drivedel {
4412     my ($vmid, $deviceid) = @_;
4413
4414     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4415     $ret =~ s/^\s+//;
4416
4417     return 1 if $ret eq "";
4418
4419     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4420     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4421
4422     die "deleting drive $deviceid failed : $ret\n";
4423 }
4424
4425 sub qemu_deviceaddverify {
4426     my ($vmid, $deviceid) = @_;
4427
4428     for (my $i = 0; $i <= 5; $i++) {
4429          my $devices_list = vm_devices_list($vmid);
4430          return 1 if defined($devices_list->{$deviceid});
4431          sleep 1;
4432     }
4433
4434     die "error on hotplug device '$deviceid'\n";
4435 }
4436
4437
4438 sub qemu_devicedelverify {
4439     my ($vmid, $deviceid) = @_;
4440
4441     # need to verify that the device is correctly removed as device_del
4442     # is async and empty return is not reliable
4443
4444     for (my $i = 0; $i <= 5; $i++) {
4445          my $devices_list = vm_devices_list($vmid);
4446          return 1 if !defined($devices_list->{$deviceid});
4447          sleep 1;
4448     }
4449
4450     die "error on hot-unplugging device '$deviceid'\n";
4451 }
4452
4453 sub qemu_findorcreatescsihw {
4454     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $scsihwid="$controller_prefix$controller";
4459     my $devices_list = vm_devices_list($vmid);
4460
4461     if (!defined($devices_list->{$scsihwid})) {
4462         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4463     }
4464
4465     return 1;
4466 }
4467
4468 sub qemu_deletescsihw {
4469     my ($conf, $vmid, $opt) = @_;
4470
4471     my $device = parse_drive($opt, $conf->{$opt});
4472
4473     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4474         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4475         return 1;
4476     }
4477
4478     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4479
4480     my $devices_list = vm_devices_list($vmid);
4481     foreach my $opt (keys %{$devices_list}) {
4482         if (is_valid_drivename($opt)) {
4483             my $drive = parse_drive($opt, $conf->{$opt});
4484             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4485                 return 1;
4486             }
4487         }
4488     }
4489
4490     my $scsihwid="scsihw$controller";
4491
4492     vm_deviceunplug($vmid, $conf, $scsihwid);
4493
4494     return 1;
4495 }
4496
4497 sub qemu_add_pci_bridge {
4498     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4499
4500     my $bridges = {};
4501
4502     my $bridgeid;
4503
4504     print_pci_addr($device, $bridges, $arch, $machine_type);
4505
4506     while (my ($k, $v) = each %$bridges) {
4507         $bridgeid = $k;
4508     }
4509     return 1 if !defined($bridgeid) || $bridgeid < 1;
4510
4511     my $bridge = "pci.$bridgeid";
4512     my $devices_list = vm_devices_list($vmid);
4513
4514     if (!defined($devices_list->{$bridge})) {
4515         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4516     }
4517
4518     return 1;
4519 }
4520
4521 sub qemu_set_link_status {
4522     my ($vmid, $device, $up) = @_;
4523
4524     mon_cmd($vmid, "set_link", name => $device,
4525                up => $up ? JSON::true : JSON::false);
4526 }
4527
4528 sub qemu_netdevadd {
4529     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4530
4531     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4532     my %options =  split(/[=,]/, $netdev);
4533
4534     if (defined(my $vhost = $options{vhost})) {
4535         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4536     }
4537
4538     if (defined(my $queues = $options{queues})) {
4539         $options{queues} = $queues + 0;
4540     }
4541
4542     mon_cmd($vmid, "netdev_add",  %options);
4543     return 1;
4544 }
4545
4546 sub qemu_netdevdel {
4547     my ($vmid, $deviceid) = @_;
4548
4549     mon_cmd($vmid, "netdev_del", id => $deviceid);
4550 }
4551
4552 sub qemu_usb_hotplug {
4553     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4554
4555     return if !$device;
4556
4557     # remove the old one first
4558     vm_deviceunplug($vmid, $conf, $deviceid);
4559
4560     # check if xhci controller is necessary and available
4561     my $devicelist = vm_devices_list($vmid);
4562
4563     if (!$devicelist->{xhci}) {
4564         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4565         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4566     }
4567
4568     # add the new one
4569     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4570 }
4571
4572 sub qemu_cpu_hotplug {
4573     my ($vmid, $conf, $vcpus) = @_;
4574
4575     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4576
4577     my $sockets = 1;
4578     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4579     $sockets = $conf->{sockets} if  $conf->{sockets};
4580     my $cores = $conf->{cores} || 1;
4581     my $maxcpus = $sockets * $cores;
4582
4583     $vcpus = $maxcpus if !$vcpus;
4584
4585     die "you can't add more vcpus than maxcpus\n"
4586         if $vcpus > $maxcpus;
4587
4588     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4589
4590     if ($vcpus < $currentvcpus) {
4591
4592         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4593
4594             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4595                 qemu_devicedel($vmid, "cpu$i");
4596                 my $retry = 0;
4597                 my $currentrunningvcpus = undef;
4598                 while (1) {
4599                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4601                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4602                     $retry++;
4603                     sleep 1;
4604                 }
4605                 #update conf after each succesfull cpu unplug
4606                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607                 PVE::QemuConfig->write_config($vmid, $conf);
4608             }
4609         } else {
4610             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4611         }
4612
4613         return;
4614     }
4615
4616     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4617     die "vcpus in running vm does not match its configuration\n"
4618         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4619
4620     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4621
4622         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4623             my $cpustr = print_cpu_device($conf, $i);
4624             qemu_deviceadd($vmid, $cpustr);
4625
4626             my $retry = 0;
4627             my $currentrunningvcpus = undef;
4628             while (1) {
4629                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4630                 last if scalar(@{$currentrunningvcpus}) == $i;
4631                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4632                 sleep 1;
4633                 $retry++;
4634             }
4635             #update conf after each succesfull cpu hotplug
4636             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4637             PVE::QemuConfig->write_config($vmid, $conf);
4638         }
4639     } else {
4640
4641         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4642             mon_cmd($vmid, "cpu-add", id => int($i));
4643         }
4644     }
4645 }
4646
4647 sub qemu_block_set_io_throttle {
4648     my ($vmid, $deviceid,
4649         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4650         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4651         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4652         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4653
4654     return if !check_running($vmid) ;
4655
4656     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4657         bps => int($bps),
4658         bps_rd => int($bps_rd),
4659         bps_wr => int($bps_wr),
4660         iops => int($iops),
4661         iops_rd => int($iops_rd),
4662         iops_wr => int($iops_wr),
4663         bps_max => int($bps_max),
4664         bps_rd_max => int($bps_rd_max),
4665         bps_wr_max => int($bps_wr_max),
4666         iops_max => int($iops_max),
4667         iops_rd_max => int($iops_rd_max),
4668         iops_wr_max => int($iops_wr_max),
4669         bps_max_length => int($bps_max_length),
4670         bps_rd_max_length => int($bps_rd_max_length),
4671         bps_wr_max_length => int($bps_wr_max_length),
4672         iops_max_length => int($iops_max_length),
4673         iops_rd_max_length => int($iops_rd_max_length),
4674         iops_wr_max_length => int($iops_wr_max_length),
4675     );
4676
4677 }
4678
4679 sub qemu_block_resize {
4680     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4681
4682     my $running = check_running($vmid);
4683
4684     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4685
4686     return if !$running;
4687
4688     my $padding = (1024 - $size % 1024) % 1024;
4689     $size = $size + $padding;
4690
4691     mon_cmd(
4692         $vmid,
4693         "block_resize",
4694         device => $deviceid,
4695         size => int($size),
4696         timeout => 60,
4697     );
4698 }
4699
4700 sub qemu_volume_snapshot {
4701     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4702
4703     my $running = check_running($vmid);
4704
4705     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4706         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4707     } else {
4708         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4709     }
4710 }
4711
4712 sub qemu_volume_snapshot_delete {
4713     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4714
4715     my $running = check_running($vmid);
4716
4717     if($running) {
4718
4719         $running = undef;
4720         my $conf = PVE::QemuConfig->load_config($vmid);
4721         PVE::QemuConfig->foreach_volume($conf, sub {
4722             my ($ds, $drive) = @_;
4723             $running = 1 if $drive->{file} eq $volid;
4724         });
4725     }
4726
4727     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4728         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4729     } else {
4730         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4731     }
4732 }
4733
4734 sub set_migration_caps {
4735     my ($vmid, $savevm) = @_;
4736
4737     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4738
4739     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4740     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4741
4742     my $cap_ref = [];
4743
4744     my $enabled_cap = {
4745         "auto-converge" => 1,
4746         "xbzrle" => 1,
4747         "x-rdma-pin-all" => 0,
4748         "zero-blocks" => 0,
4749         "compress" => 0,
4750         "dirty-bitmaps" => $dirty_bitmaps,
4751     };
4752
4753     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4754
4755     for my $supported_capability (@$supported_capabilities) {
4756         push @$cap_ref, {
4757             capability => $supported_capability->{capability},
4758             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4759         };
4760     }
4761
4762     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4763 }
4764
4765 sub foreach_volid {
4766     my ($conf, $func, @param) = @_;
4767
4768     my $volhash = {};
4769
4770     my $test_volid = sub {
4771         my ($key, $drive, $snapname, $pending) = @_;
4772
4773         my $volid = $drive->{file};
4774         return if !$volid;
4775
4776         $volhash->{$volid}->{cdrom} //= 1;
4777         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4778
4779         my $replicate = $drive->{replicate} // 1;
4780         $volhash->{$volid}->{replicate} //= 0;
4781         $volhash->{$volid}->{replicate} = 1 if $replicate;
4782
4783         $volhash->{$volid}->{shared} //= 0;
4784         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4785
4786         $volhash->{$volid}->{is_unused} //= 0;
4787         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4788
4789         $volhash->{$volid}->{is_attached} //= 0;
4790         $volhash->{$volid}->{is_attached} = 1
4791             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4792
4793         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4794             if defined($snapname);
4795
4796         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4797
4798         my $size = $drive->{size};
4799         $volhash->{$volid}->{size} //= $size if $size;
4800
4801         $volhash->{$volid}->{is_vmstate} //= 0;
4802         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4803
4804         $volhash->{$volid}->{is_tpmstate} //= 0;
4805         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4806
4807         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4808     };
4809
4810     my $include_opts = {
4811         extra_keys => ['vmstate'],
4812         include_unused => 1,
4813     };
4814
4815     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4816
4817     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4818         if defined($conf->{pending}) && $conf->{pending}->%*;
4819
4820     foreach my $snapname (keys %{$conf->{snapshots}}) {
4821         my $snap = $conf->{snapshots}->{$snapname};
4822         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4823     }
4824
4825     foreach my $volid (keys %$volhash) {
4826         &$func($volid, $volhash->{$volid}, @param);
4827     }
4828 }
4829
4830 my $fast_plug_option = {
4831     'description' => 1,
4832     'hookscript' => 1,
4833     'lock' => 1,
4834     'migrate_downtime' => 1,
4835     'migrate_speed' => 1,
4836     'name' => 1,
4837     'onboot' => 1,
4838     'protection' => 1,
4839     'shares' => 1,
4840     'startup' => 1,
4841     'tags' => 1,
4842     'vmstatestorage' => 1,
4843 };
4844
4845 for my $opt (keys %$confdesc_cloudinit) {
4846     $fast_plug_option->{$opt} = 1;
4847 };
4848
4849 # hotplug changes in [PENDING]
4850 # $selection hash can be used to only apply specified options, for
4851 # example: { cores => 1 } (only apply changed 'cores')
4852 # $errors ref is used to return error messages
4853 sub vmconfig_hotplug_pending {
4854     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4855
4856     my $defaults = load_defaults();
4857     my $arch = get_vm_arch($conf);
4858     my $machine_type = get_vm_machine($conf, undef, $arch);
4859
4860     # commit values which do not have any impact on running VM first
4861     # Note: those option cannot raise errors, we we do not care about
4862     # $selection and always apply them.
4863
4864     my $add_error = sub {
4865         my ($opt, $msg) = @_;
4866         $errors->{$opt} = "hotplug problem - $msg";
4867     };
4868
4869     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4870
4871     my $cloudinit_record_changed = sub {
4872         my ($conf, $opt, $old, $new) = @_;
4873         return if !$cloudinit_pending_properties->{$opt};
4874
4875         my $ci = ($conf->{cloudinit} //= {});
4876
4877         my $recorded = $ci->{$opt};
4878         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4879
4880         if (defined($new)) {
4881             if (defined($old)) {
4882                 # an existing value is being modified
4883                 if (defined($recorded)) {
4884                     # the value was already not in sync
4885                     if ($new eq $recorded) {
4886                         # a value is being reverted to the cloud-init state:
4887                         delete $ci->{$opt};
4888                         delete $added{$opt};
4889                     } else {
4890                         # the value was changed multiple times, do nothing
4891                     }
4892                 } elsif ($added{$opt}) {
4893                     # the value had been marked as added and is being changed, do nothing
4894                 } else {
4895                     # the value is new, record it:
4896                     $ci->{$opt} = $old;
4897                 }
4898             } else {
4899                 # a new value is being added
4900                 if (defined($recorded)) {
4901                     # it was already not in sync
4902                     if ($new eq $recorded) {
4903                         # a value is being reverted to the cloud-init state:
4904                         delete $ci->{$opt};
4905                         delete $added{$opt};
4906                     } else {
4907                         # the value had temporarily been removed, do nothing
4908                     }
4909                 } elsif ($added{$opt}) {
4910                     # the value had been marked as added already, do nothing
4911                 } else {
4912                     # the value is new, add it
4913                     $added{$opt} = 1;
4914                 }
4915             }
4916         } elsif (!defined($old)) {
4917             # a non-existent value is being removed? ignore...
4918         } else {
4919             # a value is being deleted
4920             if (defined($recorded)) {
4921                 # a value was already recorded, just keep it
4922             } elsif ($added{$opt}) {
4923                 # the value was marked as added, remove it
4924                 delete $added{$opt};
4925             } else {
4926                 # a previously unrecorded value is being removed, record the old value:
4927                 $ci->{$opt} = $old;
4928             }
4929         }
4930
4931         my $added = join(',', sort keys %added);
4932         $ci->{added} = $added if length($added);
4933     };
4934
4935     my $changes = 0;
4936     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4937         if ($fast_plug_option->{$opt}) {
4938             my $new = delete $conf->{pending}->{$opt};
4939             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4940             $conf->{$opt} = $new;
4941             $changes = 1;
4942         }
4943     }
4944
4945     if ($changes) {
4946         PVE::QemuConfig->write_config($vmid, $conf);
4947     }
4948
4949     my $ostype = $conf->{ostype};
4950     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4951     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4952     my $usb_hotplug = $hotplug_features->{usb}
4953         && min_version($version, 7, 1)
4954         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4955
4956     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4957     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4958
4959     foreach my $opt (sort keys %$pending_delete_hash) {
4960         next if $selection && !$selection->{$opt};
4961         my $force = $pending_delete_hash->{$opt}->{force};
4962         eval {
4963             if ($opt eq 'hotplug') {
4964                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4965             } elsif ($opt eq 'tablet') {
4966                 die "skip\n" if !$hotplug_features->{usb};
4967                 if ($defaults->{tablet}) {
4968                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4969                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4970                         if $arch eq 'aarch64';
4971                 } else {
4972                     vm_deviceunplug($vmid, $conf, 'tablet');
4973                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4974                 }
4975             } elsif ($opt =~ m/^usb(\d+)$/) {
4976                 my $index = $1;
4977                 die "skip\n" if !$usb_hotplug;
4978                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4979                 vm_deviceunplug($vmid, $conf, $opt);
4980             } elsif ($opt eq 'vcpus') {
4981                 die "skip\n" if !$hotplug_features->{cpu};
4982                 qemu_cpu_hotplug($vmid, $conf, undef);
4983             } elsif ($opt eq 'balloon') {
4984                 # enable balloon device is not hotpluggable
4985                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4986                 # here we reset the ballooning value to memory
4987                 my $balloon = get_current_memory($conf->{memory});
4988                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4989             } elsif ($fast_plug_option->{$opt}) {
4990                 # do nothing
4991             } elsif ($opt =~ m/^net(\d+)$/) {
4992                 die "skip\n" if !$hotplug_features->{network};
4993                 vm_deviceunplug($vmid, $conf, $opt);
4994             } elsif (is_valid_drivename($opt)) {
4995                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4996                 vm_deviceunplug($vmid, $conf, $opt);
4997                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4998             } elsif ($opt =~ m/^memory$/) {
4999                 die "skip\n" if !$hotplug_features->{memory};
5000                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
5001             } elsif ($opt eq 'cpuunits') {
5002                 $cgroup->change_cpu_shares(undef);
5003             } elsif ($opt eq 'cpulimit') {
5004                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5005             } else {
5006                 die "skip\n";
5007             }
5008         };
5009         if (my $err = $@) {
5010             &$add_error($opt, $err) if $err ne "skip\n";
5011         } else {
5012             my $old = delete $conf->{$opt};
5013             $cloudinit_record_changed->($conf, $opt, $old, undef);
5014             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5015         }
5016     }
5017
5018     my $cloudinit_opt;
5019     foreach my $opt (keys %{$conf->{pending}}) {
5020         next if $selection && !$selection->{$opt};
5021         my $value = $conf->{pending}->{$opt};
5022         eval {
5023             if ($opt eq 'hotplug') {
5024                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5025             } elsif ($opt eq 'tablet') {
5026                 die "skip\n" if !$hotplug_features->{usb};
5027                 if ($value == 1) {
5028                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5029                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5030                         if $arch eq 'aarch64';
5031                 } elsif ($value == 0) {
5032                     vm_deviceunplug($vmid, $conf, 'tablet');
5033                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5034                 }
5035             } elsif ($opt =~ m/^usb(\d+)$/) {
5036                 my $index = $1;
5037                 die "skip\n" if !$usb_hotplug;
5038                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5039                 my $id = $opt;
5040                 if ($d->{host} =~ m/^spice$/i)  {
5041                     $id = "usbredirdev$index";
5042                 }
5043                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5044             } elsif ($opt eq 'vcpus') {
5045                 die "skip\n" if !$hotplug_features->{cpu};
5046                 qemu_cpu_hotplug($vmid, $conf, $value);
5047             } elsif ($opt eq 'balloon') {
5048                 # enable/disable balloning device is not hotpluggable
5049                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5050                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5051                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5052
5053                 # allow manual ballooning if shares is set to zero
5054                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5055                     my $memory = get_current_memory($conf->{memory});
5056                     my $balloon = $conf->{pending}->{balloon} || $memory;
5057                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5058                 }
5059             } elsif ($opt =~ m/^net(\d+)$/) {
5060                 # some changes can be done without hotplug
5061                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5062                                     $vmid, $opt, $value, $arch, $machine_type);
5063             } elsif (is_valid_drivename($opt)) {
5064                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5065                 # some changes can be done without hotplug
5066                 my $drive = parse_drive($opt, $value);
5067                 if (drive_is_cloudinit($drive)) {
5068                     $cloudinit_opt = [$opt, $drive];
5069                     # apply all the other changes first, then generate the cloudinit disk
5070                     die "skip\n";
5071                 }
5072                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5073                                      $vmid, $opt, $value, $arch, $machine_type);
5074             } elsif ($opt =~ m/^memory$/) { #dimms
5075                 die "skip\n" if !$hotplug_features->{memory};
5076                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5077             } elsif ($opt eq 'cpuunits') {
5078                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5079                 $cgroup->change_cpu_shares($new_cpuunits);
5080             } elsif ($opt eq 'cpulimit') {
5081                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5082                 $cgroup->change_cpu_quota($cpulimit, 100000);
5083             } elsif ($opt eq 'agent') {
5084                 vmconfig_update_agent($conf, $opt, $value);
5085             } else {
5086                 die "skip\n";  # skip non-hot-pluggable options
5087             }
5088         };
5089         if (my $err = $@) {
5090             &$add_error($opt, $err) if $err ne "skip\n";
5091         } else {
5092             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5093             $conf->{$opt} = $value;
5094             delete $conf->{pending}->{$opt};
5095         }
5096     }
5097
5098     if (defined($cloudinit_opt)) {
5099         my ($opt, $drive) = @$cloudinit_opt;
5100         my $value = $conf->{pending}->{$opt};
5101         eval {
5102             my $temp = {%$conf, $opt => $value};
5103             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5104             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5105                                  $vmid, $opt, $value, $arch, $machine_type);
5106         };
5107         if (my $err = $@) {
5108             &$add_error($opt, $err) if $err ne "skip\n";
5109         } else {
5110             $conf->{$opt} = $value;
5111             delete $conf->{pending}->{$opt};
5112         }
5113     }
5114
5115     # unplug xhci controller if no usb device is left
5116     if ($usb_hotplug) {
5117         my $has_usb = 0;
5118         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5119             next if !defined($conf->{"usb$i"});
5120             $has_usb = 1;
5121             last;
5122         }
5123         if (!$has_usb) {
5124             vm_deviceunplug($vmid, $conf, 'xhci');
5125         }
5126     }
5127
5128     PVE::QemuConfig->write_config($vmid, $conf);
5129
5130     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5131         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5132     }
5133 }
5134
5135 sub try_deallocate_drive {
5136     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5137
5138     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5139         my $volid = $drive->{file};
5140         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5141             my $sid = PVE::Storage::parse_volume_id($volid);
5142             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5143
5144             # check if the disk is really unused
5145             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5146                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5147             PVE::Storage::vdisk_free($storecfg, $volid);
5148             return 1;
5149         } else {
5150             # If vm is not owner of this disk remove from config
5151             return 1;
5152         }
5153     }
5154
5155     return;
5156 }
5157
5158 sub vmconfig_delete_or_detach_drive {
5159     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5160
5161     my $drive = parse_drive($opt, $conf->{$opt});
5162
5163     my $rpcenv = PVE::RPCEnvironment::get();
5164     my $authuser = $rpcenv->get_user();
5165
5166     if ($force) {
5167         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5168         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5169     } else {
5170         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5171     }
5172 }
5173
5174
5175
5176 sub vmconfig_apply_pending {
5177     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5178
5179     return if !scalar(keys %{$conf->{pending}});
5180
5181     my $add_apply_error = sub {
5182         my ($opt, $msg) = @_;
5183         my $err_msg = "unable to apply pending change $opt : $msg";
5184         $errors->{$opt} = $err_msg;
5185         warn $err_msg;
5186     };
5187
5188     # cold plug
5189
5190     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5191     foreach my $opt (sort keys %$pending_delete_hash) {
5192         my $force = $pending_delete_hash->{$opt}->{force};
5193         eval {
5194             if ($opt =~ m/^unused/) {
5195                 die "internal error";
5196             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5197                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5198             }
5199         };
5200         if (my $err = $@) {
5201             $add_apply_error->($opt, $err);
5202         } else {
5203             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5204             delete $conf->{$opt};
5205         }
5206     }
5207
5208     PVE::QemuConfig->cleanup_pending($conf);
5209
5210     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5211
5212     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5213         next if $opt eq 'delete'; # just to be sure
5214         eval {
5215             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5216                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5217             }
5218         };
5219         if (my $err = $@) {
5220             $add_apply_error->($opt, $err);
5221         } else {
5222
5223             if (is_valid_drivename($opt)) {
5224                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5225                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5226             }
5227
5228             $conf->{$opt} = delete $conf->{pending}->{$opt};
5229         }
5230     }
5231
5232     # write all changes at once to avoid unnecessary i/o
5233     PVE::QemuConfig->write_config($vmid, $conf);
5234     if ($generate_cloudinit) {
5235         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5236             # After successful generation and if there were changes to be applied, update the
5237             # config to drop the {cloudinit} entry.
5238             PVE::QemuConfig->write_config($vmid, $conf);
5239         }
5240     }
5241 }
5242
5243 sub vmconfig_update_net {
5244     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5245
5246     my $newnet = parse_net($value);
5247
5248     if ($conf->{$opt}) {
5249         my $oldnet = parse_net($conf->{$opt});
5250
5251         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5252             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5253             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5254             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5255             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5256
5257             # for non online change, we try to hot-unplug
5258             die "skip\n" if !$hotplug;
5259             vm_deviceunplug($vmid, $conf, $opt);
5260         } else {
5261
5262             die "internal error" if $opt !~ m/net(\d+)/;
5263             my $iface = "tap${vmid}i$1";
5264
5265             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5266                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5267                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5268                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5269                 PVE::Network::tap_unplug($iface);
5270
5271                 if ($have_sdn) {
5272                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5273                 } else {
5274                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5275                 }
5276             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5277                 # Rate can be applied on its own but any change above needs to
5278                 # include the rate in tap_plug since OVS resets everything.
5279                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5280             }
5281
5282             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5283                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5284             }
5285
5286             return 1;
5287         }
5288     }
5289
5290     if ($hotplug) {
5291         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5292     } else {
5293         die "skip\n";
5294     }
5295 }
5296
5297 sub vmconfig_update_agent {
5298     my ($conf, $opt, $value) = @_;
5299
5300     die "skip\n" if !$conf->{$opt};
5301
5302     my $hotplug_options = { fstrim_cloned_disks => 1 };
5303
5304     my $old_agent = parse_guest_agent($conf);
5305     my $agent = parse_guest_agent({$opt => $value});
5306
5307     for my $option (keys %$agent) { # added/changed options
5308         next if defined($hotplug_options->{$option});
5309         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5310     }
5311
5312     for my $option (keys %$old_agent) { # removed options
5313         next if defined($hotplug_options->{$option});
5314         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5315     }
5316
5317     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5318 }
5319
5320 sub vmconfig_update_disk {
5321     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5322
5323     my $drive = parse_drive($opt, $value);
5324
5325     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5326         my $media = $drive->{media} || 'disk';
5327         my $oldmedia = $old_drive->{media} || 'disk';
5328         die "unable to change media type\n" if $media ne $oldmedia;
5329
5330         if (!drive_is_cdrom($old_drive)) {
5331
5332             if ($drive->{file} ne $old_drive->{file}) {
5333
5334                 die "skip\n" if !$hotplug;
5335
5336                 # unplug and register as unused
5337                 vm_deviceunplug($vmid, $conf, $opt);
5338                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5339
5340             } else {
5341                 # update existing disk
5342
5343                 # skip non hotpluggable value
5344                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5345                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5346                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5347                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5348                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5349                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5350                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5351                     die "skip\n";
5352                 }
5353
5354                 # apply throttle
5355                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5356                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5357                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5358                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5359                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5360                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5361                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5362                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5363                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5364                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5365                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5366                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5367                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5368                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5369                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5370                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5371                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5372                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5373
5374                     qemu_block_set_io_throttle(
5375                         $vmid,"drive-$opt",
5376                         ($drive->{mbps} || 0)*1024*1024,
5377                         ($drive->{mbps_rd} || 0)*1024*1024,
5378                         ($drive->{mbps_wr} || 0)*1024*1024,
5379                          $drive->{iops} || 0,
5380                          $drive->{iops_rd} || 0,
5381                          $drive->{iops_wr} || 0,
5382                         ($drive->{mbps_max} || 0)*1024*1024,
5383                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5384                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5385                          $drive->{iops_max} || 0,
5386                          $drive->{iops_rd_max} || 0,
5387                          $drive->{iops_wr_max} || 0,
5388                          $drive->{bps_max_length} || 1,
5389                          $drive->{bps_rd_max_length} || 1,
5390                          $drive->{bps_wr_max_length} || 1,
5391                          $drive->{iops_max_length} || 1,
5392                          $drive->{iops_rd_max_length} || 1,
5393                          $drive->{iops_wr_max_length} || 1,
5394                     );
5395
5396                 }
5397
5398                 return 1;
5399             }
5400
5401         } else { # cdrom
5402
5403             if ($drive->{file} eq 'none') {
5404                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5405                 if (drive_is_cloudinit($old_drive)) {
5406                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5407                 }
5408             } else {
5409                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5410
5411                 # force eject if locked
5412                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5413
5414                 if ($path) {
5415                     mon_cmd($vmid, "blockdev-change-medium",
5416                         id => "$opt", filename => "$path");
5417                 }
5418             }
5419
5420             return 1;
5421         }
5422     }
5423
5424     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5425     # hotplug new disks
5426     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5427     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5428 }
5429
5430 sub vmconfig_update_cloudinit_drive {
5431     my ($storecfg, $conf, $vmid) = @_;
5432
5433     my $cloudinit_ds = undef;
5434     my $cloudinit_drive = undef;
5435
5436     PVE::QemuConfig->foreach_volume($conf, sub {
5437         my ($ds, $drive) = @_;
5438         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5439             $cloudinit_ds = $ds;
5440             $cloudinit_drive = $drive;
5441         }
5442     });
5443
5444     return if !$cloudinit_drive;
5445
5446     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5447         PVE::QemuConfig->write_config($vmid, $conf);
5448     }
5449
5450     my $running = PVE::QemuServer::check_running($vmid);
5451
5452     if ($running) {
5453         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5454         if ($path) {
5455             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5456             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5457         }
5458     }
5459 }
5460
5461 # called in locked context by incoming migration
5462 sub vm_migrate_get_nbd_disks {
5463     my ($storecfg, $conf, $replicated_volumes) = @_;
5464
5465     my $local_volumes = {};
5466     PVE::QemuConfig->foreach_volume($conf, sub {
5467         my ($ds, $drive) = @_;
5468
5469         return if drive_is_cdrom($drive);
5470         return if $ds eq 'tpmstate0';
5471
5472         my $volid = $drive->{file};
5473
5474         return if !$volid;
5475
5476         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5477
5478         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5479         return if $scfg->{shared};
5480
5481         my $format = qemu_img_format($scfg, $volname);
5482
5483         # replicated disks re-use existing state via bitmap
5484         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5485         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5486     });
5487     return $local_volumes;
5488 }
5489
5490 # called in locked context by incoming migration
5491 sub vm_migrate_alloc_nbd_disks {
5492     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5493
5494     my $nbd = {};
5495     foreach my $opt (sort keys %$source_volumes) {
5496         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5497
5498         if ($use_existing) {
5499             $nbd->{$opt}->{drivestr} = print_drive($drive);
5500             $nbd->{$opt}->{volid} = $volid;
5501             $nbd->{$opt}->{replicated} = 1;
5502             next;
5503         }
5504
5505         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5506
5507         # order of precedence, filtered by whether storage supports it:
5508         # 1. explicit requested format
5509         # 2. default format of storage
5510         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5511         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5512
5513         my $size = $drive->{size} / 1024;
5514         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5515         my $newdrive = $drive;
5516         $newdrive->{format} = $format;
5517         $newdrive->{file} = $newvolid;
5518         my $drivestr = print_drive($newdrive);
5519         $nbd->{$opt}->{drivestr} = $drivestr;
5520         $nbd->{$opt}->{volid} = $newvolid;
5521     }
5522
5523     return $nbd;
5524 }
5525
5526 # see vm_start_nolock for parameters, additionally:
5527 # migrate_opts:
5528 #   storagemap = parsed storage map for allocating NBD disks
5529 sub vm_start {
5530     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5531
5532     return PVE::QemuConfig->lock_config($vmid, sub {
5533         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5534
5535         die "you can't start a vm if it's a template\n"
5536             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5537
5538         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5539         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5540
5541         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5542
5543         if ($has_backup_lock && $running) {
5544             # a backup is currently running, attempt to start the guest in the
5545             # existing QEMU instance
5546             return vm_resume($vmid);
5547         }
5548
5549         PVE::QemuConfig->check_lock($conf)
5550             if !($params->{skiplock} || $has_suspended_lock);
5551
5552         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5553
5554         die "VM $vmid already running\n" if $running;
5555
5556         if (my $storagemap = $migrate_opts->{storagemap}) {
5557             my $replicated = $migrate_opts->{replicated_volumes};
5558             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5559             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5560
5561             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5562                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5563             }
5564         }
5565
5566         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5567     });
5568 }
5569
5570
5571 # params:
5572 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5573 #   skiplock => 0/1, skip checking for config lock
5574 #   skiptemplate => 0/1, skip checking whether VM is template
5575 #   forcemachine => to force QEMU machine (rollback/migration)
5576 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5577 #   timeout => in seconds
5578 #   paused => start VM in paused state (backup)
5579 #   resume => resume from hibernation
5580 #   pbs-backing => {
5581 #      sata0 => {
5582 #         repository
5583 #         snapshot
5584 #         keyfile
5585 #         archive
5586 #      },
5587 #      virtio2 => ...
5588 #   }
5589 # migrate_opts:
5590 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5591 #   migratedfrom => source node
5592 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5593 #   network => CIDR of migration network
5594 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5595 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5596 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5597 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5598 #       contained in config
5599 sub vm_start_nolock {
5600     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5601
5602     my $statefile = $params->{statefile};
5603     my $resume = $params->{resume};
5604
5605     my $migratedfrom = $migrate_opts->{migratedfrom};
5606     my $migration_type = $migrate_opts->{type};
5607
5608     my $res = {};
5609
5610     # clean up leftover reboot request files
5611     eval { clear_reboot_request($vmid); };
5612     warn $@ if $@;
5613
5614     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5615         vmconfig_apply_pending($vmid, $conf, $storecfg);
5616         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5617     }
5618
5619     # don't regenerate the ISO if the VM is started as part of a live migration
5620     # this way we can reuse the old ISO with the correct config
5621     if (!$migratedfrom) {
5622         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5623             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5624             # $conf->{cloudinit}, so we could just not do this?
5625             # But we do it above, so for now let's be consistent.
5626             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5627         }
5628     }
5629
5630     # override offline migrated volumes, conf is out of date still
5631     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5632         for my $key (sort keys $offline_volumes->%*) {
5633             my $parsed = parse_drive($key, $conf->{$key});
5634             $parsed->{file} = $offline_volumes->{$key};
5635             $conf->{$key} = print_drive($parsed);
5636         }
5637     }
5638
5639     my $defaults = load_defaults();
5640
5641     # set environment variable useful inside network script
5642     # for remote migration the config is available on the target node!
5643     if (!$migrate_opts->{remote_node}) {
5644         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5645     }
5646
5647     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5648
5649     my $forcemachine = $params->{forcemachine};
5650     my $forcecpu = $params->{forcecpu};
5651     if ($resume) {
5652         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5653         $forcemachine = $conf->{runningmachine};
5654         $forcecpu = $conf->{runningcpu};
5655         print "Resuming suspended VM\n";
5656     }
5657
5658     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5659         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5660
5661     my $migration_ip;
5662     my $get_migration_ip = sub {
5663         my ($nodename) = @_;
5664
5665         return $migration_ip if defined($migration_ip);
5666
5667         my $cidr = $migrate_opts->{network};
5668
5669         if (!defined($cidr)) {
5670             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5671             $cidr = $dc_conf->{migration}->{network};
5672         }
5673
5674         if (defined($cidr)) {
5675             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5676
5677             die "could not get IP: no address configured on local " .
5678                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5679
5680             die "could not get IP: multiple addresses configured on local " .
5681                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5682
5683             $migration_ip = @$ips[0];
5684         }
5685
5686         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5687             if !defined($migration_ip);
5688
5689         return $migration_ip;
5690     };
5691
5692     if ($statefile) {
5693         if ($statefile eq 'tcp') {
5694             my $migrate = $res->{migrate} = { proto => 'tcp' };
5695             $migrate->{addr} = "localhost";
5696             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5697             my $nodename = nodename();
5698
5699             if (!defined($migration_type)) {
5700                 if (defined($datacenterconf->{migration}->{type})) {
5701                     $migration_type = $datacenterconf->{migration}->{type};
5702                 } else {
5703                     $migration_type = 'secure';
5704                 }
5705             }
5706
5707             if ($migration_type eq 'insecure') {
5708                 $migrate->{addr} = $get_migration_ip->($nodename);
5709                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5710             }
5711
5712             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5713             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5714             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5715             push @$cmd, '-incoming', $migrate->{uri};
5716             push @$cmd, '-S';
5717
5718         } elsif ($statefile eq 'unix') {
5719             # should be default for secure migrations as a ssh TCP forward
5720             # tunnel is not deterministic reliable ready and fails regurarly
5721             # to set up in time, so use UNIX socket forwards
5722             my $migrate = $res->{migrate} = { proto => 'unix' };
5723             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5724             unlink $migrate->{addr};
5725
5726             $migrate->{uri} = "unix:$migrate->{addr}";
5727             push @$cmd, '-incoming', $migrate->{uri};
5728             push @$cmd, '-S';
5729
5730         } elsif (-e $statefile) {
5731             push @$cmd, '-loadstate', $statefile;
5732         } else {
5733             my $statepath = PVE::Storage::path($storecfg, $statefile);
5734             push @$vollist, $statefile;
5735             push @$cmd, '-loadstate', $statepath;
5736         }
5737     } elsif ($params->{paused}) {
5738         push @$cmd, '-S';
5739     }
5740
5741     my $memory = get_current_memory($conf->{memory});
5742     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5743
5744     my $pci_reserve_list = [];
5745     for my $device (values $pci_devices->%*) {
5746         next if $device->{mdev}; # we don't reserve for mdev devices
5747         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5748     }
5749
5750     # reserve all PCI IDs before actually doing anything with them
5751     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5752
5753     eval {
5754         my $uuid;
5755         for my $id (sort keys %$pci_devices) {
5756             my $d = $pci_devices->{$id};
5757             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5758
5759             my $chosen_mdev;
5760             for my $dev ($d->{ids}->@*) {
5761                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5762                 if ($d->{mdev}) {
5763                     warn $@ if $@;
5764                     $chosen_mdev = $info;
5765                     last if $chosen_mdev; # if successful, we're done
5766                 } else {
5767                     die $@ if $@;
5768                 }
5769             }
5770
5771             next if !$d->{mdev};
5772             die "could not create mediated device\n" if !defined($chosen_mdev);
5773
5774             # nvidia grid needs the uuid of the mdev as qemu parameter
5775             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5776                 if (defined($conf->{smbios1})) {
5777                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5778                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5779                 }
5780                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5781             }
5782         }
5783         push @$cmd, '-uuid', $uuid if defined($uuid);
5784     };
5785     if (my $err = $@) {
5786         eval { cleanup_pci_devices($vmid, $conf) };
5787         warn $@ if $@;
5788         die $err;
5789     }
5790
5791     PVE::Storage::activate_volumes($storecfg, $vollist);
5792
5793
5794     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5795     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5796     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5797     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5798     # timeout should be more than enough here...
5799     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5800
5801     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5802
5803     my %run_params = (
5804         timeout => $statefile ? undef : $start_timeout,
5805         umask => 0077,
5806         noerr => 1,
5807     );
5808
5809     # when migrating, prefix QEMU output so other side can pick up any
5810     # errors that might occur and show the user
5811     if ($migratedfrom) {
5812         $run_params{quiet} = 1;
5813         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5814     }
5815
5816     my %systemd_properties = (
5817         Slice => 'qemu.slice',
5818         KillMode => 'process',
5819         SendSIGKILL => 0,
5820         TimeoutStopUSec => ULONG_MAX, # infinity
5821     );
5822
5823     if (PVE::CGroup::cgroup_mode() == 2) {
5824         $systemd_properties{CPUWeight} = $cpuunits;
5825     } else {
5826         $systemd_properties{CPUShares} = $cpuunits;
5827     }
5828
5829     if (my $cpulimit = $conf->{cpulimit}) {
5830         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5831     }
5832     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5833
5834     my $run_qemu = sub {
5835         PVE::Tools::run_fork sub {
5836             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5837
5838             my $tpmpid;
5839             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5840                 # start the TPM emulator so QEMU can connect on start
5841                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5842             }
5843
5844             my $exitcode = run_command($cmd, %run_params);
5845             if ($exitcode) {
5846                 if ($tpmpid) {
5847                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5848                     kill 'TERM', $tpmpid;
5849                 }
5850                 die "QEMU exited with code $exitcode\n";
5851             }
5852         };
5853     };
5854
5855     if ($conf->{hugepages}) {
5856
5857         my $code = sub {
5858             my $hotplug_features =
5859                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5860             my $hugepages_topology =
5861                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5862
5863             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5864
5865             PVE::QemuServer::Memory::hugepages_mount();
5866             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5867
5868             eval { $run_qemu->() };
5869             if (my $err = $@) {
5870                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5871                     if !$conf->{keephugepages};
5872                 die $err;
5873             }
5874
5875             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5876                 if !$conf->{keephugepages};
5877         };
5878         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5879
5880     } else {
5881         eval { $run_qemu->() };
5882     }
5883
5884     if (my $err = $@) {
5885         # deactivate volumes if start fails
5886         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5887         warn $@ if $@;
5888         eval { cleanup_pci_devices($vmid, $conf) };
5889         warn $@ if $@;
5890
5891         die "start failed: $err";
5892     }
5893
5894     # re-reserve all PCI IDs now that we can know the actual VM PID
5895     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5896     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5897     warn $@ if $@;
5898
5899     if (defined($res->{migrate})) {
5900         print "migration listens on $res->{migrate}->{uri}\n";
5901     } elsif ($statefile) {
5902         eval { mon_cmd($vmid, "cont"); };
5903         warn $@ if $@;
5904     }
5905
5906     #start nbd server for storage migration
5907     if (my $nbd = $migrate_opts->{nbd}) {
5908         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5909
5910         my $migrate_storage_uri;
5911         # nbd_protocol_version > 0 for unix socket support
5912         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5913             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5914             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5915             $migrate_storage_uri = "nbd:unix:$socket_path";
5916             $res->{migrate}->{unix_sockets} = [$socket_path];
5917         } else {
5918             my $nodename = nodename();
5919             my $localip = $get_migration_ip->($nodename);
5920             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5921             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5922
5923             mon_cmd($vmid, "nbd-server-start", addr => {
5924                 type => 'inet',
5925                 data => {
5926                     host => "${localip}",
5927                     port => "${storage_migrate_port}",
5928                 },
5929             });
5930             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5931             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5932         }
5933
5934         my $block_info = mon_cmd($vmid, "query-block");
5935         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5936
5937         foreach my $opt (sort keys %$nbd) {
5938             my $drivestr = $nbd->{$opt}->{drivestr};
5939             my $volid = $nbd->{$opt}->{volid};
5940
5941             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5942
5943             mon_cmd(
5944                 $vmid,
5945                 "block-export-add",
5946                 id => "drive-$opt",
5947                 'node-name' => $block_node,
5948                 writable => JSON::true,
5949                 type => "nbd",
5950                 name => "drive-$opt", # NBD export name
5951             );
5952
5953             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5954             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5955             print "re-using replicated volume: $opt - $volid\n"
5956                 if $nbd->{$opt}->{replicated};
5957
5958             $res->{drives}->{$opt} = $nbd->{$opt};
5959             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5960         }
5961     }
5962
5963     if ($migratedfrom) {
5964         eval {
5965             set_migration_caps($vmid);
5966         };
5967         warn $@ if $@;
5968
5969         if ($spice_port) {
5970             print "spice listens on port $spice_port\n";
5971             $res->{spice_port} = $spice_port;
5972             if ($migrate_opts->{spice_ticket}) {
5973                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5974                     $migrate_opts->{spice_ticket});
5975                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5976             }
5977         }
5978
5979     } else {
5980         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5981             if !$statefile && $conf->{balloon};
5982
5983         foreach my $opt (keys %$conf) {
5984             next if $opt !~  m/^net\d+$/;
5985             my $nicconf = parse_net($conf->{$opt});
5986             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5987         }
5988         add_nets_bridge_fdb($conf, $vmid);
5989     }
5990
5991    if (!defined($conf->{balloon}) || $conf->{balloon}) {
5992         eval {
5993             mon_cmd(
5994                 $vmid,
5995                 'qom-set',
5996                 path => "machine/peripheral/balloon0",
5997                 property => "guest-stats-polling-interval",
5998                 value => 2
5999             );
6000         };
6001         log_warn("could not set polling interval for ballooning - $@") if $@;
6002     }
6003
6004     if ($resume) {
6005         print "Resumed VM, removing state\n";
6006         if (my $vmstate = $conf->{vmstate}) {
6007             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6008             PVE::Storage::vdisk_free($storecfg, $vmstate);
6009         }
6010         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6011         PVE::QemuConfig->write_config($vmid, $conf);
6012     }
6013
6014     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6015
6016     return $res;
6017 }
6018
6019 sub vm_commandline {
6020     my ($storecfg, $vmid, $snapname) = @_;
6021
6022     my $conf = PVE::QemuConfig->load_config($vmid);
6023
6024     my ($forcemachine, $forcecpu);
6025     if ($snapname) {
6026         my $snapshot = $conf->{snapshots}->{$snapname};
6027         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6028
6029         # check for machine or CPU overrides in snapshot
6030         $forcemachine = $snapshot->{runningmachine};
6031         $forcecpu = $snapshot->{runningcpu};
6032
6033         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6034
6035         $conf = $snapshot;
6036     }
6037
6038     my $defaults = load_defaults();
6039
6040     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6041
6042     return PVE::Tools::cmd2string($cmd);
6043 }
6044
6045 sub vm_reset {
6046     my ($vmid, $skiplock) = @_;
6047
6048     PVE::QemuConfig->lock_config($vmid, sub {
6049
6050         my $conf = PVE::QemuConfig->load_config($vmid);
6051
6052         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6053
6054         mon_cmd($vmid, "system_reset");
6055     });
6056 }
6057
6058 sub get_vm_volumes {
6059     my ($conf) = @_;
6060
6061     my $vollist = [];
6062     foreach_volid($conf, sub {
6063         my ($volid, $attr) = @_;
6064
6065         return if $volid =~ m|^/|;
6066
6067         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6068         return if !$sid;
6069
6070         push @$vollist, $volid;
6071     });
6072
6073     return $vollist;
6074 }
6075
6076 sub cleanup_pci_devices {
6077     my ($vmid, $conf) = @_;
6078
6079     foreach my $key (keys %$conf) {
6080         next if $key !~ m/^hostpci(\d+)$/;
6081         my $hostpciindex = $1;
6082         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6083         my $d = parse_hostpci($conf->{$key});
6084         if ($d->{mdev}) {
6085             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6086             # don't want to break ABI just for this two liner
6087             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6088
6089             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6090             # out when we do it first. so wait for 10 seconds and then try it
6091             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6092                 sleep 10;
6093             }
6094
6095             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6096         }
6097     }
6098     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6099 }
6100
6101 sub vm_stop_cleanup {
6102     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6103
6104     eval {
6105
6106         if (!$keepActive) {
6107             my $vollist = get_vm_volumes($conf);
6108             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6109
6110             if (my $tpmdrive = $conf->{tpmstate0}) {
6111                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6112                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6113                 if ($storeid) {
6114                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6115                 }
6116             }
6117         }
6118
6119         foreach my $ext (qw(mon qmp pid vnc qga)) {
6120             unlink "/var/run/qemu-server/${vmid}.$ext";
6121         }
6122
6123         if ($conf->{ivshmem}) {
6124             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6125             # just delete it for now, VMs which have this already open do not
6126             # are affected, but new VMs will get a separated one. If this
6127             # becomes an issue we either add some sort of ref-counting or just
6128             # add a "don't delete on stop" flag to the ivshmem format.
6129             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6130         }
6131
6132         cleanup_pci_devices($vmid, $conf);
6133
6134         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6135     };
6136     warn $@ if $@; # avoid errors - just warn
6137 }
6138
6139 # call only in locked context
6140 sub _do_vm_stop {
6141     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6142
6143     my $pid = check_running($vmid, $nocheck);
6144     return if !$pid;
6145
6146     my $conf;
6147     if (!$nocheck) {
6148         $conf = PVE::QemuConfig->load_config($vmid);
6149         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6150         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6151             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6152             $timeout = $opts->{down} if $opts->{down};
6153         }
6154         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6155     }
6156
6157     eval {
6158         if ($shutdown) {
6159             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6160                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6161             } else {
6162                 mon_cmd($vmid, "system_powerdown");
6163             }
6164         } else {
6165             mon_cmd($vmid, "quit");
6166         }
6167     };
6168     my $err = $@;
6169
6170     if (!$err) {
6171         $timeout = 60 if !defined($timeout);
6172
6173         my $count = 0;
6174         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6175             $count++;
6176             sleep 1;
6177         }
6178
6179         if ($count >= $timeout) {
6180             if ($force) {
6181                 warn "VM still running - terminating now with SIGTERM\n";
6182                 kill 15, $pid;
6183             } else {
6184                 die "VM quit/powerdown failed - got timeout\n";
6185             }
6186         } else {
6187             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6188             return;
6189         }
6190     } else {
6191        if (!check_running($vmid, $nocheck)) {
6192             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6193             return;
6194        }
6195        if ($force) {
6196             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6197             kill 15, $pid;
6198         } else {
6199             die "VM quit/powerdown failed\n";
6200         }
6201     }
6202
6203     # wait again
6204     $timeout = 10;
6205
6206     my $count = 0;
6207     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6208         $count++;
6209         sleep 1;
6210     }
6211
6212     if ($count >= $timeout) {
6213         warn "VM still running - terminating now with SIGKILL\n";
6214         kill 9, $pid;
6215         sleep 1;
6216     }
6217
6218     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6219 }
6220
6221 # Note: use $nocheck to skip tests if VM configuration file exists.
6222 # We need that when migration VMs to other nodes (files already moved)
6223 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6224 sub vm_stop {
6225     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6226
6227     $force = 1 if !defined($force) && !$shutdown;
6228
6229     if ($migratedfrom){
6230         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6231         kill 15, $pid if $pid;
6232         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6233         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6234         return;
6235     }
6236
6237     PVE::QemuConfig->lock_config($vmid, sub {
6238         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6239    });
6240 }
6241
6242 sub vm_reboot {
6243     my ($vmid, $timeout) = @_;
6244
6245     PVE::QemuConfig->lock_config($vmid, sub {
6246         eval {
6247
6248             # only reboot if running, as qmeventd starts it again on a stop event
6249             return if !check_running($vmid);
6250
6251             create_reboot_request($vmid);
6252
6253             my $storecfg = PVE::Storage::config();
6254             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6255
6256         };
6257         if (my $err = $@) {
6258             # avoid that the next normal shutdown will be confused for a reboot
6259             clear_reboot_request($vmid);
6260             die $err;
6261         }
6262    });
6263 }
6264
6265 # note: if using the statestorage parameter, the caller has to check privileges
6266 sub vm_suspend {
6267     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6268
6269     my $conf;
6270     my $path;
6271     my $storecfg;
6272     my $vmstate;
6273
6274     PVE::QemuConfig->lock_config($vmid, sub {
6275
6276         $conf = PVE::QemuConfig->load_config($vmid);
6277
6278         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6279         PVE::QemuConfig->check_lock($conf)
6280             if !($skiplock || $is_backing_up);
6281
6282         die "cannot suspend to disk during backup\n"
6283             if $is_backing_up && $includestate;
6284
6285         if ($includestate) {
6286             $conf->{lock} = 'suspending';
6287             my $date = strftime("%Y-%m-%d", localtime(time()));
6288             $storecfg = PVE::Storage::config();
6289             if (!$statestorage) {
6290                 $statestorage = find_vmstate_storage($conf, $storecfg);
6291                 # check permissions for the storage
6292                 my $rpcenv = PVE::RPCEnvironment::get();
6293                 if ($rpcenv->{type} ne 'cli') {
6294                     my $authuser = $rpcenv->get_user();
6295                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6296                 }
6297             }
6298
6299
6300             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6301                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6302             $path = PVE::Storage::path($storecfg, $vmstate);
6303             PVE::QemuConfig->write_config($vmid, $conf);
6304         } else {
6305             mon_cmd($vmid, "stop");
6306         }
6307     });
6308
6309     if ($includestate) {
6310         # save vm state
6311         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6312
6313         eval {
6314             set_migration_caps($vmid, 1);
6315             mon_cmd($vmid, "savevm-start", statefile => $path);
6316             for(;;) {
6317                 my $state = mon_cmd($vmid, "query-savevm");
6318                 if (!$state->{status}) {
6319                     die "savevm not active\n";
6320                 } elsif ($state->{status} eq 'active') {
6321                     sleep(1);
6322                     next;
6323                 } elsif ($state->{status} eq 'completed') {
6324                     print "State saved, quitting\n";
6325                     last;
6326                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6327                     die "query-savevm failed with error '$state->{error}'\n"
6328                 } else {
6329                     die "query-savevm returned status '$state->{status}'\n";
6330                 }
6331             }
6332         };
6333         my $err = $@;
6334
6335         PVE::QemuConfig->lock_config($vmid, sub {
6336             $conf = PVE::QemuConfig->load_config($vmid);
6337             if ($err) {
6338                 # cleanup, but leave suspending lock, to indicate something went wrong
6339                 eval {
6340                     mon_cmd($vmid, "savevm-end");
6341                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6342                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6343                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6344                     PVE::QemuConfig->write_config($vmid, $conf);
6345                 };
6346                 warn $@ if $@;
6347                 die $err;
6348             }
6349
6350             die "lock changed unexpectedly\n"
6351                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6352
6353             mon_cmd($vmid, "quit");
6354             $conf->{lock} = 'suspended';
6355             PVE::QemuConfig->write_config($vmid, $conf);
6356         });
6357     }
6358 }
6359
6360 # $nocheck is set when called as part of a migration - in this context the
6361 # location of the config file (source or target node) is not deterministic,
6362 # since migration cannot wait for pmxcfs to process the rename
6363 sub vm_resume {
6364     my ($vmid, $skiplock, $nocheck) = @_;
6365
6366     PVE::QemuConfig->lock_config($vmid, sub {
6367         my $res = mon_cmd($vmid, 'query-status');
6368         my $resume_cmd = 'cont';
6369         my $reset = 0;
6370         my $conf;
6371         if ($nocheck) {
6372             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6373             if ($@) {
6374                 my $vmlist = PVE::Cluster::get_vmlist();
6375                 if (exists($vmlist->{ids}->{$vmid})) {
6376                     my $node = $vmlist->{ids}->{$vmid}->{node};
6377                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6378                 }
6379                 if (!$conf) {
6380                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6381                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6382                 }
6383             }
6384         } else {
6385             $conf = PVE::QemuConfig->load_config($vmid);
6386         }
6387
6388         if ($res->{status}) {
6389             return if $res->{status} eq 'running'; # job done, go home
6390             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6391             $reset = 1 if $res->{status} eq 'shutdown';
6392         }
6393
6394         if (!$nocheck) {
6395             PVE::QemuConfig->check_lock($conf)
6396                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6397         }
6398
6399         if ($reset) {
6400             # required if a VM shuts down during a backup and we get a resume
6401             # request before the backup finishes for example
6402             mon_cmd($vmid, "system_reset");
6403         }
6404
6405         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6406
6407         mon_cmd($vmid, $resume_cmd);
6408     });
6409 }
6410
6411 sub vm_sendkey {
6412     my ($vmid, $skiplock, $key) = @_;
6413
6414     PVE::QemuConfig->lock_config($vmid, sub {
6415
6416         my $conf = PVE::QemuConfig->load_config($vmid);
6417
6418         # there is no qmp command, so we use the human monitor command
6419         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6420         die $res if $res ne '';
6421     });
6422 }
6423
6424 sub check_bridge_access {
6425     my ($rpcenv, $authuser, $conf) = @_;
6426
6427     return 1 if $authuser eq 'root@pam';
6428
6429     for my $opt (sort keys $conf->%*) {
6430         next if $opt !~ m/^net\d+$/;
6431         my $net = parse_net($conf->{$opt});
6432         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6433         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6434     }
6435     return 1;
6436 };
6437
6438 sub check_mapping_access {
6439     my ($rpcenv, $user, $conf) = @_;
6440
6441     for my $opt (keys $conf->%*) {
6442         if ($opt =~ m/^usb\d+$/) {
6443             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6444             if (my $host = $device->{host}) {
6445                 die "only root can set '$opt' config for real devices\n"
6446                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6447             } elsif ($device->{mapping}) {
6448                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6449             } else {
6450                 die "either 'host' or 'mapping' must be set.\n";
6451             }
6452         } elsif ($opt =~ m/^hostpci\d+$/) {
6453             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6454             if ($device->{host}) {
6455                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6456             } elsif ($device->{mapping}) {
6457                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6458             } else {
6459                 die "either 'host' or 'mapping' must be set.\n";
6460             }
6461        }
6462    }
6463 };
6464
6465 sub check_restore_permissions {
6466     my ($rpcenv, $user, $conf) = @_;
6467
6468     check_bridge_access($rpcenv, $user, $conf);
6469     check_mapping_access($rpcenv, $user, $conf);
6470 }
6471 # vzdump restore implementaion
6472
6473 sub tar_archive_read_firstfile {
6474     my $archive = shift;
6475
6476     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6477
6478     # try to detect archive type first
6479     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6480         die "unable to open file '$archive'\n";
6481     my $firstfile = <$fh>;
6482     kill 15, $pid;
6483     close $fh;
6484
6485     die "ERROR: archive contaions no data\n" if !$firstfile;
6486     chomp $firstfile;
6487
6488     return $firstfile;
6489 }
6490
6491 sub tar_restore_cleanup {
6492     my ($storecfg, $statfile) = @_;
6493
6494     print STDERR "starting cleanup\n";
6495
6496     if (my $fd = IO::File->new($statfile, "r")) {
6497         while (defined(my $line = <$fd>)) {
6498             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6499                 my $volid = $2;
6500                 eval {
6501                     if ($volid =~ m|^/|) {
6502                         unlink $volid || die 'unlink failed\n';
6503                     } else {
6504                         PVE::Storage::vdisk_free($storecfg, $volid);
6505                     }
6506                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6507                 };
6508                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6509             } else {
6510                 print STDERR "unable to parse line in statfile - $line";
6511             }
6512         }
6513         $fd->close();
6514     }
6515 }
6516
6517 sub restore_file_archive {
6518     my ($archive, $vmid, $user, $opts) = @_;
6519
6520     return restore_vma_archive($archive, $vmid, $user, $opts)
6521         if $archive eq '-';
6522
6523     my $info = PVE::Storage::archive_info($archive);
6524     my $format = $opts->{format} // $info->{format};
6525     my $comp = $info->{compression};
6526
6527     # try to detect archive format
6528     if ($format eq 'tar') {
6529         return restore_tar_archive($archive, $vmid, $user, $opts);
6530     } else {
6531         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6532     }
6533 }
6534
6535 # hepler to remove disks that will not be used after restore
6536 my $restore_cleanup_oldconf = sub {
6537     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6538
6539     my $kept_disks = {};
6540
6541     PVE::QemuConfig->foreach_volume($oldconf, sub {
6542         my ($ds, $drive) = @_;
6543
6544         return if drive_is_cdrom($drive, 1);
6545
6546         my $volid = $drive->{file};
6547         return if !$volid || $volid =~ m|^/|;
6548
6549         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6550         return if !$path || !$owner || ($owner != $vmid);
6551
6552         # Note: only delete disk we want to restore
6553         # other volumes will become unused
6554         if ($virtdev_hash->{$ds}) {
6555             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6556             if (my $err = $@) {
6557                 warn $err;
6558             }
6559         } else {
6560             $kept_disks->{$volid} = 1;
6561         }
6562     });
6563
6564     # after the restore we have no snapshots anymore
6565     for my $snapname (keys $oldconf->{snapshots}->%*) {
6566         my $snap = $oldconf->{snapshots}->{$snapname};
6567         if ($snap->{vmstate}) {
6568             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6569             if (my $err = $@) {
6570                 warn $err;
6571             }
6572         }
6573
6574         for my $volid (keys $kept_disks->%*) {
6575             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6576             warn $@ if $@;
6577         }
6578     }
6579 };
6580
6581 # Helper to parse vzdump backup device hints
6582 #
6583 # $rpcenv: Environment, used to ckeck storage permissions
6584 # $user: User ID, to check storage permissions
6585 # $storecfg: Storage configuration
6586 # $fh: the file handle for reading the configuration
6587 # $devinfo: should contain device sizes for all backu-up'ed devices
6588 # $options: backup options (pool, default storage)
6589 #
6590 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6591 my $parse_backup_hints = sub {
6592     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6593
6594     my $check_storage = sub { # assert if an image can be allocate
6595         my ($storeid, $scfg) = @_;
6596         die "Content type 'images' is not available on storage '$storeid'\n"
6597             if !$scfg->{content}->{images};
6598         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6599             if $user ne 'root@pam';
6600     };
6601
6602     my $virtdev_hash = {};
6603     while (defined(my $line = <$fh>)) {
6604         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6605             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6606             die "archive does not contain data for drive '$virtdev'\n"
6607                 if !$devinfo->{$devname};
6608
6609             if (defined($options->{storage})) {
6610                 $storeid = $options->{storage} || 'local';
6611             } elsif (!$storeid) {
6612                 $storeid = 'local';
6613             }
6614             $format = 'raw' if !$format;
6615             $devinfo->{$devname}->{devname} = $devname;
6616             $devinfo->{$devname}->{virtdev} = $virtdev;
6617             $devinfo->{$devname}->{format} = $format;
6618             $devinfo->{$devname}->{storeid} = $storeid;
6619
6620             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6621             $check_storage->($storeid, $scfg); # permission and content type check
6622
6623             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6624         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6625             my $virtdev = $1;
6626             my $drive = parse_drive($virtdev, $2);
6627
6628             if (drive_is_cloudinit($drive)) {
6629                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6630                 $storeid = $options->{storage} if defined ($options->{storage});
6631                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6632                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6633
6634                 $check_storage->($storeid, $scfg); # permission and content type check
6635
6636                 $virtdev_hash->{$virtdev} = {
6637                     format => $format,
6638                     storeid => $storeid,
6639                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6640                     is_cloudinit => 1,
6641                 };
6642             }
6643         }
6644     }
6645
6646     return $virtdev_hash;
6647 };
6648
6649 # Helper to allocate and activate all volumes required for a restore
6650 #
6651 # $storecfg: Storage configuration
6652 # $virtdev_hash: as returned by parse_backup_hints()
6653 #
6654 # Returns: { $virtdev => $volid }
6655 my $restore_allocate_devices = sub {
6656     my ($storecfg, $virtdev_hash, $vmid) = @_;
6657
6658     my $map = {};
6659     foreach my $virtdev (sort keys %$virtdev_hash) {
6660         my $d = $virtdev_hash->{$virtdev};
6661         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6662         my $storeid = $d->{storeid};
6663         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6664
6665         # test if requested format is supported
6666         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6667         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6668         $d->{format} = $defFormat if !$supported;
6669
6670         my $name;
6671         if ($d->{is_cloudinit}) {
6672             $name = "vm-$vmid-cloudinit";
6673             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6674             if ($scfg->{path}) {
6675                 $name .= ".$d->{format}";
6676             }
6677         }
6678
6679         my $volid = PVE::Storage::vdisk_alloc(
6680             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6681
6682         print STDERR "new volume ID is '$volid'\n";
6683         $d->{volid} = $volid;
6684
6685         PVE::Storage::activate_volumes($storecfg, [$volid]);
6686
6687         $map->{$virtdev} = $volid;
6688     }
6689
6690     return $map;
6691 };
6692
6693 sub restore_update_config_line {
6694     my ($cookie, $map, $line, $unique) = @_;
6695
6696     return '' if $line =~ m/^\#qmdump\#/;
6697     return '' if $line =~ m/^\#vzdump\#/;
6698     return '' if $line =~ m/^lock:/;
6699     return '' if $line =~ m/^unused\d+:/;
6700     return '' if $line =~ m/^parent:/;
6701
6702     my $res = '';
6703
6704     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6705     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6706         # try to convert old 1.X settings
6707         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6708         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6709             my ($model, $macaddr) = split(/\=/, $devconfig);
6710             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6711             my $net = {
6712                 model => $model,
6713                 bridge => "vmbr$ind",
6714                 macaddr => $macaddr,
6715             };
6716             my $netstr = print_net($net);
6717
6718             $res .= "net$cookie->{netcount}: $netstr\n";
6719             $cookie->{netcount}++;
6720         }
6721     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6722         my ($id, $netstr) = ($1, $2);
6723         my $net = parse_net($netstr);
6724         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6725         $netstr = print_net($net);
6726         $res .= "$id: $netstr\n";
6727     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6728         my $virtdev = $1;
6729         my $value = $3;
6730         my $di = parse_drive($virtdev, $value);
6731         if (defined($di->{backup}) && !$di->{backup}) {
6732             $res .= "#$line";
6733         } elsif ($map->{$virtdev}) {
6734             delete $di->{format}; # format can change on restore
6735             $di->{file} = $map->{$virtdev};
6736             $value = print_drive($di);
6737             $res .= "$virtdev: $value\n";
6738         } else {
6739             $res .= $line;
6740         }
6741     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6742         my $vmgenid = $1;
6743         if ($vmgenid ne '0') {
6744             # always generate a new vmgenid if there was a valid one setup
6745             $vmgenid = generate_uuid();
6746         }
6747         $res .= "vmgenid: $vmgenid\n";
6748     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6749         my ($uuid, $uuid_str);
6750         UUID::generate($uuid);
6751         UUID::unparse($uuid, $uuid_str);
6752         my $smbios1 = parse_smbios1($2);
6753         $smbios1->{uuid} = $uuid_str;
6754         $res .= $1.print_smbios1($smbios1)."\n";
6755     } else {
6756         $res .= $line;
6757     }
6758
6759     return $res;
6760 }
6761
6762 my $restore_deactivate_volumes = sub {
6763     my ($storecfg, $virtdev_hash) = @_;
6764
6765     my $vollist = [];
6766     for my $dev (values $virtdev_hash->%*) {
6767         push $vollist->@*, $dev->{volid} if $dev->{volid};
6768     }
6769
6770     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6771     print STDERR $@ if $@;
6772 };
6773
6774 my $restore_destroy_volumes = sub {
6775     my ($storecfg, $virtdev_hash) = @_;
6776
6777     for my $dev (values $virtdev_hash->%*) {
6778         my $volid = $dev->{volid} or next;
6779         eval {
6780             PVE::Storage::vdisk_free($storecfg, $volid);
6781             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6782         };
6783         print STDERR "unable to cleanup '$volid' - $@" if $@;
6784     }
6785 };
6786
6787 sub restore_merge_config {
6788     my ($filename, $backup_conf_raw, $override_conf) = @_;
6789
6790     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6791     for my $key (keys $override_conf->%*) {
6792         $backup_conf->{$key} = $override_conf->{$key};
6793     }
6794
6795     return $backup_conf;
6796 }
6797
6798 sub scan_volids {
6799     my ($cfg, $vmid) = @_;
6800
6801     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6802
6803     my $volid_hash = {};
6804     foreach my $storeid (keys %$info) {
6805         foreach my $item (@{$info->{$storeid}}) {
6806             next if !($item->{volid} && $item->{size});
6807             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6808             $volid_hash->{$item->{volid}} = $item;
6809         }
6810     }
6811
6812     return $volid_hash;
6813 }
6814
6815 sub update_disk_config {
6816     my ($vmid, $conf, $volid_hash) = @_;
6817
6818     my $changes;
6819     my $prefix = "VM $vmid";
6820
6821     # used and unused disks
6822     my $referenced = {};
6823
6824     # Note: it is allowed to define multiple storages with same path (alias), so
6825     # we need to check both 'volid' and real 'path' (two different volid can point
6826     # to the same path).
6827
6828     my $referencedpath = {};
6829
6830     # update size info
6831     PVE::QemuConfig->foreach_volume($conf, sub {
6832         my ($opt, $drive) = @_;
6833
6834         my $volid = $drive->{file};
6835         return if !$volid;
6836         my $volume = $volid_hash->{$volid};
6837
6838         # mark volid as "in-use" for next step
6839         $referenced->{$volid} = 1;
6840         if ($volume && (my $path = $volume->{path})) {
6841             $referencedpath->{$path} = 1;
6842         }
6843
6844         return if drive_is_cdrom($drive);
6845         return if !$volume;
6846
6847         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6848         if (defined($updated)) {
6849             $changes = 1;
6850             $conf->{$opt} = print_drive($updated);
6851             print "$prefix ($opt): $msg\n";
6852         }
6853     });
6854
6855     # remove 'unusedX' entry if volume is used
6856     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6857         my ($opt, $drive) = @_;
6858
6859         my $volid = $drive->{file};
6860         return if !$volid;
6861
6862         my $path;
6863         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6864         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6865             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6866             $changes = 1;
6867             delete $conf->{$opt};
6868         }
6869
6870         $referenced->{$volid} = 1;
6871         $referencedpath->{$path} = 1 if $path;
6872     });
6873
6874     foreach my $volid (sort keys %$volid_hash) {
6875         next if $volid =~ m/vm-$vmid-state-/;
6876         next if $referenced->{$volid};
6877         my $path = $volid_hash->{$volid}->{path};
6878         next if !$path; # just to be sure
6879         next if $referencedpath->{$path};
6880         $changes = 1;
6881         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6882         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6883         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6884     }
6885
6886     return $changes;
6887 }
6888
6889 sub rescan {
6890     my ($vmid, $nolock, $dryrun) = @_;
6891
6892     my $cfg = PVE::Storage::config();
6893
6894     print "rescan volumes...\n";
6895     my $volid_hash = scan_volids($cfg, $vmid);
6896
6897     my $updatefn =  sub {
6898         my ($vmid) = @_;
6899
6900         my $conf = PVE::QemuConfig->load_config($vmid);
6901
6902         PVE::QemuConfig->check_lock($conf);
6903
6904         my $vm_volids = {};
6905         foreach my $volid (keys %$volid_hash) {
6906             my $info = $volid_hash->{$volid};
6907             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6908         }
6909
6910         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6911
6912         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6913     };
6914
6915     if (defined($vmid)) {
6916         if ($nolock) {
6917             &$updatefn($vmid);
6918         } else {
6919             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6920         }
6921     } else {
6922         my $vmlist = config_list();
6923         foreach my $vmid (keys %$vmlist) {
6924             if ($nolock) {
6925                 &$updatefn($vmid);
6926             } else {
6927                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6928             }
6929         }
6930     }
6931 }
6932
6933 sub restore_proxmox_backup_archive {
6934     my ($archive, $vmid, $user, $options) = @_;
6935
6936     my $storecfg = PVE::Storage::config();
6937
6938     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6939     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6940
6941     my $fingerprint = $scfg->{fingerprint};
6942     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6943
6944     my $repo = PVE::PBSClient::get_repository($scfg);
6945     my $namespace = $scfg->{namespace};
6946
6947     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6948     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6949     local $ENV{PBS_PASSWORD} = $password;
6950     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6951
6952     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6953         PVE::Storage::parse_volname($storecfg, $archive);
6954
6955     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6956
6957     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6958
6959     my $tmpdir = "/var/tmp/vzdumptmp$$";
6960     rmtree $tmpdir;
6961     mkpath $tmpdir;
6962
6963     my $conffile = PVE::QemuConfig->config_file($vmid);
6964      # disable interrupts (always do cleanups)
6965     local $SIG{INT} =
6966         local $SIG{TERM} =
6967         local $SIG{QUIT} =
6968         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6969
6970     # Note: $oldconf is undef if VM does not exists
6971     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6972     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6973     my $new_conf_raw = '';
6974
6975     my $rpcenv = PVE::RPCEnvironment::get();
6976     my $devinfo = {}; # info about drives included in backup
6977     my $virtdev_hash = {}; # info about allocated drives
6978
6979     eval {
6980         # enable interrupts
6981         local $SIG{INT} =
6982             local $SIG{TERM} =
6983             local $SIG{QUIT} =
6984             local $SIG{HUP} =
6985             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6986
6987         my $cfgfn = "$tmpdir/qemu-server.conf";
6988         my $firewall_config_fn = "$tmpdir/fw.conf";
6989         my $index_fn = "$tmpdir/index.json";
6990
6991         my $cmd = "restore";
6992
6993         my $param = [$pbs_backup_name, "index.json", $index_fn];
6994         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6995         my $index = PVE::Tools::file_get_contents($index_fn);
6996         $index = decode_json($index);
6997
6998         foreach my $info (@{$index->{files}}) {
6999             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7000                 my $devname = $1;
7001                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7002                     $devinfo->{$devname}->{size} = $1;
7003                 } else {
7004                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7005                 }
7006             }
7007         }
7008
7009         my $is_qemu_server_backup = scalar(
7010             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7011         );
7012         if (!$is_qemu_server_backup) {
7013             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7014         }
7015         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7016
7017         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7018         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7019
7020         if ($has_firewall_config) {
7021             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7022             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7023
7024             my $pve_firewall_dir = '/etc/pve/firewall';
7025             mkdir $pve_firewall_dir; # make sure the dir exists
7026             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7027         }
7028
7029         my $fh = IO::File->new($cfgfn, "r") ||
7030             die "unable to read qemu-server.conf - $!\n";
7031
7032         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7033
7034         # fixme: rate limit?
7035
7036         # create empty/temp config
7037         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7038
7039         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7040
7041         # allocate volumes
7042         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7043
7044         foreach my $virtdev (sort keys %$virtdev_hash) {
7045             my $d = $virtdev_hash->{$virtdev};
7046             next if $d->{is_cloudinit}; # no need to restore cloudinit
7047
7048             # this fails if storage is unavailable
7049             my $volid = $d->{volid};
7050             my $path = PVE::Storage::path($storecfg, $volid);
7051
7052             # for live-restore we only want to preload the efidisk and TPM state
7053             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7054
7055             my @ns_arg;
7056             if (defined(my $ns = $scfg->{namespace})) {
7057                 @ns_arg = ('--ns', $ns);
7058             }
7059
7060             my $pbs_restore_cmd = [
7061                 '/usr/bin/pbs-restore',
7062                 '--repository', $repo,
7063                 @ns_arg,
7064                 $pbs_backup_name,
7065                 "$d->{devname}.img.fidx",
7066                 $path,
7067                 '--verbose',
7068                 ];
7069
7070             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7071             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7072
7073             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7074                 push @$pbs_restore_cmd, '--skip-zero';
7075             }
7076
7077             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7078             print "restore proxmox backup image: $dbg_cmdstring\n";
7079             run_command($pbs_restore_cmd);
7080         }
7081
7082         $fh->seek(0, 0) || die "seek failed - $!\n";
7083
7084         my $cookie = { netcount => 0 };
7085         while (defined(my $line = <$fh>)) {
7086             $new_conf_raw .= restore_update_config_line(
7087                 $cookie,
7088                 $map,
7089                 $line,
7090                 $options->{unique},
7091             );
7092         }
7093
7094         $fh->close();
7095     };
7096     my $err = $@;
7097
7098     if ($err || !$options->{live}) {
7099         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7100     }
7101
7102     rmtree $tmpdir;
7103
7104     if ($err) {
7105         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7106         die $err;
7107     }
7108
7109     if ($options->{live}) {
7110         # keep lock during live-restore
7111         $new_conf_raw .= "\nlock: create";
7112     }
7113
7114     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7115     check_restore_permissions($rpcenv, $user, $new_conf);
7116     PVE::QemuConfig->write_config($vmid, $new_conf);
7117
7118     eval { rescan($vmid, 1); };
7119     warn $@ if $@;
7120
7121     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7122
7123     if ($options->{live}) {
7124         # enable interrupts
7125         local $SIG{INT} =
7126             local $SIG{TERM} =
7127             local $SIG{QUIT} =
7128             local $SIG{HUP} =
7129             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7130
7131         my $conf = PVE::QemuConfig->load_config($vmid);
7132         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7133
7134         # these special drives are already restored before start
7135         delete $devinfo->{'drive-efidisk0'};
7136         delete $devinfo->{'drive-tpmstate0-backup'};
7137
7138         my $pbs_opts = {
7139             repo => $repo,
7140             keyfile => $keyfile,
7141             snapshot => $pbs_backup_name,
7142             namespace => $namespace,
7143         };
7144         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7145
7146         PVE::QemuConfig->remove_lock($vmid, "create");
7147     }
7148 }
7149
7150 sub pbs_live_restore {
7151     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7152
7153     print "starting VM for live-restore\n";
7154     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7155
7156     my $pbs_backing = {};
7157     for my $ds (keys %$restored_disks) {
7158         $ds =~ m/^drive-(.*)$/;
7159         my $confname = $1;
7160         $pbs_backing->{$confname} = {
7161             repository => $opts->{repo},
7162             snapshot => $opts->{snapshot},
7163             archive => "$ds.img.fidx",
7164         };
7165         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7166         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7167
7168         my $drive = parse_drive($confname, $conf->{$confname});
7169         print "restoring '$ds' to '$drive->{file}'\n";
7170     }
7171
7172     my $drives_streamed = 0;
7173     eval {
7174         # make sure HA doesn't interrupt our restore by stopping the VM
7175         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7176             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7177         }
7178
7179         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7180         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7181         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7182
7183         my $qmeventd_fd = register_qmeventd_handle($vmid);
7184
7185         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7186         # this will effectively collapse the backing image chain consisting of
7187         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7188         # removes itself once all backing images vanish with 'auto-remove=on')
7189         my $jobs = {};
7190         for my $ds (sort keys %$restored_disks) {
7191             my $job_id = "restore-$ds";
7192             mon_cmd($vmid, 'block-stream',
7193                 'job-id' => $job_id,
7194                 device => "$ds",
7195             );
7196             $jobs->{$job_id} = {};
7197         }
7198
7199         mon_cmd($vmid, 'cont');
7200         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7201
7202         print "restore-drive jobs finished successfully, removing all tracking block devices"
7203             ." to disconnect from Proxmox Backup Server\n";
7204
7205         for my $ds (sort keys %$restored_disks) {
7206             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7207         }
7208
7209         close($qmeventd_fd);
7210     };
7211
7212     my $err = $@;
7213
7214     if ($err) {
7215         warn "An error occurred during live-restore: $err\n";
7216         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7217         die "live-restore failed\n";
7218     }
7219 }
7220
7221 sub restore_vma_archive {
7222     my ($archive, $vmid, $user, $opts, $comp) = @_;
7223
7224     my $readfrom = $archive;
7225
7226     my $cfg = PVE::Storage::config();
7227     my $commands = [];
7228     my $bwlimit = $opts->{bwlimit};
7229
7230     my $dbg_cmdstring = '';
7231     my $add_pipe = sub {
7232         my ($cmd) = @_;
7233         push @$commands, $cmd;
7234         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7235         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7236         $readfrom = '-';
7237     };
7238
7239     my $input = undef;
7240     if ($archive eq '-') {
7241         $input = '<&STDIN';
7242     } else {
7243         # If we use a backup from a PVE defined storage we also consider that
7244         # storage's rate limit:
7245         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7246         if (defined($volid)) {
7247             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7248             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7249             if ($readlimit) {
7250                 print STDERR "applying read rate limit: $readlimit\n";
7251                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7252                 $add_pipe->($cstream);
7253             }
7254         }
7255     }
7256
7257     if ($comp) {
7258         my $info = PVE::Storage::decompressor_info('vma', $comp);
7259         my $cmd = $info->{decompressor};
7260         push @$cmd, $readfrom;
7261         $add_pipe->($cmd);
7262     }
7263
7264     my $tmpdir = "/var/tmp/vzdumptmp$$";
7265     rmtree $tmpdir;
7266
7267     # disable interrupts (always do cleanups)
7268     local $SIG{INT} =
7269         local $SIG{TERM} =
7270         local $SIG{QUIT} =
7271         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7272
7273     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7274     POSIX::mkfifo($mapfifo, 0600);
7275     my $fifofh;
7276     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7277
7278     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7279
7280     my $devinfo = {}; # info about drives included in backup
7281     my $virtdev_hash = {}; # info about allocated drives
7282
7283     my $rpcenv = PVE::RPCEnvironment::get();
7284
7285     my $conffile = PVE::QemuConfig->config_file($vmid);
7286
7287     # Note: $oldconf is undef if VM does not exist
7288     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7289     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7290     my $new_conf_raw = '';
7291
7292     my %storage_limits;
7293
7294     my $print_devmap = sub {
7295         my $cfgfn = "$tmpdir/qemu-server.conf";
7296
7297         # we can read the config - that is already extracted
7298         my $fh = IO::File->new($cfgfn, "r") ||
7299             die "unable to read qemu-server.conf - $!\n";
7300
7301         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7302         if (-f $fwcfgfn) {
7303             my $pve_firewall_dir = '/etc/pve/firewall';
7304             mkdir $pve_firewall_dir; # make sure the dir exists
7305             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7306         }
7307
7308         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7309
7310         foreach my $info (values %{$virtdev_hash}) {
7311             my $storeid = $info->{storeid};
7312             next if defined($storage_limits{$storeid});
7313
7314             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7315             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7316             $storage_limits{$storeid} = $limit * 1024;
7317         }
7318
7319         foreach my $devname (keys %$devinfo) {
7320             die "found no device mapping information for device '$devname'\n"
7321                 if !$devinfo->{$devname}->{virtdev};
7322         }
7323
7324         # create empty/temp config
7325         if ($oldconf) {
7326             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7327             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7328         }
7329
7330         # allocate volumes
7331         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7332
7333         # print restore information to $fifofh
7334         foreach my $virtdev (sort keys %$virtdev_hash) {
7335             my $d = $virtdev_hash->{$virtdev};
7336             next if $d->{is_cloudinit}; # no need to restore cloudinit
7337
7338             my $storeid = $d->{storeid};
7339             my $volid = $d->{volid};
7340
7341             my $map_opts = '';
7342             if (my $limit = $storage_limits{$storeid}) {
7343                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7344             }
7345
7346             my $write_zeros = 1;
7347             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7348                 $write_zeros = 0;
7349             }
7350
7351             my $path = PVE::Storage::path($cfg, $volid);
7352
7353             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7354
7355             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7356         }
7357
7358         $fh->seek(0, 0) || die "seek failed - $!\n";
7359
7360         my $cookie = { netcount => 0 };
7361         while (defined(my $line = <$fh>)) {
7362             $new_conf_raw .= restore_update_config_line(
7363                 $cookie,
7364                 $map,
7365                 $line,
7366                 $opts->{unique},
7367             );
7368         }
7369
7370         $fh->close();
7371     };
7372
7373     my $oldtimeout;
7374
7375     eval {
7376         # enable interrupts
7377         local $SIG{INT} =
7378             local $SIG{TERM} =
7379             local $SIG{QUIT} =
7380             local $SIG{HUP} =
7381             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7382         local $SIG{ALRM} = sub { die "got timeout\n"; };
7383
7384         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7385
7386         my $parser = sub {
7387             my $line = shift;
7388
7389             print "$line\n";
7390
7391             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7392                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7393                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7394             } elsif ($line =~ m/^CTIME: /) {
7395                 # we correctly received the vma config, so we can disable
7396                 # the timeout now for disk allocation
7397                 alarm($oldtimeout || 0);
7398                 $oldtimeout = undef;
7399                 &$print_devmap();
7400                 print $fifofh "done\n";
7401                 close($fifofh);
7402                 $fifofh = undef;
7403             }
7404         };
7405
7406         print "restore vma archive: $dbg_cmdstring\n";
7407         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7408     };
7409     my $err = $@;
7410
7411     alarm($oldtimeout) if $oldtimeout;
7412
7413     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7414
7415     close($fifofh) if $fifofh;
7416     unlink $mapfifo;
7417     rmtree $tmpdir;
7418
7419     if ($err) {
7420         $restore_destroy_volumes->($cfg, $virtdev_hash);
7421         die $err;
7422     }
7423
7424     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7425     check_restore_permissions($rpcenv, $user, $new_conf);
7426     PVE::QemuConfig->write_config($vmid, $new_conf);
7427
7428     eval { rescan($vmid, 1); };
7429     warn $@ if $@;
7430
7431     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7432 }
7433
7434 sub restore_tar_archive {
7435     my ($archive, $vmid, $user, $opts) = @_;
7436
7437     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7438         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7439         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7440     }
7441
7442     if ($archive ne '-') {
7443         my $firstfile = tar_archive_read_firstfile($archive);
7444         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7445             if $firstfile ne 'qemu-server.conf';
7446     }
7447
7448     my $storecfg = PVE::Storage::config();
7449
7450     # avoid zombie disks when restoring over an existing VM -> cleanup first
7451     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7452     # skiplock=1 because qmrestore has set the 'create' lock itself already
7453     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7454     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7455
7456     my $tocmd = "/usr/lib/qemu-server/qmextract";
7457
7458     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7459     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7460     $tocmd .= ' --prealloc' if $opts->{prealloc};
7461     $tocmd .= ' --info' if $opts->{info};
7462
7463     # tar option "xf" does not autodetect compression when read from STDIN,
7464     # so we pipe to zcat
7465     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7466         PVE::Tools::shellquote("--to-command=$tocmd");
7467
7468     my $tmpdir = "/var/tmp/vzdumptmp$$";
7469     mkpath $tmpdir;
7470
7471     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7472     local $ENV{VZDUMP_VMID} = $vmid;
7473     local $ENV{VZDUMP_USER} = $user;
7474
7475     my $conffile = PVE::QemuConfig->config_file($vmid);
7476     my $new_conf_raw = '';
7477
7478     # disable interrupts (always do cleanups)
7479     local $SIG{INT} =
7480         local $SIG{TERM} =
7481         local $SIG{QUIT} =
7482         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7483
7484     eval {
7485         # enable interrupts
7486         local $SIG{INT} =
7487             local $SIG{TERM} =
7488             local $SIG{QUIT} =
7489             local $SIG{HUP} =
7490             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7491
7492         if ($archive eq '-') {
7493             print "extracting archive from STDIN\n";
7494             run_command($cmd, input => "<&STDIN");
7495         } else {
7496             print "extracting archive '$archive'\n";
7497             run_command($cmd);
7498         }
7499
7500         return if $opts->{info};
7501
7502         # read new mapping
7503         my $map = {};
7504         my $statfile = "$tmpdir/qmrestore.stat";
7505         if (my $fd = IO::File->new($statfile, "r")) {
7506             while (defined (my $line = <$fd>)) {
7507                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7508                     $map->{$1} = $2 if $1;
7509                 } else {
7510                     print STDERR "unable to parse line in statfile - $line\n";
7511                 }
7512             }
7513             $fd->close();
7514         }
7515
7516         my $confsrc = "$tmpdir/qemu-server.conf";
7517
7518         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7519
7520         my $cookie = { netcount => 0 };
7521         while (defined (my $line = <$srcfd>)) {
7522             $new_conf_raw .= restore_update_config_line(
7523                 $cookie,
7524                 $map,
7525                 $line,
7526                 $opts->{unique},
7527             );
7528         }
7529
7530         $srcfd->close();
7531     };
7532     if (my $err = $@) {
7533         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7534         die $err;
7535     }
7536
7537     rmtree $tmpdir;
7538
7539     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7540
7541     PVE::Cluster::cfs_update(); # make sure we read new file
7542
7543     eval { rescan($vmid, 1); };
7544     warn $@ if $@;
7545 };
7546
7547 sub foreach_storage_used_by_vm {
7548     my ($conf, $func) = @_;
7549
7550     my $sidhash = {};
7551
7552     PVE::QemuConfig->foreach_volume($conf, sub {
7553         my ($ds, $drive) = @_;
7554         return if drive_is_cdrom($drive);
7555
7556         my $volid = $drive->{file};
7557
7558         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7559         $sidhash->{$sid} = $sid if $sid;
7560     });
7561
7562     foreach my $sid (sort keys %$sidhash) {
7563         &$func($sid);
7564     }
7565 }
7566
7567 my $qemu_snap_storage = {
7568     rbd => 1,
7569 };
7570 sub do_snapshots_with_qemu {
7571     my ($storecfg, $volid, $deviceid) = @_;
7572
7573     return if $deviceid =~ m/tpmstate0/;
7574
7575     my $storage_name = PVE::Storage::parse_volume_id($volid);
7576     my $scfg = $storecfg->{ids}->{$storage_name};
7577     die "could not find storage '$storage_name'\n" if !defined($scfg);
7578
7579     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7580         return 1;
7581     }
7582
7583     if ($volid =~ m/\.(qcow2|qed)$/){
7584         return 1;
7585     }
7586
7587     return;
7588 }
7589
7590 sub qga_check_running {
7591     my ($vmid, $nowarn) = @_;
7592
7593     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7594     if ($@) {
7595         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7596         return 0;
7597     }
7598     return 1;
7599 }
7600
7601 sub template_create {
7602     my ($vmid, $conf, $disk) = @_;
7603
7604     my $storecfg = PVE::Storage::config();
7605
7606     PVE::QemuConfig->foreach_volume($conf, sub {
7607         my ($ds, $drive) = @_;
7608
7609         return if drive_is_cdrom($drive);
7610         return if $disk && $ds ne $disk;
7611
7612         my $volid = $drive->{file};
7613         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7614
7615         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7616         $drive->{file} = $voliddst;
7617         $conf->{$ds} = print_drive($drive);
7618         PVE::QemuConfig->write_config($vmid, $conf);
7619     });
7620 }
7621
7622 sub convert_iscsi_path {
7623     my ($path) = @_;
7624
7625     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7626         my $portal = $1;
7627         my $target = $2;
7628         my $lun = $3;
7629
7630         my $initiator_name = get_initiator_name();
7631
7632         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7633                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7634     }
7635
7636     die "cannot convert iscsi path '$path', unkown format\n";
7637 }
7638
7639 sub qemu_img_convert {
7640     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7641
7642     my $storecfg = PVE::Storage::config();
7643     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7644     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7645
7646     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7647
7648     my $cachemode;
7649     my $src_path;
7650     my $src_is_iscsi = 0;
7651     my $src_format;
7652
7653     if ($src_storeid) {
7654         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7655         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7656         $src_format = qemu_img_format($src_scfg, $src_volname);
7657         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7658         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7659         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7660     } elsif (-f $src_volid || -b $src_volid) {
7661         $src_path = $src_volid;
7662         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7663             $src_format = $1;
7664         }
7665     }
7666
7667     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7668
7669     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7670     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7671     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7672     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7673
7674     my $cmd = [];
7675     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7676     push @$cmd, '-l', "snapshot.name=$snapname"
7677         if $snapname && $src_format && $src_format eq "qcow2";
7678     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7679     push @$cmd, '-T', $cachemode if defined($cachemode);
7680     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7681
7682     if ($src_is_iscsi) {
7683         push @$cmd, '--image-opts';
7684         $src_path = convert_iscsi_path($src_path);
7685     } elsif ($src_format) {
7686         push @$cmd, '-f', $src_format;
7687     }
7688
7689     if ($dst_is_iscsi) {
7690         push @$cmd, '--target-image-opts';
7691         $dst_path = convert_iscsi_path($dst_path);
7692     } else {
7693         push @$cmd, '-O', $dst_format;
7694     }
7695
7696     push @$cmd, $src_path;
7697
7698     if (!$dst_is_iscsi && $is_zero_initialized) {
7699         push @$cmd, "zeroinit:$dst_path";
7700     } else {
7701         push @$cmd, $dst_path;
7702     }
7703
7704     my $parser = sub {
7705         my $line = shift;
7706         if($line =~ m/\((\S+)\/100\%\)/){
7707             my $percent = $1;
7708             my $transferred = int($size * $percent / 100);
7709             my $total_h = render_bytes($size, 1);
7710             my $transferred_h = render_bytes($transferred, 1);
7711
7712             print "transferred $transferred_h of $total_h ($percent%)\n";
7713         }
7714
7715     };
7716
7717     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7718     my $err = $@;
7719     die "copy failed: $err" if $err;
7720 }
7721
7722 sub qemu_img_format {
7723     my ($scfg, $volname) = @_;
7724
7725     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7726         return $1;
7727     } else {
7728         return "raw";
7729     }
7730 }
7731
7732 sub qemu_drive_mirror {
7733     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7734
7735     $jobs = {} if !$jobs;
7736
7737     my $qemu_target;
7738     my $format;
7739     $jobs->{"drive-$drive"} = {};
7740
7741     if ($dst_volid =~ /^nbd:/) {
7742         $qemu_target = $dst_volid;
7743         $format = "nbd";
7744     } else {
7745         my $storecfg = PVE::Storage::config();
7746         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7747
7748         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7749
7750         $format = qemu_img_format($dst_scfg, $dst_volname);
7751
7752         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7753
7754         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7755     }
7756
7757     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7758     $opts->{format} = $format if $format;
7759
7760     if (defined($src_bitmap)) {
7761         $opts->{sync} = 'incremental';
7762         $opts->{bitmap} = $src_bitmap;
7763         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7764     }
7765
7766     if (defined($bwlimit)) {
7767         $opts->{speed} = $bwlimit * 1024;
7768         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7769     } else {
7770         print "drive mirror is starting for drive-$drive\n";
7771     }
7772
7773     # if a job already runs for this device we get an error, catch it for cleanup
7774     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7775     if (my $err = $@) {
7776         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7777         warn "$@\n" if $@;
7778         die "mirroring error: $err\n";
7779     }
7780
7781     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7782 }
7783
7784 # $completion can be either
7785 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7786 # 'cancel': wait until all jobs are ready, block-job-cancel them
7787 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7788 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7789 sub qemu_drive_mirror_monitor {
7790     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7791
7792     $completion //= 'complete';
7793     $op //= "mirror";
7794
7795     eval {
7796         my $err_complete = 0;
7797
7798         my $starttime = time ();
7799         while (1) {
7800             die "block job ('$op') timed out\n" if $err_complete > 300;
7801
7802             my $stats = mon_cmd($vmid, "query-block-jobs");
7803             my $ctime = time();
7804
7805             my $running_jobs = {};
7806             for my $stat (@$stats) {
7807                 next if $stat->{type} ne $op;
7808                 $running_jobs->{$stat->{device}} = $stat;
7809             }
7810
7811             my $readycounter = 0;
7812
7813             for my $job_id (sort keys %$jobs) {
7814                 my $job = $running_jobs->{$job_id};
7815
7816                 my $vanished = !defined($job);
7817                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7818                 if($complete || ($vanished && $completion eq 'auto')) {
7819                     print "$job_id: $op-job finished\n";
7820                     delete $jobs->{$job_id};
7821                     next;
7822                 }
7823
7824                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7825
7826                 my $busy = $job->{busy};
7827                 my $ready = $job->{ready};
7828                 if (my $total = $job->{len}) {
7829                     my $transferred = $job->{offset} || 0;
7830                     my $remaining = $total - $transferred;
7831                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7832
7833                     my $duration = $ctime - $starttime;
7834                     my $total_h = render_bytes($total, 1);
7835                     my $transferred_h = render_bytes($transferred, 1);
7836
7837                     my $status = sprintf(
7838                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7839                         render_duration($duration),
7840                     );
7841
7842                     if ($ready) {
7843                         if ($busy) {
7844                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7845                         } else {
7846                             $status .= ", ready";
7847                         }
7848                     }
7849                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7850                     $jobs->{$job_id}->{ready} = $ready;
7851                 }
7852
7853                 $readycounter++ if $job->{ready};
7854             }
7855
7856             last if scalar(keys %$jobs) == 0;
7857
7858             if ($readycounter == scalar(keys %$jobs)) {
7859                 print "all '$op' jobs are ready\n";
7860
7861                 # do the complete later (or has already been done)
7862                 last if $completion eq 'skip' || $completion eq 'auto';
7863
7864                 if ($vmiddst && $vmiddst != $vmid) {
7865                     my $agent_running = $qga && qga_check_running($vmid);
7866                     if ($agent_running) {
7867                         print "freeze filesystem\n";
7868                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7869                         warn $@ if $@;
7870                     } else {
7871                         print "suspend vm\n";
7872                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7873                         warn $@ if $@;
7874                     }
7875
7876                     # if we clone a disk for a new target vm, we don't switch the disk
7877                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7878
7879                     if ($agent_running) {
7880                         print "unfreeze filesystem\n";
7881                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7882                         warn $@ if $@;
7883                     } else {
7884                         print "resume vm\n";
7885                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7886                         warn $@ if $@;
7887                     }
7888
7889                     last;
7890                 } else {
7891
7892                     for my $job_id (sort keys %$jobs) {
7893                         # try to switch the disk if source and destination are on the same guest
7894                         print "$job_id: Completing block job_id...\n";
7895
7896                         my $op;
7897                         if ($completion eq 'complete') {
7898                             $op = 'block-job-complete';
7899                         } elsif ($completion eq 'cancel') {
7900                             $op = 'block-job-cancel';
7901                         } else {
7902                             die "invalid completion value: $completion\n";
7903                         }
7904                         eval { mon_cmd($vmid, $op, device => $job_id) };
7905                         if ($@ =~ m/cannot be completed/) {
7906                             print "$job_id: block job cannot be completed, trying again.\n";
7907                             $err_complete++;
7908                         }else {
7909                             print "$job_id: Completed successfully.\n";
7910                             $jobs->{$job_id}->{complete} = 1;
7911                         }
7912                     }
7913                 }
7914             }
7915             sleep 1;
7916         }
7917     };
7918     my $err = $@;
7919
7920     if ($err) {
7921         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7922         die "block job ($op) error: $err";
7923     }
7924 }
7925
7926 sub qemu_blockjobs_cancel {
7927     my ($vmid, $jobs) = @_;
7928
7929     foreach my $job (keys %$jobs) {
7930         print "$job: Cancelling block job\n";
7931         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7932         $jobs->{$job}->{cancel} = 1;
7933     }
7934
7935     while (1) {
7936         my $stats = mon_cmd($vmid, "query-block-jobs");
7937
7938         my $running_jobs = {};
7939         foreach my $stat (@$stats) {
7940             $running_jobs->{$stat->{device}} = $stat;
7941         }
7942
7943         foreach my $job (keys %$jobs) {
7944
7945             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7946                 print "$job: Done.\n";
7947                 delete $jobs->{$job};
7948             }
7949         }
7950
7951         last if scalar(keys %$jobs) == 0;
7952
7953         sleep 1;
7954     }
7955 }
7956
7957 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7958 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7959 my sub clone_disk_check_io_uring {
7960     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7961
7962     return if !$use_drive_mirror;
7963
7964     # Don't complain when not changing storage.
7965     # Assume if it works for the source, it'll work for the target too.
7966     return if $src_storeid eq $dst_storeid;
7967
7968     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7969     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7970
7971     my $cache_direct = drive_uses_cache_direct($src_drive);
7972
7973     my $src_uses_io_uring;
7974     if ($src_drive->{aio}) {
7975         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
7976     } else {
7977         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
7978     }
7979
7980     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
7981         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
7982 }
7983
7984 sub clone_disk {
7985     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7986
7987     my ($vmid, $running) = $source->@{qw(vmid running)};
7988     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7989
7990     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7991     my ($storage, $format) = $dest->@{qw(storage format)};
7992
7993     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7994
7995     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7996         die "cloning from/to EFI disk requires EFI disk\n"
7997             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7998         die "cloning from/to TPM state requires TPM state\n"
7999             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8000
8001         # This would lead to two device nodes in QEMU pointing to the same backing image!
8002         die "cannot change drive name when cloning disk from/to the same VM\n"
8003             if $use_drive_mirror && $vmid == $newvmid;
8004     }
8005
8006     die "cannot move TPM state while VM is running\n"
8007         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8008
8009     my $newvolid;
8010
8011     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8012     print "$src_drivename " if $src_drivename;
8013     print "($drive->{file})\n";
8014
8015     if (!$full) {
8016         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8017         push @$newvollist, $newvolid;
8018     } else {
8019         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8020         my $storeid = $storage || $src_storeid;
8021
8022         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8023
8024         my $name = undef;
8025         my $size = undef;
8026         if (drive_is_cloudinit($drive)) {
8027             $name = "vm-$newvmid-cloudinit";
8028             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8029             if ($scfg->{path}) {
8030                 $name .= ".$dst_format";
8031             }
8032             $snapname = undef;
8033             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8034         } elsif ($dst_drivename eq 'efidisk0') {
8035             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8036         } elsif ($dst_drivename eq 'tpmstate0') {
8037             $dst_format = 'raw';
8038             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8039         } else {
8040             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8041
8042             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8043         }
8044         $newvolid = PVE::Storage::vdisk_alloc(
8045             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8046         );
8047         push @$newvollist, $newvolid;
8048
8049         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8050
8051         if (drive_is_cloudinit($drive)) {
8052             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8053             # if this is the case, we have to complete any block-jobs still there from
8054             # previous drive-mirrors
8055             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8056                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8057             }
8058             goto no_data_clone;
8059         }
8060
8061         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8062         if ($use_drive_mirror) {
8063             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8064                 $completion, $qga, $bwlimit);
8065         } else {
8066             if ($dst_drivename eq 'efidisk0') {
8067                 # the relevant data on the efidisk may be smaller than the source
8068                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8069                 # that is given by the OVMF_VARS.fd
8070                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8071                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8072
8073                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8074
8075                 # better for Ceph if block size is not too small, see bug #3324
8076                 my $bs = 1024*1024;
8077
8078                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8079
8080                 if ($src_format eq 'qcow2' && $snapname) {
8081                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8082                         if !min_version(kvm_user_version(), 6, 2);
8083                     push $cmd->@*, '-l', $snapname;
8084                 }
8085                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8086                 run_command($cmd);
8087             } else {
8088                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8089             }
8090         }
8091     }
8092
8093 no_data_clone:
8094     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8095
8096     my $disk = dclone($drive);
8097     delete $disk->{format};
8098     $disk->{file} = $newvolid;
8099     $disk->{size} = $size if defined($size);
8100
8101     return $disk;
8102 }
8103
8104 sub get_running_qemu_version {
8105     my ($vmid) = @_;
8106     my $res = mon_cmd($vmid, "query-version");
8107     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8108 }
8109
8110 sub qemu_use_old_bios_files {
8111     my ($machine_type) = @_;
8112
8113     return if !$machine_type;
8114
8115     my $use_old_bios_files = undef;
8116
8117     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8118         $machine_type = $1;
8119         $use_old_bios_files = 1;
8120     } else {
8121         my $version = extract_version($machine_type, kvm_user_version());
8122         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8123         # load new efi bios files on migration. So this hack is required to allow
8124         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8125         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8126         $use_old_bios_files = !min_version($version, 2, 4);
8127     }
8128
8129     return ($use_old_bios_files, $machine_type);
8130 }
8131
8132 sub get_efivars_size {
8133     my ($conf, $efidisk) = @_;
8134
8135     my $arch = get_vm_arch($conf);
8136     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8137     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8138     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8139     return -s $ovmf_vars;
8140 }
8141
8142 sub update_efidisk_size {
8143     my ($conf) = @_;
8144
8145     return if !defined($conf->{efidisk0});
8146
8147     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8148     $disk->{size} = get_efivars_size($conf);
8149     $conf->{efidisk0} = print_drive($disk);
8150
8151     return;
8152 }
8153
8154 sub update_tpmstate_size {
8155     my ($conf) = @_;
8156
8157     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8158     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8159     $conf->{tpmstate0} = print_drive($disk);
8160 }
8161
8162 sub create_efidisk($$$$$$$) {
8163     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8164
8165     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8166
8167     my $vars_size_b = -s $ovmf_vars;
8168     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8169     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8170     PVE::Storage::activate_volumes($storecfg, [$volid]);
8171
8172     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8173     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8174
8175     return ($volid, $size/1024);
8176 }
8177
8178 sub vm_iothreads_list {
8179     my ($vmid) = @_;
8180
8181     my $res = mon_cmd($vmid, 'query-iothreads');
8182
8183     my $iothreads = {};
8184     foreach my $iothread (@$res) {
8185         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8186     }
8187
8188     return $iothreads;
8189 }
8190
8191 sub scsihw_infos {
8192     my ($conf, $drive) = @_;
8193
8194     my $maxdev = 0;
8195
8196     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8197         $maxdev = 7;
8198     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8199         $maxdev = 1;
8200     } else {
8201         $maxdev = 256;
8202     }
8203
8204     my $controller = int($drive->{index} / $maxdev);
8205     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8206         ? "virtioscsi"
8207         : "scsihw";
8208
8209     return ($maxdev, $controller, $controller_prefix);
8210 }
8211
8212 sub resolve_dst_disk_format {
8213         my ($storecfg, $storeid, $src_volname, $format) = @_;
8214         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8215
8216         if (!$format) {
8217             # if no target format is specified, use the source disk format as hint
8218             if ($src_volname) {
8219                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8220                 $format = qemu_img_format($scfg, $src_volname);
8221             } else {
8222                 return $defFormat;
8223             }
8224         }
8225
8226         # test if requested format is supported - else use default
8227         my $supported = grep { $_ eq $format } @$validFormats;
8228         $format = $defFormat if !$supported;
8229         return $format;
8230 }
8231
8232 # NOTE: if this logic changes, please update docs & possibly gui logic
8233 sub find_vmstate_storage {
8234     my ($conf, $storecfg) = @_;
8235
8236     # first, return storage from conf if set
8237     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8238
8239     my ($target, $shared, $local);
8240
8241     foreach_storage_used_by_vm($conf, sub {
8242         my ($sid) = @_;
8243         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8244         my $dst = $scfg->{shared} ? \$shared : \$local;
8245         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8246     });
8247
8248     # second, use shared storage where VM has at least one disk
8249     # third, use local storage where VM has at least one disk
8250     # fall back to local storage
8251     $target = $shared // $local // 'local';
8252
8253     return $target;
8254 }
8255
8256 sub generate_uuid {
8257     my ($uuid, $uuid_str);
8258     UUID::generate($uuid);
8259     UUID::unparse($uuid, $uuid_str);
8260     return $uuid_str;
8261 }
8262
8263 sub generate_smbios1_uuid {
8264     return "uuid=".generate_uuid();
8265 }
8266
8267 sub nbd_stop {
8268     my ($vmid) = @_;
8269
8270     mon_cmd($vmid, 'nbd-server-stop');
8271 }
8272
8273 sub create_reboot_request {
8274     my ($vmid) = @_;
8275     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8276         or die "failed to create reboot trigger file: $!\n";
8277     close($fh);
8278 }
8279
8280 sub clear_reboot_request {
8281     my ($vmid) = @_;
8282     my $path = "/run/qemu-server/$vmid.reboot";
8283     my $res = 0;
8284
8285     $res = unlink($path);
8286     die "could not remove reboot request for $vmid: $!"
8287         if !$res && $! != POSIX::ENOENT;
8288
8289     return $res;
8290 }
8291
8292 sub bootorder_from_legacy {
8293     my ($conf, $bootcfg) = @_;
8294
8295     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8296     my $bootindex_hash = {};
8297     my $i = 1;
8298     foreach my $o (split(//, $boot)) {
8299         $bootindex_hash->{$o} = $i*100;
8300         $i++;
8301     }
8302
8303     my $bootorder = {};
8304
8305     PVE::QemuConfig->foreach_volume($conf, sub {
8306         my ($ds, $drive) = @_;
8307
8308         if (drive_is_cdrom ($drive, 1)) {
8309             if ($bootindex_hash->{d}) {
8310                 $bootorder->{$ds} = $bootindex_hash->{d};
8311                 $bootindex_hash->{d} += 1;
8312             }
8313         } elsif ($bootindex_hash->{c}) {
8314             $bootorder->{$ds} = $bootindex_hash->{c}
8315                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8316             $bootindex_hash->{c} += 1;
8317         }
8318     });
8319
8320     if ($bootindex_hash->{n}) {
8321         for (my $i = 0; $i < $MAX_NETS; $i++) {
8322             my $netname = "net$i";
8323             next if !$conf->{$netname};
8324             $bootorder->{$netname} = $bootindex_hash->{n};
8325             $bootindex_hash->{n} += 1;
8326         }
8327     }
8328
8329     return $bootorder;
8330 }
8331
8332 # Generate default device list for 'boot: order=' property. Matches legacy
8333 # default boot order, but with explicit device names. This is important, since
8334 # the fallback for when neither 'order' nor the old format is specified relies
8335 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8336 sub get_default_bootdevices {
8337     my ($conf) = @_;
8338
8339     my @ret = ();
8340
8341     # harddisk
8342     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8343     push @ret, $first if $first;
8344
8345     # cdrom
8346     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8347     push @ret, $first if $first;
8348
8349     # network
8350     for (my $i = 0; $i < $MAX_NETS; $i++) {
8351         my $netname = "net$i";
8352         next if !$conf->{$netname};
8353         push @ret, $netname;
8354         last;
8355     }
8356
8357     return \@ret;
8358 }
8359
8360 sub device_bootorder {
8361     my ($conf) = @_;
8362
8363     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8364
8365     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8366
8367     my $bootorder = {};
8368     if (!defined($boot) || $boot->{legacy}) {
8369         $bootorder = bootorder_from_legacy($conf, $boot);
8370     } elsif ($boot->{order}) {
8371         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8372         for my $dev (PVE::Tools::split_list($boot->{order})) {
8373             $bootorder->{$dev} = $i++;
8374         }
8375     }
8376
8377     return $bootorder;
8378 }
8379
8380 sub register_qmeventd_handle {
8381     my ($vmid) = @_;
8382
8383     my $fh;
8384     my $peer = "/var/run/qmeventd.sock";
8385     my $count = 0;
8386
8387     for (;;) {
8388         $count++;
8389         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8390         last if $fh;
8391         if ($! != EINTR && $! != EAGAIN) {
8392             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8393         }
8394         if ($count > 4) {
8395             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8396               . "after $count retries\n";
8397         }
8398         usleep(25000);
8399     }
8400
8401     # send handshake to mark VM as backing up
8402     print $fh to_json({vzdump => {vmid => "$vmid"}});
8403
8404     # return handle to be closed later when inhibit is no longer required
8405     return $fh;
8406 }
8407
8408 # bash completion helper
8409
8410 sub complete_backup_archives {
8411     my ($cmdname, $pname, $cvalue) = @_;
8412
8413     my $cfg = PVE::Storage::config();
8414
8415     my $storeid;
8416
8417     if ($cvalue =~ m/^([^:]+):/) {
8418         $storeid = $1;
8419     }
8420
8421     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8422
8423     my $res = [];
8424     foreach my $id (keys %$data) {
8425         foreach my $item (@{$data->{$id}}) {
8426             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8427             push @$res, $item->{volid} if defined($item->{volid});
8428         }
8429     }
8430
8431     return $res;
8432 }
8433
8434 my $complete_vmid_full = sub {
8435     my ($running) = @_;
8436
8437     my $idlist = vmstatus();
8438
8439     my $res = [];
8440
8441     foreach my $id (keys %$idlist) {
8442         my $d = $idlist->{$id};
8443         if (defined($running)) {
8444             next if $d->{template};
8445             next if $running && $d->{status} ne 'running';
8446             next if !$running && $d->{status} eq 'running';
8447         }
8448         push @$res, $id;
8449
8450     }
8451     return $res;
8452 };
8453
8454 sub complete_vmid {
8455     return &$complete_vmid_full();
8456 }
8457
8458 sub complete_vmid_stopped {
8459     return &$complete_vmid_full(0);
8460 }
8461
8462 sub complete_vmid_running {
8463     return &$complete_vmid_full(1);
8464 }
8465
8466 sub complete_storage {
8467
8468     my $cfg = PVE::Storage::config();
8469     my $ids = $cfg->{ids};
8470
8471     my $res = [];
8472     foreach my $sid (keys %$ids) {
8473         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8474         next if !$ids->{$sid}->{content}->{images};
8475         push @$res, $sid;
8476     }
8477
8478     return $res;
8479 }
8480
8481 sub complete_migration_storage {
8482     my ($cmd, $param, $current_value, $all_args) = @_;
8483
8484     my $targetnode = @$all_args[1];
8485
8486     my $cfg = PVE::Storage::config();
8487     my $ids = $cfg->{ids};
8488
8489     my $res = [];
8490     foreach my $sid (keys %$ids) {
8491         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8492         next if !$ids->{$sid}->{content}->{images};
8493         push @$res, $sid;
8494     }
8495
8496     return $res;
8497 }
8498
8499 sub vm_is_paused {
8500     my ($vmid) = @_;
8501     my $qmpstatus = eval {
8502         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8503         mon_cmd($vmid, "query-status");
8504     };
8505     warn "$@\n" if $@;
8506     return $qmpstatus && (
8507         $qmpstatus->{status} eq "paused" ||
8508         $qmpstatus->{status} eq "suspended" ||
8509         $qmpstatus->{status} eq "prelaunch"
8510     );
8511 }
8512
8513 sub check_volume_storage_type {
8514     my ($storecfg, $vol) = @_;
8515
8516     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8517     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8518     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8519
8520     die "storage '$storeid' does not support content-type '$vtype'\n"
8521         if !$scfg->{content}->{$vtype};
8522
8523     return 1;
8524 }
8525
8526 sub add_nets_bridge_fdb {
8527     my ($conf, $vmid) = @_;
8528
8529     for my $opt (keys %$conf) {
8530         next if $opt !~ m/^net(\d+)$/;
8531         my $iface = "tap${vmid}i$1";
8532         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8533         my $net = parse_net($conf->{$opt}, 1) or next;
8534
8535         my $mac = $net->{macaddr};
8536         if (!$mac) {
8537             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8538                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8539             next;
8540         }
8541
8542         my $bridge = $net->{bridge};
8543         if (!$bridge) {
8544             log_warn("Interface '$iface' not attached to any bridge.");
8545             next;
8546         }
8547         if ($have_sdn) {
8548             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8549         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8550             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8551         }
8552     }
8553 }
8554
8555 sub del_nets_bridge_fdb {
8556     my ($conf, $vmid) = @_;
8557
8558     for my $opt (keys %$conf) {
8559         next if $opt !~ m/^net(\d+)$/;
8560         my $iface = "tap${vmid}i$1";
8561
8562         my $net = parse_net($conf->{$opt}) or next;
8563         my $mac = $net->{macaddr} or next;
8564
8565         my $bridge = $net->{bridge};
8566         if ($have_sdn) {
8567             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8568         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8569             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8570         }
8571     }
8572 }
8573
8574 1;