PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory qw(get_current_memory);
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  62 use PVE::QemuServer::USB;
  63
  64 my $have_sdn;
  65 eval {
  66     require PVE::Network::SDN::Zones;
  67     $have_sdn = 1;
  68 };
  69
  70 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  71 my $OVMF = {
  72     x86_64 => {
  73         '4m-no-smm' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  76         ],
  77         '4m-no-smm-ms' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  80         ],
  81         '4m' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  84         ],
  85         '4m-ms' => [
  86             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  88         ],
  89         default => [
  90             "$EDK2_FW_BASE/OVMF_CODE.fd",
  91             "$EDK2_FW_BASE/OVMF_VARS.fd",
  92         ],
  93     },
  94     aarch64 => {
  95         default => [
  96             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  97             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  98         ],
  99     },
 100 };
 101
 102 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 103
 104 # Note about locking: we use flock on the config file protect against concurent actions.
 105 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 106 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 107 # But you can ignore this kind of lock with the --skiplock flag.
 108
 109 cfs_register_file(
 110     '/qemu-server/',
 111     \&parse_vm_config,
 112     \&write_vm_config
 113 );
 114
 115 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 116     description => "Some command save/restore state from this location.",
 117     type => 'string',
 118     maxLength => 128,
 119     optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 123         description => "Specifies the QEMU machine type.",
 124         type => 'string',
 125         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 126         maxLength => 40,
 127         optional => 1,
 128 });
 129
 130 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0,
 167     },
 168     'freeze-fs-on-backup' => {
 169         description => "Freeze/thaw guest filesystems on backup for consistency.",
 170         type => 'boolean',
 171         optional => 1,
 172         default => 1,
 173     },
 174     type => {
 175         description => "Select the agent type",
 176         type => 'string',
 177         default => 'virtio',
 178         optional => 1,
 179         enum => [qw(virtio isa)],
 180     },
 181 };
 182
 183 my $vga_fmt = {
 184     type => {
 185         description => "Select the VGA type.",
 186         type => 'string',
 187         default => 'std',
 188         optional => 1,
 189         default_key => 1,
 190         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 191     },
 192     memory => {
 193         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 194         type => 'integer',
 195         optional => 1,
 196         minimum => 4,
 197         maximum => 512,
 198     },
 199 };
 200
 201 my $ivshmem_fmt = {
 202     size => {
 203         type => 'integer',
 204         minimum => 1,
 205         description => "The size of the file in MB.",
 206     },
 207     name => {
 208         type => 'string',
 209         pattern => '[a-zA-Z0-9\-]+',
 210         optional => 1,
 211         format_description => 'string',
 212         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 213     },
 214 };
 215
 216 my $audio_fmt = {
 217     device => {
 218         type => 'string',
 219         enum => [qw(ich9-intel-hda intel-hda AC97)],
 220         description =>  "Configure an audio device."
 221     },
 222     driver =>  {
 223         type => 'string',
 224         enum => ['spice', 'none'],
 225         default => 'spice',
 226         optional => 1,
 227         description => "Driver backend for the audio device."
 228     },
 229 };
 230
 231 my $spice_enhancements_fmt = {
 232     foldersharing => {
 233         type => 'boolean',
 234         optional => 1,
 235         default => '0',
 236         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 237     },
 238     videostreaming =>  {
 239         type => 'string',
 240         enum => ['off', 'all', 'filter'],
 241         default => 'off',
 242         optional => 1,
 243         description => "Enable video streaming. Uses compression for detected video streams."
 244     },
 245 };
 246
 247 my $rng_fmt = {
 248     source => {
 249         type => 'string',
 250         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 251         default_key => 1,
 252         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 253             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 254             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 255             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 256             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 257             ." a hardware RNG from the host.",
 258     },
 259     max_bytes => {
 260         type => 'integer',
 261         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 262             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 263             ." `0` to disable limiting (potentially dangerous!).",
 264         optional => 1,
 265
 266         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 267         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 268         # reading from /dev/urandom
 269         default => 1024,
 270     },
 271     period => {
 272         type => 'integer',
 273         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 274             ." the guest to retrieve another 'max_bytes' of entropy.",
 275         optional => 1,
 276         default => 1000,
 277     },
 278 };
 279
 280 my $meta_info_fmt = {
 281     'ctime' => {
 282         type => 'integer',
 283         description => "The guest creation timestamp as UNIX epoch time",
 284         minimum => 0,
 285         optional => 1,
 286     },
 287     'creation-qemu' => {
 288         type => 'string',
 289         description => "The QEMU (machine) version from the time this VM was created.",
 290         pattern => '\d+(\.\d+)+',
 291         optional => 1,
 292     },
 293 };
 294
 295 my $confdesc = {
 296     onboot => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Specifies whether a VM will be started during system bootup.",
 300         default => 0,
 301     },
 302     autostart => {
 303         optional => 1,
 304         type => 'boolean',
 305         description => "Automatic restart after crash (currently ignored).",
 306         default => 0,
 307     },
 308     hotplug => {
 309         optional => 1,
 310         type => 'string', format => 'pve-hotplug-features',
 311         description => "Selectively enable hotplug features. This is a comma separated list of"
 312             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 313             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 314             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 315             ." windows > 7.",
 316         default => 'network,disk,usb',
 317     },
 318     reboot => {
 319         optional => 1,
 320         type => 'boolean',
 321         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 322         default => 1,
 323     },
 324     lock => {
 325         optional => 1,
 326         type => 'string',
 327         description => "Lock/unlock the VM.",
 328         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 329     },
 330     cpulimit => {
 331         optional => 1,
 332         type => 'number',
 333         description => "Limit of CPU usage.",
 334         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 335             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 336         minimum => 0,
 337         maximum => 128,
 338         default => 0,
 339     },
 340     cpuunits => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 344         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 345             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 346             ." weights of all the other running VMs.",
 347         minimum => 1,
 348         maximum => 262144,
 349         default => 'cgroup v1: 1024, cgroup v2: 100',
 350     },
 351     memory => {
 352         optional => 1,
 353         type => 'string',
 354         description => "Memory properties.",
 355         format => $PVE::QemuServer::Memory::memory_fmt
 356     },
 357     balloon => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 361         minimum => 0,
 362     },
 363     shares => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 367             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 368             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 369         minimum => 0,
 370         maximum => 50000,
 371         default => 1000,
 372     },
 373     keyboard => {
 374         optional => 1,
 375         type => 'string',
 376         description => "Keyboard layout for VNC server. This option is generally not required and"
 377         ." is often better handled from within the guest OS.",
 378         enum => PVE::Tools::kvmkeymaplist(),
 379         default => undef,
 380     },
 381     name => {
 382         optional => 1,
 383         type => 'string', format => 'dns-name',
 384         description => "Set a name for the VM. Only used on the configuration web interface.",
 385     },
 386     scsihw => {
 387         optional => 1,
 388         type => 'string',
 389         description => "SCSI controller model",
 390         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 391         default => 'lsi',
 392     },
 393     description => {
 394         optional => 1,
 395         type => 'string',
 396         description => "Description for the VM. Shown in the web-interface VM's summary."
 397             ." This is saved as comment inside the configuration file.",
 398         maxLength => 1024 * 8,
 399     },
 400     ostype => {
 401         optional => 1,
 402         type => 'string',
 403         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 404         description => "Specify guest operating system.",
 405         verbose_description => <<EODESC,
 406 Specify guest operating system. This is used to enable special
 407 optimization/features for specific operating systems:
 408
 409 [horizontal]
 410 other;; unspecified OS
 411 wxp;; Microsoft Windows XP
 412 w2k;; Microsoft Windows 2000
 413 w2k3;; Microsoft Windows 2003
 414 w2k8;; Microsoft Windows 2008
 415 wvista;; Microsoft Windows Vista
 416 win7;; Microsoft Windows 7
 417 win8;; Microsoft Windows 8/2012/2012r2
 418 win10;; Microsoft Windows 10/2016/2019
 419 win11;; Microsoft Windows 11/2022
 420 l24;; Linux 2.4 Kernel
 421 l26;; Linux 2.6 - 6.X Kernel
 422 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 423 EODESC
 424     },
 425     boot => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-boot',
 428         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 429             ." key or 'legacy=' is deprecated.",
 430     },
 431     bootdisk => {
 432         optional => 1,
 433         type => 'string', format => 'pve-qm-bootdisk',
 434         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 435         pattern => '(ide|sata|scsi|virtio)\d+',
 436     },
 437     smp => {
 438         optional => 1,
 439         type => 'integer',
 440         description => "The number of CPUs. Please use option -sockets instead.",
 441         minimum => 1,
 442         default => 1,
 443     },
 444     sockets => {
 445         optional => 1,
 446         type => 'integer',
 447         description => "The number of CPU sockets.",
 448         minimum => 1,
 449         default => 1,
 450     },
 451     cores => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "The number of cores per socket.",
 455         minimum => 1,
 456         default => 1,
 457     },
 458     numa => {
 459         optional => 1,
 460         type => 'boolean',
 461         description => "Enable/disable NUMA.",
 462         default => 0,
 463     },
 464     hugepages => {
 465         optional => 1,
 466         type => 'string',
 467         description => "Enable/disable hugepages memory.",
 468         enum => [qw(any 2 1024)],
 469     },
 470     keephugepages => {
 471         optional => 1,
 472         type => 'boolean',
 473         default => 0,
 474         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 475             ." after VM shutdown and can be used for subsequent starts.",
 476     },
 477     vcpus => {
 478         optional => 1,
 479         type => 'integer',
 480         description => "Number of hotplugged vcpus.",
 481         minimum => 1,
 482         default => 0,
 483     },
 484     acpi => {
 485         optional => 1,
 486         type => 'boolean',
 487         description => "Enable/disable ACPI.",
 488         default => 1,
 489     },
 490     agent => {
 491         optional => 1,
 492         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 493         type => 'string',
 494         format => $agent_fmt,
 495     },
 496     kvm => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable KVM hardware virtualization.",
 500         default => 1,
 501     },
 502     tdf => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Enable/disable time drift fix.",
 506         default => 0,
 507     },
 508     localtime => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 512             ." the `ostype` indicates a Microsoft Windows OS.",
 513     },
 514     freeze => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 518     },
 519     vga => {
 520         optional => 1,
 521         type => 'string', format => $vga_fmt,
 522         description => "Configure the VGA hardware.",
 523         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 524             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 525             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 526             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 527             ." display server. For win* OS you can select how many independent displays you want,"
 528             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 529             ." using a serial device as terminal.",
 530     },
 531     watchdog => {
 532         optional => 1,
 533         type => 'string', format => 'pve-qm-watchdog',
 534         description => "Create a virtual hardware watchdog device.",
 535         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 536             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 537             ." else the watchdog will reset the guest (or execute the respective action specified)",
 538     },
 539     startdate => {
 540         optional => 1,
 541         type => 'string',
 542         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 543         description => "Set the initial date of the real time clock. Valid format for date are:"
 544             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 545         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 546         default => 'now',
 547     },
 548     startup =>  get_standard_option('pve-startup-order'),
 549     template => {
 550         optional => 1,
 551         type => 'boolean',
 552         description => "Enable/disable Template.",
 553         default => 0,
 554     },
 555     args => {
 556         optional => 1,
 557         type => 'string',
 558         description => "Arbitrary arguments passed to kvm.",
 559         verbose_description => <<EODESCR,
 560 Arbitrary arguments passed to kvm, for example:
 561
 562 args: -no-reboot -smbios 'type=0,vendor=FOO'
 563
 564 NOTE: this option is for experts only.
 565 EODESCR
 566     },
 567     tablet => {
 568         optional => 1,
 569         type => 'boolean',
 570         default => 1,
 571         description => "Enable/disable the USB tablet device.",
 572         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 573             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 574             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 575             ." may consider disabling this to save some context switches. This is turned off by"
 576             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 577     },
 578     migrate_speed => {
 579         optional => 1,
 580         type => 'integer',
 581         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 582         minimum => 0,
 583         default => 0,
 584     },
 585     migrate_downtime => {
 586         optional => 1,
 587         type => 'number',
 588         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 589         minimum => 0,
 590         default => 0.1,
 591     },
 592     cdrom => {
 593         optional => 1,
 594         type => 'string', format => 'pve-qm-ide',
 595         typetext => '<volume>',
 596         description => "This is an alias for option -ide2",
 597     },
 598     cpu => {
 599         optional => 1,
 600         description => "Emulated CPU type.",
 601         type => 'string',
 602         format => 'pve-vm-cpu-conf',
 603     },
 604     parent => get_standard_option('pve-snapshot-name', {
 605         optional => 1,
 606         description => "Parent snapshot name. This is used internally, and should not be modified.",
 607     }),
 608     snaptime => {
 609         optional => 1,
 610         description => "Timestamp for snapshots.",
 611         type => 'integer',
 612         minimum => 0,
 613     },
 614     vmstate => {
 615         optional => 1,
 616         type => 'string', format => 'pve-volume-id',
 617         description => "Reference to a volume which stores the VM state. This is used internally"
 618             ." for snapshots.",
 619     },
 620     vmstatestorage => get_standard_option('pve-storage-id', {
 621         description => "Default storage for VM state volumes/files.",
 622         optional => 1,
 623     }),
 624     runningmachine => get_standard_option('pve-qemu-machine', {
 625         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 626             ." for snapshots.",
 627     }),
 628     runningcpu => {
 629         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 630             ." internally for snapshots.",
 631         optional => 1,
 632         type => 'string',
 633         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 634         format_description => 'QEMU -cpu parameter'
 635     },
 636     machine => get_standard_option('pve-qemu-machine'),
 637     arch => {
 638         description => "Virtual processor architecture. Defaults to the host.",
 639         optional => 1,
 640         type => 'string',
 641         enum => [qw(x86_64 aarch64)],
 642     },
 643     smbios1 => {
 644         description => "Specify SMBIOS type 1 fields.",
 645         type => 'string', format => 'pve-qm-smbios1',
 646         maxLength => 512,
 647         optional => 1,
 648     },
 649     protection => {
 650         optional => 1,
 651         type => 'boolean',
 652         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 653             ." remove disk operations.",
 654         default => 0,
 655     },
 656     bios => {
 657         optional => 1,
 658         type => 'string',
 659         enum => [ qw(seabios ovmf) ],
 660         description => "Select BIOS implementation.",
 661         default => 'seabios',
 662     },
 663     vmgenid => {
 664         type => 'string',
 665         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 666         format_description => 'UUID',
 667         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 668             ." to disable explicitly.",
 669         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 670             ." value identifier to the guest OS. This allows to notify the guest operating system"
 671             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 672             ." execution or creation from a template). The guest operating system notices the"
 673             ." change, and is then able to react as appropriate by marking its copies of"
 674             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 675             ."Note that auto-creation only works when done through API/CLI create or update methods"
 676             .", but not when manually editing the config file.",
 677         default => "1 (autogenerated)",
 678         optional => 1,
 679     },
 680     hookscript => {
 681         type => 'string',
 682         format => 'pve-volume-id',
 683         optional => 1,
 684         description => "Script that will be executed during various steps in the vms lifetime.",
 685     },
 686     ivshmem => {
 687         type => 'string',
 688         format => $ivshmem_fmt,
 689         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 690             ." the host.",
 691         optional => 1,
 692     },
 693     audio0 => {
 694         type => 'string',
 695         format => $audio_fmt,
 696         description => "Configure a audio device, useful in combination with QXL/Spice.",
 697         optional => 1
 698     },
 699     spice_enhancements => {
 700         type => 'string',
 701         format => $spice_enhancements_fmt,
 702         description => "Configure additional enhancements for SPICE.",
 703         optional => 1
 704     },
 705     tags => {
 706         type => 'string', format => 'pve-tag-list',
 707         description => 'Tags of the VM. This is only meta information.',
 708         optional => 1,
 709     },
 710     rng0 => {
 711         type => 'string',
 712         format => $rng_fmt,
 713         description => "Configure a VirtIO-based Random Number Generator.",
 714         optional => 1,
 715     },
 716     meta => {
 717         type => 'string',
 718         format => $meta_info_fmt,
 719         description => "Some (read-only) meta-information about this guest.",
 720         optional => 1,
 721     },
 722     affinity => {
 723         type => 'string', format => 'pve-cpuset',
 724         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 725         optional => 1,
 726     },
 727 };
 728
 729 my $cicustom_fmt = {
 730     meta => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all meta data passed to the VM via"
 734             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     network => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 742         format => 'pve-volume-id',
 743         format_description => 'volume',
 744     },
 745     user => {
 746         type => 'string',
 747         optional => 1,
 748         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 749         format => 'pve-volume-id',
 750         format_description => 'volume',
 751     },
 752     vendor => {
 753         type => 'string',
 754         optional => 1,
 755         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 756         format => 'pve-volume-id',
 757         format_description => 'volume',
 758     },
 759 };
 760 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 761
 762 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 763 my $confdesc_cloudinit = {
 764     citype => {
 765         optional => 1,
 766         type => 'string',
 767         description => 'Specifies the cloud-init configuration format. The default depends on the'
 768             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 769             .' and `configdrive2` for windows.',
 770         enum => ['configdrive2', 'nocloud', 'opennebula'],
 771     },
 772     ciuser => {
 773         optional => 1,
 774         type => 'string',
 775         description => "cloud-init: User name to change ssh keys and password for instead of the"
 776             ." image's configured default user.",
 777     },
 778     cipassword => {
 779         optional => 1,
 780         type => 'string',
 781         description => 'cloud-init: Password to assign the user. Using this is generally not'
 782             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 783             .' support hashed passwords.',
 784     },
 785     ciupgrade => {
 786         optional => 1,
 787         type => 'boolean',
 788         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 789         default => 1,
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.',
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.',
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_NETS = 32;
 842 my $MAX_SERIAL_PORTS = 4;
 843 my $MAX_PARALLEL_PORTS = 3;
 844
 845 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 846     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 847 }
 848
 849 my $nic_model_list = [
 850     'e1000',
 851     'e1000-82540em',
 852     'e1000-82544gc',
 853     'e1000-82545em',
 854     'e1000e',
 855     'i82551',
 856     'i82557b',
 857     'i82559er',
 858     'ne2k_isa',
 859     'ne2k_pci',
 860     'pcnet',
 861     'rtl8139',
 862     'virtio',
 863     'vmxnet3',
 864 ];
 865 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 866
 867 my $net_fmt_bridge_descr = <<__EOD__;
 868 Bridge to attach the network device to. The Proxmox VE standard bridge
 869 is called 'vmbr0'.
 870
 871 If you do not specify a bridge, we create a kvm user (NATed) network
 872 device, which provides DHCP and DNS services. The following addresses
 873 are used:
 874
 875  10.0.2.2   Gateway
 876  10.0.2.3   DNS Server
 877  10.0.2.4   SMB Server
 878
 879 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 880 __EOD__
 881
 882 my $net_fmt = {
 883     macaddr  => get_standard_option('mac-addr', {
 884         description => "MAC address. That address must be unique withing your network. This is"
 885             ." automatically generated if not specified.",
 886     }),
 887     model => {
 888         type => 'string',
 889         description => "Network Card Model. The 'virtio' model provides the best performance with"
 890             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 891             ." best to use 'e1000'.",
 892         enum => $nic_model_list,
 893         default_key => 1,
 894     },
 895     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 896     bridge => get_standard_option('pve-bridge-id', {
 897         description => $net_fmt_bridge_descr,
 898         optional => 1,
 899     }),
 900     queues => {
 901         type => 'integer',
 902         minimum => 0, maximum => 64,
 903         description => 'Number of packet queues to be used on the device.',
 904         optional => 1,
 905     },
 906     rate => {
 907         type => 'number',
 908         minimum => 0,
 909         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 910         optional => 1,
 911     },
 912     tag => {
 913         type => 'integer',
 914         minimum => 1, maximum => 4094,
 915         description => 'VLAN tag to apply to packets on this interface.',
 916         optional => 1,
 917     },
 918     trunks => {
 919         type => 'string',
 920         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 921         description => 'VLAN trunks to pass through this interface.',
 922         format_description => 'vlanid[;vlanid...]',
 923         optional => 1,
 924     },
 925     firewall => {
 926         type => 'boolean',
 927         description => 'Whether this interface should be protected by the firewall.',
 928         optional => 1,
 929     },
 930     link_down => {
 931         type => 'boolean',
 932         description => 'Whether this interface should be disconnected (like pulling the plug).',
 933         optional => 1,
 934     },
 935     mtu => {
 936         type => 'integer',
 937         minimum => 1, maximum => 65520,
 938         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 939         optional => 1,
 940     },
 941 };
 942
 943 my $netdesc = {
 944     optional => 1,
 945     type => 'string', format => $net_fmt,
 946     description => "Specify network devices.",
 947 };
 948
 949 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 950
 951 my $ipconfig_fmt = {
 952     ip => {
 953         type => 'string',
 954         format => 'pve-ipv4-config',
 955         format_description => 'IPv4Format/CIDR',
 956         description => 'IPv4 address in CIDR format.',
 957         optional => 1,
 958         default => 'dhcp',
 959     },
 960     gw => {
 961         type => 'string',
 962         format => 'ipv4',
 963         format_description => 'GatewayIPv4',
 964         description => 'Default gateway for IPv4 traffic.',
 965         optional => 1,
 966         requires => 'ip',
 967     },
 968     ip6 => {
 969         type => 'string',
 970         format => 'pve-ipv6-config',
 971         format_description => 'IPv6Format/CIDR',
 972         description => 'IPv6 address in CIDR format.',
 973         optional => 1,
 974         default => 'dhcp',
 975     },
 976     gw6 => {
 977         type => 'string',
 978         format => 'ipv6',
 979         format_description => 'GatewayIPv6',
 980         description => 'Default gateway for IPv6 traffic.',
 981         optional => 1,
 982         requires => 'ip6',
 983     },
 984 };
 985 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 986 my $ipconfigdesc = {
 987     optional => 1,
 988     type => 'string', format => 'pve-qm-ipconfig',
 989     description => <<'EODESCR',
 990 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 991
 992 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 993
 994 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 995 gateway should be provided.
 996 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 997 cloud-init 19.4 or newer.
 998
 999 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1000 dhcp on IPv4.
1001 EODESCR
1002 };
1003 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1004
1005 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1006     $confdesc->{"net$i"} = $netdesc;
1007     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1008 }
1009
1010 foreach my $key (keys %$confdesc_cloudinit) {
1011     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1012 }
1013
1014 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1015 sub pve_verify_cpuset {
1016     my ($set_text, $noerr) = @_;
1017
1018     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1019
1020     if ($@) {
1021         return if $noerr;
1022         die "unable to parse cpuset option\n";
1023     }
1024
1025     return PVE::CpuSet->new($members)->short_string();
1026 }
1027
1028 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1029 sub verify_volume_id_or_qm_path {
1030     my ($volid, $noerr) = @_;
1031
1032     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1033
1034     return verify_volume_id_or_absolute_path($volid, $noerr);
1035 }
1036
1037 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1038 sub verify_volume_id_or_absolute_path {
1039     my ($volid, $noerr) = @_;
1040
1041     return $volid if $volid =~ m|^/|;
1042
1043     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1044     if ($@) {
1045         return if $noerr;
1046         die $@;
1047     }
1048     return $volid;
1049 }
1050
1051 my $serialdesc = {
1052         optional => 1,
1053         type => 'string',
1054         pattern => '(/dev/.+|socket)',
1055         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1056         verbose_description =>  <<EODESCR,
1057 Create a serial device inside the VM (n is 0 to 3), and pass through a
1058 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1059 host side (use 'qm terminal' to open a terminal connection).
1060
1061 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1062 use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 my $paralleldesc= {
1069         optional => 1,
1070         type => 'string',
1071         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1072         description =>  "Map host parallel devices (n is 0 to 2).",
1073         verbose_description =>  <<EODESCR,
1074 Map host parallel devices (n is 0 to 2).
1075
1076 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1077 machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082
1083 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1084     $confdesc->{"parallel$i"} = $paralleldesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1088     $confdesc->{"serial$i"} = $serialdesc;
1089 }
1090
1091 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1092     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1093 }
1094
1095 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1096     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1097 }
1098
1099 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1100     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1101 }
1102
1103 my $boot_fmt = {
1104     legacy => {
1105         optional => 1,
1106         default_key => 1,
1107         type => 'string',
1108         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1109                      . " Deprecated, use 'order=' instead.",
1110         pattern => '[acdn]{1,4}',
1111         format_description => "[acdn]{1,4}",
1112
1113         # note: this is also the fallback if boot: is not given at all
1114         default => 'cdn',
1115     },
1116     order => {
1117         optional => 1,
1118         type => 'string',
1119         format => 'pve-qm-bootdev-list',
1120         format_description => "device[;device...]",
1121         description => <<EODESC,
1122 The guest will attempt to boot from devices in the order they appear here.
1123
1124 Disks, optical drives and passed-through storage USB devices will be directly
1125 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1126 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1127
1128 Note that only devices in this list will be marked as bootable and thus loaded
1129 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1130 (e.g. software-raid), you need to specify all of them here.
1131
1132 Overrides the deprecated 'legacy=[acdn]*' value when given.
1133 EODESC
1134     },
1135 };
1136 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1137
1138 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1139 sub verify_bootdev {
1140     my ($dev, $noerr) = @_;
1141
1142     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1143     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1144
1145     my $check = sub {
1146         my ($base) = @_;
1147         return 0 if $dev !~ m/^$base\d+$/;
1148         return 0 if !$confdesc->{$dev};
1149         return 1;
1150     };
1151
1152     return $dev if $check->("net");
1153     return $dev if $check->("usb");
1154     return $dev if $check->("hostpci");
1155
1156     return if $noerr;
1157     die "invalid boot device '$dev'\n";
1158 }
1159
1160 sub print_bootorder {
1161     my ($devs) = @_;
1162     return "" if !@$devs;
1163     my $data = { order => join(';', @$devs) };
1164     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1165 }
1166
1167 my $kvm_api_version = 0;
1168
1169 sub kvm_version {
1170     return $kvm_api_version if $kvm_api_version;
1171
1172     open my $fh, '<', '/dev/kvm' or return;
1173
1174     # 0xae00 => KVM_GET_API_VERSION
1175     $kvm_api_version = ioctl($fh, 0xae00, 0);
1176     close($fh);
1177
1178     return $kvm_api_version;
1179 }
1180
1181 my $kvm_user_version = {};
1182 my $kvm_mtime = {};
1183
1184 sub kvm_user_version {
1185     my ($binary) = @_;
1186
1187     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1188     my $st = stat($binary);
1189
1190     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1191     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1192         $cachedmtime == $st->mtime;
1193
1194     $kvm_user_version->{$binary} = 'unknown';
1195     $kvm_mtime->{$binary} = $st->mtime;
1196
1197     my $code = sub {
1198         my $line = shift;
1199         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1200             $kvm_user_version->{$binary} = $2;
1201         }
1202     };
1203
1204     eval { run_command([$binary, '--version'], outfunc => $code); };
1205     warn $@ if $@;
1206
1207     return $kvm_user_version->{$binary};
1208
1209 }
1210 my sub extract_version {
1211     my ($machine_type, $version) = @_;
1212     $version = kvm_user_version() if !defined($version);
1213     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1214 }
1215
1216 sub kernel_has_vhost_net {
1217     return -c '/dev/vhost-net';
1218 }
1219
1220 sub option_exists {
1221     my $key = shift;
1222     return defined($confdesc->{$key});
1223 }
1224
1225 my $cdrom_path;
1226 sub get_cdrom_path {
1227
1228     return $cdrom_path if defined($cdrom_path);
1229
1230     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1231
1232     if (!defined($cdrom_path)) {
1233         log_warn("no physical CD-ROM available, ignoring");
1234         $cdrom_path = '';
1235     }
1236
1237     return $cdrom_path;
1238 }
1239
1240 sub get_iso_path {
1241     my ($storecfg, $vmid, $cdrom) = @_;
1242
1243     if ($cdrom eq 'cdrom') {
1244         return get_cdrom_path();
1245     } elsif ($cdrom eq 'none') {
1246         return '';
1247     } elsif ($cdrom =~ m|^/|) {
1248         return $cdrom;
1249     } else {
1250         return PVE::Storage::path($storecfg, $cdrom);
1251     }
1252 }
1253
1254 # try to convert old style file names to volume IDs
1255 sub filename_to_volume_id {
1256     my ($vmid, $file, $media) = @_;
1257
1258      if (!($file eq 'none' || $file eq 'cdrom' ||
1259           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1260
1261         return if $file =~ m|/|;
1262
1263         if ($media && $media eq 'cdrom') {
1264             $file = "local:iso/$file";
1265         } else {
1266             $file = "local:$vmid/$file";
1267         }
1268     }
1269
1270     return $file;
1271 }
1272
1273 sub verify_media_type {
1274     my ($opt, $vtype, $media) = @_;
1275
1276     return if !$media;
1277
1278     my $etype;
1279     if ($media eq 'disk') {
1280         $etype = 'images';
1281     } elsif ($media eq 'cdrom') {
1282         $etype = 'iso';
1283     } else {
1284         die "internal error";
1285     }
1286
1287     return if ($vtype eq $etype);
1288
1289     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1290 }
1291
1292 sub cleanup_drive_path {
1293     my ($opt, $storecfg, $drive) = @_;
1294
1295     # try to convert filesystem paths to volume IDs
1296
1297     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1298         ($drive->{file} !~ m|^/dev/.+|) &&
1299         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1300         ($drive->{file} !~ m/^\d+$/)) {
1301         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1302         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1303             if !$vtype;
1304         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1305         verify_media_type($opt, $vtype, $drive->{media});
1306         $drive->{file} = $volid;
1307     }
1308
1309     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1310 }
1311
1312 sub parse_hotplug_features {
1313     my ($data) = @_;
1314
1315     my $res = {};
1316
1317     return $res if $data eq '0';
1318
1319     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1320
1321     foreach my $feature (PVE::Tools::split_list($data)) {
1322         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1323             $res->{$1} = 1;
1324         } else {
1325             die "invalid hotplug feature '$feature'\n";
1326         }
1327     }
1328     return $res;
1329 }
1330
1331 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1332 sub pve_verify_hotplug_features {
1333     my ($value, $noerr) = @_;
1334
1335     return $value if parse_hotplug_features($value);
1336
1337     return if $noerr;
1338
1339     die "unable to parse hotplug option\n";
1340 }
1341
1342 sub scsi_inquiry {
1343     my($fh, $noerr) = @_;
1344
1345     my $SG_IO = 0x2285;
1346     my $SG_GET_VERSION_NUM = 0x2282;
1347
1348     my $versionbuf = "\x00" x 8;
1349     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1350     if (!$ret) {
1351         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1352         return;
1353     }
1354     my $version = unpack("I", $versionbuf);
1355     if ($version < 30000) {
1356         die "scsi generic interface too old\n"  if !$noerr;
1357         return;
1358     }
1359
1360     my $buf = "\x00" x 36;
1361     my $sensebuf = "\x00" x 8;
1362     my $cmd = pack("C x3 C x1", 0x12, 36);
1363
1364     # see /usr/include/scsi/sg.h
1365     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1366
1367     my $packet = pack(
1368         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1369     );
1370
1371     $ret = ioctl($fh, $SG_IO, $packet);
1372     if (!$ret) {
1373         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1374         return;
1375     }
1376
1377     my @res = unpack($sg_io_hdr_t, $packet);
1378     if ($res[17] || $res[18]) {
1379         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1380         return;
1381     }
1382
1383     my $res = {};
1384     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1385
1386     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1387     $res->{type} &= 0x1F;
1388
1389     return $res;
1390 }
1391
1392 sub path_is_scsi {
1393     my ($path) = @_;
1394
1395     my $fh = IO::File->new("+<$path") || return;
1396     my $res = scsi_inquiry($fh, 1);
1397     close($fh);
1398
1399     return $res;
1400 }
1401
1402 sub print_tabletdevice_full {
1403     my ($conf, $arch) = @_;
1404
1405     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1406
1407     # we use uhci for old VMs because tablet driver was buggy in older qemu
1408     my $usbbus;
1409     if ($q35 || $arch eq 'aarch64') {
1410         $usbbus = 'ehci';
1411     } else {
1412         $usbbus = 'uhci';
1413     }
1414
1415     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1416 }
1417
1418 sub print_keyboarddevice_full {
1419     my ($conf, $arch) = @_;
1420
1421     return if $arch ne 'aarch64';
1422
1423     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1424 }
1425
1426 my sub get_drive_id {
1427     my ($drive) = @_;
1428     return "$drive->{interface}$drive->{index}";
1429 }
1430
1431 sub print_drivedevice_full {
1432     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1433
1434     my $device = '';
1435     my $maxdev = 0;
1436
1437     my $drive_id = get_drive_id($drive);
1438     if ($drive->{interface} eq 'virtio') {
1439         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1440         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1441         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1442     } elsif ($drive->{interface} eq 'scsi') {
1443
1444         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1445         my $unit = $drive->{index} % $maxdev;
1446         my $devicetype = 'hd';
1447         my $path = '';
1448         if (drive_is_cdrom($drive)) {
1449             $devicetype = 'cd';
1450         } else {
1451             if ($drive->{file} =~ m|^/|) {
1452                 $path = $drive->{file};
1453                 if (my $info = path_is_scsi($path)) {
1454                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1455                         $devicetype = 'block';
1456                     } elsif ($info->{type} == 1) { # tape
1457                         $devicetype = 'generic';
1458                     }
1459                 }
1460             } else {
1461                  $path = PVE::Storage::path($storecfg, $drive->{file});
1462             }
1463
1464             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1465             my $version = extract_version($machine_type, kvm_user_version());
1466             if ($path =~ m/^iscsi\:\/\// &&
1467                !min_version($version, 4, 1)) {
1468                 $devicetype = 'generic';
1469             }
1470         }
1471
1472         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1473             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1474         } else {
1475             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1476                 .",lun=$drive->{index}";
1477         }
1478         $device .= ",drive=drive-$drive_id,id=$drive_id";
1479
1480         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1481             $device .= ",rotation_rate=1";
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484
1485     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1486         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1487         my $controller = int($drive->{index} / $maxdev);
1488         my $unit = $drive->{index} % $maxdev;
1489
1490         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1491         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1492         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1493         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1494         # were before. Move odd ones up by 2 where they don't clash.
1495         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1496             $controller += 2 * ($unit % 2);
1497             $unit = 0;
1498         }
1499
1500         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1501
1502         $device = "ide-$devicetype";
1503         if ($drive->{interface} eq 'ide') {
1504             $device .= ",bus=ide.$controller,unit=$unit";
1505         } else {
1506             $device .= ",bus=ahci$controller.$unit";
1507         }
1508         $device .= ",drive=drive-$drive_id,id=$drive_id";
1509
1510         if ($devicetype eq 'hd') {
1511             if (my $model = $drive->{model}) {
1512                 $model = URI::Escape::uri_unescape($model);
1513                 $device .= ",model=$model";
1514             }
1515             if ($drive->{ssd}) {
1516                 $device .= ",rotation_rate=1";
1517             }
1518         }
1519         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1520     } elsif ($drive->{interface} eq 'usb') {
1521         die "implement me";
1522         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1523     } else {
1524         die "unsupported interface type";
1525     }
1526
1527     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1528
1529     if (my $serial = $drive->{serial}) {
1530         $serial = URI::Escape::uri_unescape($serial);
1531         $device .= ",serial=$serial";
1532     }
1533
1534
1535     return $device;
1536 }
1537
1538 sub get_initiator_name {
1539     my $initiator;
1540
1541     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1542     while (defined(my $line = <$fh>)) {
1543         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1544         $initiator = $1;
1545         last;
1546     }
1547     $fh->close();
1548
1549     return $initiator;
1550 }
1551
1552 my sub storage_allows_io_uring_default {
1553     my ($scfg, $cache_direct) = @_;
1554
1555     # io_uring with cache mode writeback or writethrough on krbd will hang...
1556     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1557
1558     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1559     # sometimes, just plain disable...
1560     return if $scfg && $scfg->{type} eq 'lvm';
1561
1562     # io_uring causes problems when used with CIFS since kernel 5.15
1563     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1564     return if $scfg && $scfg->{type} eq 'cifs';
1565
1566     return 1;
1567 }
1568
1569 my sub drive_uses_cache_direct {
1570     my ($drive, $scfg) = @_;
1571
1572     my $cache_direct = 0;
1573
1574     if (my $cache = $drive->{cache}) {
1575         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1576     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1577         $cache_direct = 1;
1578     }
1579
1580     return $cache_direct;
1581 }
1582
1583 sub print_drive_commandline_full {
1584     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1585
1586     my $path;
1587     my $volid = $drive->{file};
1588     my $format = $drive->{format};
1589     my $drive_id = get_drive_id($drive);
1590
1591     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1592     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1593
1594     if (drive_is_cdrom($drive)) {
1595         $path = get_iso_path($storecfg, $vmid, $volid);
1596         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1597     } else {
1598         if ($storeid) {
1599             $path = PVE::Storage::path($storecfg, $volid);
1600             $format //= qemu_img_format($scfg, $volname);
1601         } else {
1602             $path = $volid;
1603             $format //= "raw";
1604         }
1605    }
1606
1607    my $is_rbd = $path =~ m/^rbd:/;
1608
1609     my $opts = '';
1610     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1611     foreach my $o (@qemu_drive_options) {
1612         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1613     }
1614
1615     # snapshot only accepts on|off
1616     if (defined($drive->{snapshot})) {
1617         my $v = $drive->{snapshot} ? 'on' : 'off';
1618         $opts .= ",snapshot=$v";
1619     }
1620
1621     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1622         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1623     }
1624
1625     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1626         my ($dir, $qmpname) = @$type;
1627         if (my $v = $drive->{"mbps$dir"}) {
1628             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1629         }
1630         if (my $v = $drive->{"mbps${dir}_max"}) {
1631             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1632         }
1633         if (my $v = $drive->{"bps${dir}_max_length"}) {
1634             $opts .= ",throttling.bps$qmpname-max-length=$v";
1635         }
1636         if (my $v = $drive->{"iops${dir}"}) {
1637             $opts .= ",throttling.iops$qmpname=$v";
1638         }
1639         if (my $v = $drive->{"iops${dir}_max"}) {
1640             $opts .= ",throttling.iops$qmpname-max=$v";
1641         }
1642         if (my $v = $drive->{"iops${dir}_max_length"}) {
1643             $opts .= ",throttling.iops$qmpname-max-length=$v";
1644         }
1645     }
1646
1647     if ($pbs_name) {
1648         $format = "rbd" if $is_rbd;
1649         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1650             if !$format;
1651         $opts .= ",format=alloc-track,file.driver=$format";
1652     } elsif ($format) {
1653         $opts .= ",format=$format";
1654     }
1655
1656     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1657
1658     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1659
1660     if (!$drive->{aio}) {
1661         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1662             # io_uring supports all cache modes
1663             $opts .= ",aio=io_uring";
1664         } else {
1665             # aio native works only with O_DIRECT
1666             if($cache_direct) {
1667                 $opts .= ",aio=native";
1668             } else {
1669                 $opts .= ",aio=threads";
1670             }
1671         }
1672     }
1673
1674     if (!drive_is_cdrom($drive)) {
1675         my $detectzeroes;
1676         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1677             $detectzeroes = 'off';
1678         } elsif ($drive->{discard}) {
1679             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1680         } else {
1681             # This used to be our default with discard not being specified:
1682             $detectzeroes = 'on';
1683         }
1684
1685         # note: 'detect-zeroes' works per blockdev and we want it to persist
1686         # after the alloc-track is removed, so put it on 'file' directly
1687         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1688         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1689     }
1690
1691     if ($pbs_name) {
1692         $opts .= ",backing=$pbs_name";
1693         $opts .= ",auto-remove=on";
1694     }
1695
1696     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1697     my $file_param = "file";
1698     if ($pbs_name) {
1699         # non-rbd drivers require the underlying file to be a seperate block
1700         # node, so add a second .file indirection
1701         $file_param .= ".file" if !$is_rbd;
1702         $file_param .= ".filename";
1703     }
1704     my $pathinfo = $path ? "$file_param=$path," : '';
1705
1706     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1707 }
1708
1709 sub print_pbs_blockdev {
1710     my ($pbs_conf, $pbs_name) = @_;
1711     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1712     $blockdev .= ",repository=$pbs_conf->{repository}";
1713     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1714     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1715     $blockdev .= ",archive=$pbs_conf->{archive}";
1716     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1717     return $blockdev;
1718 }
1719
1720 sub print_netdevice_full {
1721     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1722
1723     my $device = $net->{model};
1724     if ($net->{model} eq 'virtio') {
1725          $device = 'virtio-net-pci';
1726      };
1727
1728     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1729     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1730     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1731         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1732         # and out of each queue plus one config interrupt and control vector queue
1733         my $vectors = $net->{queues} * 2 + 2;
1734         $tmpstr .= ",vectors=$vectors,mq=on";
1735         if (min_version($machine_version, 7, 1)) {
1736             $tmpstr .= ",packed=on";
1737         }
1738     }
1739
1740     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1741         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1742     }
1743
1744     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1745
1746     if (my $mtu = $net->{mtu}) {
1747         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1748             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1749             if ($mtu == 1) {
1750                  $mtu = $bridge_mtu;
1751             } elsif ($mtu < 576) {
1752                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1753             } elsif ($mtu > $bridge_mtu) {
1754                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1755             }
1756             $tmpstr .= ",host_mtu=$mtu";
1757         } else {
1758             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1759         }
1760     }
1761
1762     if ($use_old_bios_files) {
1763         my $romfile;
1764         if ($device eq 'virtio-net-pci') {
1765             $romfile = 'pxe-virtio.rom';
1766         } elsif ($device eq 'e1000') {
1767             $romfile = 'pxe-e1000.rom';
1768         } elsif ($device eq 'e1000e') {
1769             $romfile = 'pxe-e1000e.rom';
1770         } elsif ($device eq 'ne2k') {
1771             $romfile = 'pxe-ne2k_pci.rom';
1772         } elsif ($device eq 'pcnet') {
1773             $romfile = 'pxe-pcnet.rom';
1774         } elsif ($device eq 'rtl8139') {
1775             $romfile = 'pxe-rtl8139.rom';
1776         }
1777         $tmpstr .= ",romfile=$romfile" if $romfile;
1778     }
1779
1780     return $tmpstr;
1781 }
1782
1783 sub print_netdev_full {
1784     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1785
1786     my $i = '';
1787     if ($netid =~ m/^net(\d+)$/) {
1788         $i = int($1);
1789     }
1790
1791     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1792
1793     my $ifname = "tap${vmid}i$i";
1794
1795     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1796     die "interface name '$ifname' is too long (max 15 character)\n"
1797         if length($ifname) >= 16;
1798
1799     my $vhostparam = '';
1800     if (is_native($arch)) {
1801         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1802     }
1803
1804     my $vmname = $conf->{name} || "vm$vmid";
1805
1806     my $netdev = "";
1807     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1808
1809     if ($net->{bridge}) {
1810         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1811             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1812     } else {
1813         $netdev = "type=user,id=$netid,hostname=$vmname";
1814     }
1815
1816     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1817
1818     return $netdev;
1819 }
1820
1821 my $vga_map = {
1822     'cirrus' => 'cirrus-vga',
1823     'std' => 'VGA',
1824     'vmware' => 'vmware-svga',
1825     'virtio' => 'virtio-vga',
1826     'virtio-gl' => 'virtio-vga-gl',
1827 };
1828
1829 sub print_vga_device {
1830     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1831
1832     my $type = $vga_map->{$vga->{type}};
1833     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1834         $type = 'virtio-gpu';
1835     }
1836     my $vgamem_mb = $vga->{memory};
1837
1838     my $max_outputs = '';
1839     if ($qxlnum) {
1840         $type = $id ? 'qxl' : 'qxl-vga';
1841
1842         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1843             # set max outputs so linux can have up to 4 qxl displays with one device
1844             if (min_version($machine_version, 4, 1)) {
1845                 $max_outputs = ",max_outputs=4";
1846             }
1847         }
1848     }
1849
1850     die "no devicetype for $vga->{type}\n" if !$type;
1851
1852     my $memory = "";
1853     if ($vgamem_mb) {
1854         if ($vga->{type} =~ /^virtio/) {
1855             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1856             $memory = ",max_hostmem=$bytes";
1857         } elsif ($qxlnum) {
1858             # from https://www.spice-space.org/multiple-monitors.html
1859             $memory = ",vgamem_mb=$vga->{memory}";
1860             my $ram = $vgamem_mb * 4;
1861             my $vram = $vgamem_mb * 2;
1862             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1863         } else {
1864             $memory = ",vgamem_mb=$vga->{memory}";
1865         }
1866     } elsif ($qxlnum && $id) {
1867         $memory = ",ram_size=67108864,vram_size=33554432";
1868     }
1869
1870     my $edidoff = "";
1871     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1872         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1873     }
1874
1875     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1876     my $vgaid = "vga" . ($id // '');
1877     my $pciaddr;
1878     if ($q35 && $vgaid eq 'vga') {
1879         # the first display uses pcie.0 bus on q35 machines
1880         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1881     } else {
1882         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1883     }
1884
1885     if ($vga->{type} eq 'virtio-gl') {
1886         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1887         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1888             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1889
1890         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1891             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1892     }
1893
1894     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1895 }
1896
1897 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1898 sub parse_net {
1899     my ($data, $disable_mac_autogen) = @_;
1900
1901     my $res = eval { parse_property_string($net_fmt, $data) };
1902     if ($@) {
1903         warn $@;
1904         return;
1905     }
1906     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1907         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1908         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1909     }
1910     return $res;
1911 }
1912
1913 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1914 sub parse_ipconfig {
1915     my ($data) = @_;
1916
1917     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1918     if ($@) {
1919         warn $@;
1920         return;
1921     }
1922
1923     if ($res->{gw} && !$res->{ip}) {
1924         warn 'gateway specified without specifying an IP address';
1925         return;
1926     }
1927     if ($res->{gw6} && !$res->{ip6}) {
1928         warn 'IPv6 gateway specified without specifying an IPv6 address';
1929         return;
1930     }
1931     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1932         warn 'gateway specified together with DHCP';
1933         return;
1934     }
1935     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1936         # gw6 + auto/dhcp
1937         warn "IPv6 gateway specified together with $res->{ip6} address";
1938         return;
1939     }
1940
1941     if (!$res->{ip} && !$res->{ip6}) {
1942         return { ip => 'dhcp', ip6 => 'dhcp' };
1943     }
1944
1945     return $res;
1946 }
1947
1948 sub print_net {
1949     my $net = shift;
1950
1951     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1952 }
1953
1954 sub add_random_macs {
1955     my ($settings) = @_;
1956
1957     foreach my $opt (keys %$settings) {
1958         next if $opt !~ m/^net(\d+)$/;
1959         my $net = parse_net($settings->{$opt});
1960         next if !$net;
1961         $settings->{$opt} = print_net($net);
1962     }
1963 }
1964
1965 sub vm_is_volid_owner {
1966     my ($storecfg, $vmid, $volid) = @_;
1967
1968     if ($volid !~  m|^/|) {
1969         my ($path, $owner);
1970         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1971         if ($owner && ($owner == $vmid)) {
1972             return 1;
1973         }
1974     }
1975
1976     return;
1977 }
1978
1979 sub vmconfig_register_unused_drive {
1980     my ($storecfg, $vmid, $conf, $drive) = @_;
1981
1982     if (drive_is_cloudinit($drive)) {
1983         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1984         warn $@ if $@;
1985         delete $conf->{cloudinit};
1986     } elsif (!drive_is_cdrom($drive)) {
1987         my $volid = $drive->{file};
1988         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1989             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1990         }
1991     }
1992 }
1993
1994 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1995 my $smbios1_fmt = {
1996     uuid => {
1997         type => 'string',
1998         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1999         format_description => 'UUID',
2000         description => "Set SMBIOS1 UUID.",
2001         optional => 1,
2002     },
2003     version => {
2004         type => 'string',
2005         pattern => '[A-Za-z0-9+\/]+={0,2}',
2006         format_description => 'Base64 encoded string',
2007         description => "Set SMBIOS1 version.",
2008         optional => 1,
2009     },
2010     serial => {
2011         type => 'string',
2012         pattern => '[A-Za-z0-9+\/]+={0,2}',
2013         format_description => 'Base64 encoded string',
2014         description => "Set SMBIOS1 serial number.",
2015         optional => 1,
2016     },
2017     manufacturer => {
2018         type => 'string',
2019         pattern => '[A-Za-z0-9+\/]+={0,2}',
2020         format_description => 'Base64 encoded string',
2021         description => "Set SMBIOS1 manufacturer.",
2022         optional => 1,
2023     },
2024     product => {
2025         type => 'string',
2026         pattern => '[A-Za-z0-9+\/]+={0,2}',
2027         format_description => 'Base64 encoded string',
2028         description => "Set SMBIOS1 product ID.",
2029         optional => 1,
2030     },
2031     sku => {
2032         type => 'string',
2033         pattern => '[A-Za-z0-9+\/]+={0,2}',
2034         format_description => 'Base64 encoded string',
2035         description => "Set SMBIOS1 SKU string.",
2036         optional => 1,
2037     },
2038     family => {
2039         type => 'string',
2040         pattern => '[A-Za-z0-9+\/]+={0,2}',
2041         format_description => 'Base64 encoded string',
2042         description => "Set SMBIOS1 family string.",
2043         optional => 1,
2044     },
2045     base64 => {
2046         type => 'boolean',
2047         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2048         optional => 1,
2049     },
2050 };
2051
2052 sub parse_smbios1 {
2053     my ($data) = @_;
2054
2055     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2056     warn $@ if $@;
2057     return $res;
2058 }
2059
2060 sub print_smbios1 {
2061     my ($smbios1) = @_;
2062     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2063 }
2064
2065 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2066
2067 sub parse_watchdog {
2068     my ($value) = @_;
2069
2070     return if !$value;
2071
2072     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2073     warn $@ if $@;
2074     return $res;
2075 }
2076
2077 sub parse_guest_agent {
2078     my ($conf) = @_;
2079
2080     return {} if !defined($conf->{agent});
2081
2082     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2083     warn $@ if $@;
2084
2085     # if the agent is disabled ignore the other potentially set properties
2086     return {} if !$res->{enabled};
2087     return $res;
2088 }
2089
2090 sub get_qga_key {
2091     my ($conf, $key) = @_;
2092     return undef if !defined($conf->{agent});
2093
2094     my $agent = parse_guest_agent($conf);
2095     return $agent->{$key};
2096 }
2097
2098 sub parse_vga {
2099     my ($value) = @_;
2100
2101     return {} if !$value;
2102     my $res = eval { parse_property_string($vga_fmt, $value) };
2103     warn $@ if $@;
2104     return $res;
2105 }
2106
2107 sub parse_rng {
2108     my ($value) = @_;
2109
2110     return if !$value;
2111
2112     my $res = eval { parse_property_string($rng_fmt, $value) };
2113     warn $@ if $@;
2114     return $res;
2115 }
2116
2117 sub parse_meta_info {
2118     my ($value) = @_;
2119
2120     return if !$value;
2121
2122     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub new_meta_info_string {
2128     my () = @_; # for now do not allow to override any value
2129
2130     return PVE::JSONSchema::print_property_string(
2131         {
2132             'creation-qemu' => kvm_user_version(),
2133             ctime => "". int(time()),
2134         },
2135         $meta_info_fmt
2136     );
2137 }
2138
2139 sub qemu_created_version_fixups {
2140     my ($conf, $forcemachine, $kvmver) = @_;
2141
2142     my $meta = parse_meta_info($conf->{meta}) // {};
2143     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2144
2145     # check if we need to apply some handling for VMs that always use the latest machine version but
2146     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2147     # would be required (we do not want to pin machine version for non-windows OS type)
2148     if (
2149         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2150         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2151         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2152         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2153     ) {
2154         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2155         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2156             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2157             # and thus with the predictable interface naming of systemd
2158             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2159         }
2160     }
2161     return;
2162 }
2163
2164 # add JSON properties for create and set function
2165 sub json_config_properties {
2166     my ($prop, $with_disk_alloc) = @_;
2167
2168     my $skip_json_config_opts = {
2169         parent => 1,
2170         snaptime => 1,
2171         vmstate => 1,
2172         runningmachine => 1,
2173         runningcpu => 1,
2174         meta => 1,
2175     };
2176
2177     foreach my $opt (keys %$confdesc) {
2178         next if $skip_json_config_opts->{$opt};
2179
2180         if ($with_disk_alloc && is_valid_drivename($opt)) {
2181             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2182         } else {
2183             $prop->{$opt} = $confdesc->{$opt};
2184         }
2185     }
2186
2187     return $prop;
2188 }
2189
2190 # Properties that we can read from an OVF file
2191 sub json_ovf_properties {
2192     my $prop = {};
2193
2194     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2195         $prop->{$device} = {
2196             type => 'string',
2197             format => 'pve-volume-id-or-absolute-path',
2198             description => "Disk image that gets imported to $device",
2199             optional => 1,
2200         };
2201     }
2202
2203     $prop->{cores} = {
2204         type => 'integer',
2205         description => "The number of CPU cores.",
2206         optional => 1,
2207     };
2208     $prop->{memory} = {
2209         type => 'integer',
2210         description => "Amount of RAM for the VM in MB.",
2211         optional => 1,
2212     };
2213     $prop->{name} = {
2214         type => 'string',
2215         description => "Name of the VM.",
2216         optional => 1,
2217     };
2218
2219     return $prop;
2220 }
2221
2222 # return copy of $confdesc_cloudinit to generate documentation
2223 sub cloudinit_config_properties {
2224
2225     return dclone($confdesc_cloudinit);
2226 }
2227
2228 sub cloudinit_pending_properties {
2229     my $p = {
2230         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2231         name => 1,
2232     };
2233     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2234     return $p;
2235 }
2236
2237 sub check_type {
2238     my ($key, $value) = @_;
2239
2240     die "unknown setting '$key'\n" if !$confdesc->{$key};
2241
2242     my $type = $confdesc->{$key}->{type};
2243
2244     if (!defined($value)) {
2245         die "got undefined value\n";
2246     }
2247
2248     if ($value =~ m/[\n\r]/) {
2249         die "property contains a line feed\n";
2250     }
2251
2252     if ($type eq 'boolean') {
2253         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2254         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2255         die "type check ('boolean') failed - got '$value'\n";
2256     } elsif ($type eq 'integer') {
2257         return int($1) if $value =~ m/^(\d+)$/;
2258         die "type check ('integer') failed - got '$value'\n";
2259     } elsif ($type eq 'number') {
2260         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2261         die "type check ('number') failed - got '$value'\n";
2262     } elsif ($type eq 'string') {
2263         if (my $fmt = $confdesc->{$key}->{format}) {
2264             PVE::JSONSchema::check_format($fmt, $value);
2265             return $value;
2266         }
2267         $value =~ s/^\"(.*)\"$/$1/;
2268         return $value;
2269     } else {
2270         die "internal error"
2271     }
2272 }
2273
2274 sub destroy_vm {
2275     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2276
2277     my $conf = PVE::QemuConfig->load_config($vmid);
2278
2279     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2280         PVE::QemuConfig->check_lock($conf);
2281     }
2282
2283     if ($conf->{template}) {
2284         # check if any base image is still used by a linked clone
2285         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2286                 my ($ds, $drive) = @_;
2287                 return if drive_is_cdrom($drive);
2288
2289                 my $volid = $drive->{file};
2290                 return if !$volid || $volid =~ m|^/|;
2291
2292                 die "base volume '$volid' is still in use by linked cloned\n"
2293                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2294
2295         });
2296     }
2297
2298     my $volids = {};
2299     my $remove_owned_drive = sub {
2300         my ($ds, $drive) = @_;
2301         return if drive_is_cdrom($drive, 1);
2302
2303         my $volid = $drive->{file};
2304         return if !$volid || $volid =~ m|^/|;
2305         return if $volids->{$volid};
2306
2307         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2308         return if !$path || !$owner || ($owner != $vmid);
2309
2310         $volids->{$volid} = 1;
2311         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2312         warn "Could not remove disk '$volid', check manually: $@" if $@;
2313     };
2314
2315     # only remove disks owned by this VM (referenced in the config)
2316     my $include_opts = {
2317         include_unused => 1,
2318         extra_keys => ['vmstate'],
2319     };
2320     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2321
2322     for my $snap (values %{$conf->{snapshots}}) {
2323         next if !defined($snap->{vmstate});
2324         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2325         next if !defined($drive);
2326         $remove_owned_drive->('vmstate', $drive);
2327     }
2328
2329     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2330
2331     if ($purge_unreferenced) { # also remove unreferenced disk
2332         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2333         PVE::Storage::foreach_volid($vmdisks, sub {
2334             my ($volid, $sid, $volname, $d) = @_;
2335             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2336             warn $@ if $@;
2337         });
2338     }
2339
2340     if (defined $replacement_conf) {
2341         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2342     } else {
2343         PVE::QemuConfig->destroy_config($vmid);
2344     }
2345 }
2346
2347 sub parse_vm_config {
2348     my ($filename, $raw, $strict) = @_;
2349
2350     return if !defined($raw);
2351
2352     my $res = {
2353         digest => Digest::SHA::sha1_hex($raw),
2354         snapshots => {},
2355         pending => {},
2356         cloudinit => {},
2357     };
2358
2359     my $handle_error = sub {
2360         my ($msg) = @_;
2361
2362         if ($strict) {
2363             die $msg;
2364         } else {
2365             warn $msg;
2366         }
2367     };
2368
2369     $filename =~ m|/qemu-server/(\d+)\.conf$|
2370         || die "got strange filename '$filename'";
2371
2372     my $vmid = $1;
2373
2374     my $conf = $res;
2375     my $descr;
2376     my $finish_description = sub {
2377         if (defined($descr)) {
2378             $descr =~ s/\s+$//;
2379             $conf->{description} = $descr;
2380         }
2381         $descr = undef;
2382     };
2383     my $section = '';
2384
2385     my @lines = split(/\n/, $raw);
2386     foreach my $line (@lines) {
2387         next if $line =~ m/^\s*$/;
2388
2389         if ($line =~ m/^\[PENDING\]\s*$/i) {
2390             $section = 'pending';
2391             $finish_description->();
2392             $conf = $res->{$section} = {};
2393             next;
2394         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2395             $section = 'cloudinit';
2396             $finish_description->();
2397             $conf = $res->{$section} = {};
2398             next;
2399
2400         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2401             $section = $1;
2402             $finish_description->();
2403             $conf = $res->{snapshots}->{$section} = {};
2404             next;
2405         }
2406
2407         if ($line =~ m/^\#(.*)$/) {
2408             $descr = '' if !defined($descr);
2409             $descr .= PVE::Tools::decode_text($1) . "\n";
2410             next;
2411         }
2412
2413         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($2);
2416         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2417             $conf->{snapstate} = $1;
2418         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2419             my $key = $1;
2420             my $value = $2;
2421             $conf->{$key} = $value;
2422         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2423             my $value = $1;
2424             if ($section eq 'pending') {
2425                 $conf->{delete} = $value; # we parse this later
2426             } else {
2427                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2428             }
2429         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2430             my $key = $1;
2431             my $value = $2;
2432             if ($section eq 'cloudinit') {
2433                 # ignore validation only used for informative purpose
2434                 $conf->{$key} = $value;
2435                 next;
2436             }
2437             eval { $value = check_type($key, $value); };
2438             if ($@) {
2439                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2440             } else {
2441                 $key = 'ide2' if $key eq 'cdrom';
2442                 my $fmt = $confdesc->{$key}->{format};
2443                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2444                     my $v = parse_drive($key, $value);
2445                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2446                         $v->{file} = $volid;
2447                         $value = print_drive($v);
2448                     } else {
2449                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2450                         next;
2451                     }
2452                 }
2453
2454                 $conf->{$key} = $value;
2455             }
2456         } else {
2457             $handle_error->("vm $vmid - unable to parse config: $line\n");
2458         }
2459     }
2460
2461     $finish_description->();
2462     delete $res->{snapstate}; # just to be sure
2463
2464     return $res;
2465 }
2466
2467 sub write_vm_config {
2468     my ($filename, $conf) = @_;
2469
2470     delete $conf->{snapstate}; # just to be sure
2471
2472     if ($conf->{cdrom}) {
2473         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2474         $conf->{ide2} = $conf->{cdrom};
2475         delete $conf->{cdrom};
2476     }
2477
2478     # we do not use 'smp' any longer
2479     if ($conf->{sockets}) {
2480         delete $conf->{smp};
2481     } elsif ($conf->{smp}) {
2482         $conf->{sockets} = $conf->{smp};
2483         delete $conf->{cores};
2484         delete $conf->{smp};
2485     }
2486
2487     my $used_volids = {};
2488
2489     my $cleanup_config = sub {
2490         my ($cref, $pending, $snapname) = @_;
2491
2492         foreach my $key (keys %$cref) {
2493             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2494                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2495             my $value = $cref->{$key};
2496             if ($key eq 'delete') {
2497                 die "propertry 'delete' is only allowed in [PENDING]\n"
2498                     if !$pending;
2499                 # fixme: check syntax?
2500                 next;
2501             }
2502             eval { $value = check_type($key, $value); };
2503             die "unable to parse value of '$key' - $@" if $@;
2504
2505             $cref->{$key} = $value;
2506
2507             if (!$snapname && is_valid_drivename($key)) {
2508                 my $drive = parse_drive($key, $value);
2509                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2510             }
2511         }
2512     };
2513
2514     &$cleanup_config($conf);
2515
2516     &$cleanup_config($conf->{pending}, 1);
2517
2518     foreach my $snapname (keys %{$conf->{snapshots}}) {
2519         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2520         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2521     }
2522
2523     # remove 'unusedX' settings if we re-add a volume
2524     foreach my $key (keys %$conf) {
2525         my $value = $conf->{$key};
2526         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2527             delete $conf->{$key};
2528         }
2529     }
2530
2531     my $generate_raw_config = sub {
2532         my ($conf, $pending) = @_;
2533
2534         my $raw = '';
2535
2536         # add description as comment to top of file
2537         if (defined(my $descr = $conf->{description})) {
2538             if ($descr) {
2539                 foreach my $cl (split(/\n/, $descr)) {
2540                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2541                 }
2542             } else {
2543                 $raw .= "#\n" if $pending;
2544             }
2545         }
2546
2547         foreach my $key (sort keys %$conf) {
2548             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2549             $raw .= "$key: $conf->{$key}\n";
2550         }
2551         return $raw;
2552     };
2553
2554     my $raw = &$generate_raw_config($conf);
2555
2556     if (scalar(keys %{$conf->{pending}})){
2557         $raw .= "\n[PENDING]\n";
2558         $raw .= &$generate_raw_config($conf->{pending}, 1);
2559     }
2560
2561     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2562         $raw .= "\n[special:cloudinit]\n";
2563         $raw .= &$generate_raw_config($conf->{cloudinit});
2564     }
2565
2566     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2567         $raw .= "\n[$snapname]\n";
2568         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2569     }
2570
2571     return $raw;
2572 }
2573
2574 sub load_defaults {
2575
2576     my $res = {};
2577
2578     # we use static defaults from our JSON schema configuration
2579     foreach my $key (keys %$confdesc) {
2580         if (defined(my $default = $confdesc->{$key}->{default})) {
2581             $res->{$key} = $default;
2582         }
2583     }
2584
2585     return $res;
2586 }
2587
2588 sub config_list {
2589     my $vmlist = PVE::Cluster::get_vmlist();
2590     my $res = {};
2591     return $res if !$vmlist || !$vmlist->{ids};
2592     my $ids = $vmlist->{ids};
2593     my $nodename = nodename();
2594
2595     foreach my $vmid (keys %$ids) {
2596         my $d = $ids->{$vmid};
2597         next if !$d->{node} || $d->{node} ne $nodename;
2598         next if !$d->{type} || $d->{type} ne 'qemu';
2599         $res->{$vmid}->{exists} = 1;
2600     }
2601     return $res;
2602 }
2603
2604 # test if VM uses local resources (to prevent migration)
2605 sub check_local_resources {
2606     my ($conf, $noerr) = @_;
2607
2608     my @loc_res = ();
2609     my $mapped_res = [];
2610
2611     my $nodelist = PVE::Cluster::get_nodelist();
2612     my $pci_map = PVE::Mapping::PCI::config();
2613     my $usb_map = PVE::Mapping::USB::config();
2614
2615     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2616
2617     my $add_missing_mapping = sub {
2618         my ($type, $key, $id) = @_;
2619         for my $node (@$nodelist) {
2620             my $entry;
2621             if ($type eq 'pci') {
2622                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2623             } elsif ($type eq 'usb') {
2624                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2625             }
2626             if (!scalar($entry->@*)) {
2627                 push @{$missing_mappings_by_node->{$node}}, $key;
2628             }
2629         }
2630     };
2631
2632     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2633     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2634
2635     push @loc_res, "ivshmem" if $conf->{ivshmem};
2636
2637     foreach my $k (keys %$conf) {
2638         if ($k =~ m/^usb/) {
2639             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2640             next if $entry->{host} =~ m/^spice$/i;
2641             if ($entry->{mapping}) {
2642                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2643                 push @$mapped_res, $k;
2644             }
2645         }
2646         if ($k =~ m/^hostpci/) {
2647             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2648             if ($entry->{mapping}) {
2649                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2650                 push @$mapped_res, $k;
2651             }
2652         }
2653         # sockets are safe: they will recreated be on the target side post-migrate
2654         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2655         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2656     }
2657
2658     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2659
2660     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2661 }
2662
2663 # check if used storages are available on all nodes (use by migrate)
2664 sub check_storage_availability {
2665     my ($storecfg, $conf, $node) = @_;
2666
2667     PVE::QemuConfig->foreach_volume($conf, sub {
2668         my ($ds, $drive) = @_;
2669
2670         my $volid = $drive->{file};
2671         return if !$volid;
2672
2673         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2674         return if !$sid;
2675
2676         # check if storage is available on both nodes
2677         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2678         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2679
2680         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2681
2682         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2683             if !$scfg->{content}->{$vtype};
2684    });
2685 }
2686
2687 # list nodes where all VM images are available (used by has_feature API)
2688 sub shared_nodes {
2689     my ($conf, $storecfg) = @_;
2690
2691     my $nodelist = PVE::Cluster::get_nodelist();
2692     my $nodehash = { map { $_ => 1 } @$nodelist };
2693     my $nodename = nodename();
2694
2695     PVE::QemuConfig->foreach_volume($conf, sub {
2696         my ($ds, $drive) = @_;
2697
2698         my $volid = $drive->{file};
2699         return if !$volid;
2700
2701         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2702         if ($storeid) {
2703             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2704             if ($scfg->{disable}) {
2705                 $nodehash = {};
2706             } elsif (my $avail = $scfg->{nodes}) {
2707                 foreach my $node (keys %$nodehash) {
2708                     delete $nodehash->{$node} if !$avail->{$node};
2709                 }
2710             } elsif (!$scfg->{shared}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     delete $nodehash->{$node} if $node ne $nodename
2713                 }
2714             }
2715         }
2716     });
2717
2718     return $nodehash
2719 }
2720
2721 sub check_local_storage_availability {
2722     my ($conf, $storecfg) = @_;
2723
2724     my $nodelist = PVE::Cluster::get_nodelist();
2725     my $nodehash = { map { $_ => {} } @$nodelist };
2726
2727     PVE::QemuConfig->foreach_volume($conf, sub {
2728         my ($ds, $drive) = @_;
2729
2730         my $volid = $drive->{file};
2731         return if !$volid;
2732
2733         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2734         if ($storeid) {
2735             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2736
2737             if ($scfg->{disable}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2740                 }
2741             } elsif (my $avail = $scfg->{nodes}) {
2742                 foreach my $node (keys %$nodehash) {
2743                     if (!$avail->{$node}) {
2744                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2745                     }
2746                 }
2747             }
2748         }
2749     });
2750
2751     foreach my $node (values %$nodehash) {
2752         if (my $unavail = $node->{unavailable_storages}) {
2753             $node->{unavailable_storages} = [ sort keys %$unavail ];
2754         }
2755     }
2756
2757     return $nodehash
2758 }
2759
2760 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2761 sub check_running {
2762     my ($vmid, $nocheck, $node) = @_;
2763
2764     # $nocheck is set when called during a migration, in which case the config
2765     # file might still or already reside on the *other* node
2766     # - because rename has already happened, and current node is source
2767     # - because rename hasn't happened yet, and current node is target
2768     # - because rename has happened, current node is target, but hasn't yet
2769     # processed it yet
2770     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2771     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2772 }
2773
2774 sub vzlist {
2775
2776     my $vzlist = config_list();
2777
2778     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2779
2780     while (defined(my $de = $fd->read)) {
2781         next if $de !~ m/^(\d+)\.pid$/;
2782         my $vmid = $1;
2783         next if !defined($vzlist->{$vmid});
2784         if (my $pid = check_running($vmid)) {
2785             $vzlist->{$vmid}->{pid} = $pid;
2786         }
2787     }
2788
2789     return $vzlist;
2790 }
2791
2792 our $vmstatus_return_properties = {
2793     vmid => get_standard_option('pve-vmid'),
2794     status => {
2795         description => "QEMU process status.",
2796         type => 'string',
2797         enum => ['stopped', 'running'],
2798     },
2799     maxmem => {
2800         description => "Maximum memory in bytes.",
2801         type => 'integer',
2802         optional => 1,
2803         renderer => 'bytes',
2804     },
2805     maxdisk => {
2806         description => "Root disk size in bytes.",
2807         type => 'integer',
2808         optional => 1,
2809         renderer => 'bytes',
2810     },
2811     name => {
2812         description => "VM name.",
2813         type => 'string',
2814         optional => 1,
2815     },
2816     qmpstatus => {
2817         description => "VM run state from the 'query-status' QMP monitor command.",
2818         type => 'string',
2819         optional => 1,
2820     },
2821     pid => {
2822         description => "PID of running qemu process.",
2823         type => 'integer',
2824         optional => 1,
2825     },
2826     uptime => {
2827         description => "Uptime.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'duration',
2831     },
2832     cpus => {
2833         description => "Maximum usable CPUs.",
2834         type => 'number',
2835         optional => 1,
2836     },
2837     lock => {
2838         description => "The current config lock, if any.",
2839         type => 'string',
2840         optional => 1,
2841     },
2842     tags => {
2843         description  => "The current configured tags, if any",
2844         type => 'string',
2845         optional => 1,
2846     },
2847     'running-machine' => {
2848         description => "The currently running machine type (if running).",
2849         type => 'string',
2850         optional => 1,
2851     },
2852     'running-qemu' => {
2853         description => "The currently running QEMU version (if running).",
2854         type => 'string',
2855         optional => 1,
2856     },
2857 };
2858
2859 my $last_proc_pid_stat;
2860
2861 # get VM status information
2862 # This must be fast and should not block ($full == false)
2863 # We only query KVM using QMP if $full == true (this can be slow)
2864 sub vmstatus {
2865     my ($opt_vmid, $full) = @_;
2866
2867     my $res = {};
2868
2869     my $storecfg = PVE::Storage::config();
2870
2871     my $list = vzlist();
2872     my $defaults = load_defaults();
2873
2874     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2875
2876     my $cpucount = $cpuinfo->{cpus} || 1;
2877
2878     foreach my $vmid (keys %$list) {
2879         next if $opt_vmid && ($vmid ne $opt_vmid);
2880
2881         my $conf = PVE::QemuConfig->load_config($vmid);
2882
2883         my $d = { vmid => int($vmid) };
2884         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2885
2886         # fixme: better status?
2887         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2888
2889         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2890         if (defined($size)) {
2891             $d->{disk} = 0; # no info available
2892             $d->{maxdisk} = $size;
2893         } else {
2894             $d->{disk} = 0;
2895             $d->{maxdisk} = 0;
2896         }
2897
2898         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2899             * ($conf->{cores} || $defaults->{cores});
2900         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2901         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2902
2903         $d->{name} = $conf->{name} || "VM $vmid";
2904         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2905
2906         if ($conf->{balloon}) {
2907             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2908             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2909                 : $defaults->{shares};
2910         }
2911
2912         $d->{uptime} = 0;
2913         $d->{cpu} = 0;
2914         $d->{mem} = 0;
2915
2916         $d->{netout} = 0;
2917         $d->{netin} = 0;
2918
2919         $d->{diskread} = 0;
2920         $d->{diskwrite} = 0;
2921
2922         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2923
2924         $d->{serial} = 1 if conf_has_serial($conf);
2925         $d->{lock} = $conf->{lock} if $conf->{lock};
2926         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2927
2928         $res->{$vmid} = $d;
2929     }
2930
2931     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2932     foreach my $dev (keys %$netdev) {
2933         next if $dev !~ m/^tap([1-9]\d*)i/;
2934         my $vmid = $1;
2935         my $d = $res->{$vmid};
2936         next if !$d;
2937
2938         $d->{netout} += $netdev->{$dev}->{receive};
2939         $d->{netin} += $netdev->{$dev}->{transmit};
2940
2941         if ($full) {
2942             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2943             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2944         }
2945
2946     }
2947
2948     my $ctime = gettimeofday;
2949
2950     foreach my $vmid (keys %$list) {
2951
2952         my $d = $res->{$vmid};
2953         my $pid = $d->{pid};
2954         next if !$pid;
2955
2956         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2957         next if !$pstat; # not running
2958
2959         my $used = $pstat->{utime} + $pstat->{stime};
2960
2961         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2962
2963         if ($pstat->{vsize}) {
2964             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2965         }
2966
2967         my $old = $last_proc_pid_stat->{$pid};
2968         if (!$old) {
2969             $last_proc_pid_stat->{$pid} = {
2970                 time => $ctime,
2971                 used => $used,
2972                 cpu => 0,
2973             };
2974             next;
2975         }
2976
2977         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2978
2979         if ($dtime > 1000) {
2980             my $dutime = $used -  $old->{used};
2981
2982             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2983             $last_proc_pid_stat->{$pid} = {
2984                 time => $ctime,
2985                 used => $used,
2986                 cpu => $d->{cpu},
2987             };
2988         } else {
2989             $d->{cpu} = $old->{cpu};
2990         }
2991     }
2992
2993     return $res if !$full;
2994
2995     my $qmpclient = PVE::QMPClient->new();
2996
2997     my $ballooncb = sub {
2998         my ($vmid, $resp) = @_;
2999
3000         my $info = $resp->{'return'};
3001         return if !$info->{max_mem};
3002
3003         my $d = $res->{$vmid};
3004
3005         # use memory assigned to VM
3006         $d->{maxmem} = $info->{max_mem};
3007         $d->{balloon} = $info->{actual};
3008
3009         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3010             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3011             $d->{freemem} = $info->{free_mem};
3012         }
3013
3014         $d->{ballooninfo} = $info;
3015     };
3016
3017     my $blockstatscb = sub {
3018         my ($vmid, $resp) = @_;
3019         my $data = $resp->{'return'} || [];
3020         my $totalrdbytes = 0;
3021         my $totalwrbytes = 0;
3022
3023         for my $blockstat (@$data) {
3024             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3025             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3026
3027             $blockstat->{device} =~ s/drive-//;
3028             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3029         }
3030         $res->{$vmid}->{diskread} = $totalrdbytes;
3031         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3032     };
3033
3034     my $machinecb = sub {
3035         my ($vmid, $resp) = @_;
3036         my $data = $resp->{'return'} || [];
3037
3038         $res->{$vmid}->{'running-machine'} =
3039             PVE::QemuServer::Machine::current_from_query_machines($data);
3040     };
3041
3042     my $versioncb = sub {
3043         my ($vmid, $resp) = @_;
3044         my $data = $resp->{'return'} // {};
3045         my $version = 'unknown';
3046
3047         if (my $v = $data->{qemu}) {
3048             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3049         }
3050
3051         $res->{$vmid}->{'running-qemu'} = $version;
3052     };
3053
3054     my $statuscb = sub {
3055         my ($vmid, $resp) = @_;
3056
3057         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3058         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3059         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3060         # this fails if ballon driver is not loaded, so this must be
3061         # the last commnand (following command are aborted if this fails).
3062         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3063
3064         my $status = 'unknown';
3065         if (!defined($status = $resp->{'return'}->{status})) {
3066             warn "unable to get VM status\n";
3067             return;
3068         }
3069
3070         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3071     };
3072
3073     foreach my $vmid (keys %$list) {
3074         next if $opt_vmid && ($vmid ne $opt_vmid);
3075         next if !$res->{$vmid}->{pid}; # not running
3076         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3077     }
3078
3079     $qmpclient->queue_execute(undef, 2);
3080
3081     foreach my $vmid (keys %$list) {
3082         next if $opt_vmid && ($vmid ne $opt_vmid);
3083         next if !$res->{$vmid}->{pid}; #not running
3084
3085         # we can't use the $qmpclient since it might have already aborted on
3086         # 'query-balloon', but this might also fail for older versions...
3087         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3088         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3089     }
3090
3091     foreach my $vmid (keys %$list) {
3092         next if $opt_vmid && ($vmid ne $opt_vmid);
3093         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3094     }
3095
3096     return $res;
3097 }
3098
3099 sub conf_has_serial {
3100     my ($conf) = @_;
3101
3102     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3103         if ($conf->{"serial$i"}) {
3104             return 1;
3105         }
3106     }
3107
3108     return 0;
3109 }
3110
3111 sub conf_has_audio {
3112     my ($conf, $id) = @_;
3113
3114     $id //= 0;
3115     my $audio = $conf->{"audio$id"};
3116     return if !defined($audio);
3117
3118     my $audioproperties = parse_property_string($audio_fmt, $audio);
3119     my $audiodriver = $audioproperties->{driver} // 'spice';
3120
3121     return {
3122         dev => $audioproperties->{device},
3123         dev_id => "audiodev$id",
3124         backend => $audiodriver,
3125         backend_id => "$audiodriver-backend${id}",
3126     };
3127 }
3128
3129 sub audio_devs {
3130     my ($audio, $audiopciaddr, $machine_version) = @_;
3131
3132     my $devs = [];
3133
3134     my $id = $audio->{dev_id};
3135     my $audiodev = "";
3136     if (min_version($machine_version, 4, 2)) {
3137         $audiodev = ",audiodev=$audio->{backend_id}";
3138     }
3139
3140     if ($audio->{dev} eq 'AC97') {
3141         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3142     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3143         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3144         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3145         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3146     } else {
3147         die "unkown audio device '$audio->{dev}', implement me!";
3148     }
3149
3150     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3151
3152     return $devs;
3153 }
3154
3155 sub get_tpm_paths {
3156     my ($vmid) = @_;
3157     return {
3158         socket => "/var/run/qemu-server/$vmid.swtpm",
3159         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3160     };
3161 }
3162
3163 sub add_tpm_device {
3164     my ($vmid, $devices, $conf) = @_;
3165
3166     return if !$conf->{tpmstate0};
3167
3168     my $paths = get_tpm_paths($vmid);
3169
3170     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3171     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3172     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3173 }
3174
3175 sub start_swtpm {
3176     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3177
3178     return if !$tpmdrive;
3179
3180     my $state;
3181     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3182     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3183     if ($storeid) {
3184         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3185     } else {
3186         $state = $tpm->{file};
3187     }
3188
3189     my $paths = get_tpm_paths($vmid);
3190
3191     # during migration, we will get state from remote
3192     #
3193     if (!$migration) {
3194         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3195         my $setup_cmd = [
3196             "swtpm_setup",
3197             "--tpmstate",
3198             "file://$state",
3199             "--createek",
3200             "--create-ek-cert",
3201             "--create-platform-cert",
3202             "--lock-nvram",
3203             "--config",
3204             "/etc/swtpm_setup.conf", # do not use XDG configs
3205             "--runas",
3206             "0", # force creation as root, error if not possible
3207             "--not-overwrite", # ignore existing state, do not modify
3208         ];
3209
3210         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3211         # TPM 2.0 supports ECC crypto, use if possible
3212         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3213
3214         run_command($setup_cmd, outfunc => sub {
3215             print "swtpm_setup: $1\n";
3216         });
3217     }
3218
3219     # Used to distinguish different invocations in the log.
3220     my $log_prefix = "[id=" . int(time()) . "] ";
3221
3222     my $emulator_cmd = [
3223         "swtpm",
3224         "socket",
3225         "--tpmstate",
3226         "backend-uri=file://$state,mode=0600",
3227         "--ctrl",
3228         "type=unixio,path=$paths->{socket},mode=0600",
3229         "--pid",
3230         "file=$paths->{pid}",
3231         "--terminate", # terminate on QEMU disconnect
3232         "--daemon",
3233         "--log",
3234         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3235     ];
3236     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3237     run_command($emulator_cmd, outfunc => sub { print $1; });
3238
3239     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3240     while (! -e $paths->{pid}) {
3241         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3242         usleep(50_000);
3243     }
3244
3245     # return untainted PID of swtpm daemon so it can be killed on error
3246     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3247     return $1;
3248 }
3249
3250 sub vga_conf_has_spice {
3251     my ($vga) = @_;
3252
3253     my $vgaconf = parse_vga($vga);
3254     my $vgatype = $vgaconf->{type};
3255     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3256
3257     return $1 || 1;
3258 }
3259
3260 sub is_native($) {
3261     my ($arch) = @_;
3262     return get_host_arch() eq $arch;
3263 }
3264
3265 sub get_vm_arch {
3266     my ($conf) = @_;
3267     return $conf->{arch} // get_host_arch();
3268 }
3269
3270 my $default_machines = {
3271     x86_64 => 'pc',
3272     aarch64 => 'virt',
3273 };
3274
3275 sub get_installed_machine_version {
3276     my ($kvmversion) = @_;
3277     $kvmversion = kvm_user_version() if !defined($kvmversion);
3278     $kvmversion =~ m/^(\d+\.\d+)/;
3279     return $1;
3280 }
3281
3282 sub windows_get_pinned_machine_version {
3283     my ($machine, $base_version, $kvmversion) = @_;
3284
3285     my $pin_version = $base_version;
3286     if (!defined($base_version) ||
3287         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3288     ) {
3289         $pin_version = get_installed_machine_version($kvmversion);
3290     }
3291     if (!$machine || $machine eq 'pc') {
3292         $machine = "pc-i440fx-$pin_version";
3293     } elsif ($machine eq 'q35') {
3294         $machine = "pc-q35-$pin_version";
3295     } elsif ($machine eq 'virt') {
3296         $machine = "virt-$pin_version";
3297     } else {
3298         warn "unknown machine type '$machine', not touching that!\n";
3299     }
3300
3301     return $machine;
3302 }
3303
3304 sub get_vm_machine {
3305     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3306
3307     my $machine = $forcemachine || $conf->{machine};
3308
3309     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3310         $kvmversion //= kvm_user_version();
3311         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3312         # layout which confuses windows quite a bit and may result in various regressions..
3313         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3314         if (windows_version($conf->{ostype})) {
3315             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3316         }
3317         $arch //= 'x86_64';
3318         $machine ||= $default_machines->{$arch};
3319         if ($add_pve_version) {
3320             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3321             $machine .= "+pve$pvever";
3322         }
3323     }
3324
3325     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3326         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3327         $machine = $1 if $is_pxe;
3328
3329         # for version-pinned machines that do not include a pve-version (e.g.
3330         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3331         $machine .= '+pve0';
3332
3333         $machine .= '.pxe' if $is_pxe;
3334     }
3335
3336     return $machine;
3337 }
3338
3339 sub get_ovmf_files($$$) {
3340     my ($arch, $efidisk, $smm) = @_;
3341
3342     my $types = $OVMF->{$arch}
3343         or die "no OVMF images known for architecture '$arch'\n";
3344
3345     my $type = 'default';
3346     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3347         $type = $smm ? "4m" : "4m-no-smm";
3348         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3349     }
3350
3351     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3352     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3353     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3354
3355     return ($ovmf_code, $ovmf_vars);
3356 }
3357
3358 my $Arch2Qemu = {
3359     aarch64 => '/usr/bin/qemu-system-aarch64',
3360     x86_64 => '/usr/bin/qemu-system-x86_64',
3361 };
3362 sub get_command_for_arch($) {
3363     my ($arch) = @_;
3364     return '/usr/bin/kvm' if is_native($arch);
3365
3366     my $cmd = $Arch2Qemu->{$arch}
3367         or die "don't know how to emulate architecture '$arch'\n";
3368     return $cmd;
3369 }
3370
3371 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3372 # to use in a QEMU command line (-cpu element), first array_intersect the result
3373 # of query_supported_ with query_understood_. This is necessary because:
3374 #
3375 # a) query_understood_ returns flags the host cannot use and
3376 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3377 #    flags, but CPU settings - with most of them being flags. Those settings
3378 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3379 #
3380 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3381 # expensive. If you need the value returned from this, you can get it much
3382 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3383 # $accel being 'kvm' or 'tcg'.
3384 #
3385 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3386 # changes, automatically populating pmxcfs.
3387 #
3388 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3389 # since kvm and tcg machines support different flags
3390 #
3391 sub query_supported_cpu_flags {
3392     my ($arch) = @_;
3393
3394     $arch //= get_host_arch();
3395     my $default_machine = $default_machines->{$arch};
3396
3397     my $flags = {};
3398
3399     # FIXME: Once this is merged, the code below should work for ARM as well:
3400     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3401     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3402         $arch eq "aarch64";
3403
3404     my $kvm_supported = defined(kvm_version());
3405     my $qemu_cmd = get_command_for_arch($arch);
3406     my $fakevmid = -1;
3407     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3408
3409     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3410     my $query_supported_run_qemu = sub {
3411         my ($kvm) = @_;
3412
3413         my $flags = {};
3414         my $cmd = [
3415             $qemu_cmd,
3416             '-machine', $default_machine,
3417             '-display', 'none',
3418             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3419             '-mon', 'chardev=qmp,mode=control',
3420             '-pidfile', $pidfile,
3421             '-S', '-daemonize'
3422         ];
3423
3424         if (!$kvm) {
3425             push @$cmd, '-accel', 'tcg';
3426         }
3427
3428         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3429         die "QEMU flag querying VM exited with code " . $rc if $rc;
3430
3431         eval {
3432             my $cmd_result = mon_cmd(
3433                 $fakevmid,
3434                 'query-cpu-model-expansion',
3435                 type => 'full',
3436                 model => { name => 'host' }
3437             );
3438
3439             my $props = $cmd_result->{model}->{props};
3440             foreach my $prop (keys %$props) {
3441                 next if $props->{$prop} ne '1';
3442                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3443                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3444                 # We only keep those with underscores, to match /proc/cpuinfo
3445                 $prop =~ s/\.|-/_/g;
3446                 $flags->{$prop} = 1;
3447             }
3448         };
3449         my $err = $@;
3450
3451         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3452         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3453
3454         die $err if $err;
3455
3456         return [ sort keys %$flags ];
3457     };
3458
3459     # We need to query QEMU twice, since KVM and TCG have different supported flags
3460     PVE::QemuConfig->lock_config($fakevmid, sub {
3461         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3462         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3463
3464         if ($kvm_supported) {
3465             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3466             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3467         }
3468     });
3469
3470     return $flags;
3471 }
3472
3473 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3474 my $understood_cpu_flag_dir = "/usr/share/kvm";
3475 sub query_understood_cpu_flags {
3476     my $arch = get_host_arch();
3477     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3478
3479     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3480         if ! -e $filepath;
3481
3482     my $raw = file_get_contents($filepath);
3483     $raw =~ s/^\s+|\s+$//g;
3484     my @flags = split(/\s+/, $raw);
3485
3486     return \@flags;
3487 }
3488
3489 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3490 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3491 my sub should_disable_smm {
3492     my ($conf, $vga, $machine) = @_;
3493
3494     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3495
3496     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3497         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3498 }
3499
3500 my sub print_ovmf_drive_commandlines {
3501     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3502
3503     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3504
3505     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3506
3507     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3508     if ($d) {
3509         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3510         my ($path, $format) = $d->@{'file', 'format'};
3511         if ($storeid) {
3512             $path = PVE::Storage::path($storecfg, $d->{file});
3513             if (!defined($format)) {
3514                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3515                 $format = qemu_img_format($scfg, $volname);
3516             }
3517         } elsif (!defined($format)) {
3518             die "efidisk format must be specified\n";
3519         }
3520         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3521         if ($path =~ m/^rbd:/) {
3522             $var_drive_str .= ',cache=writeback';
3523             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3524         }
3525         $var_drive_str .= ",format=$format,file=$path";
3526
3527         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3528         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3529     } else {
3530         log_warn("no efidisk configured! Using temporary efivars disk.");
3531         my $path = "/tmp/$vmid-ovmf.fd";
3532         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3533         $var_drive_str .= ",format=raw,file=$path";
3534         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3535     }
3536
3537     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3538 }
3539
3540 sub config_to_command {
3541     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3542         $pbs_backing) = @_;
3543
3544     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3545     my $devices = [];
3546     my $bridges = {};
3547     my $ostype = $conf->{ostype};
3548     my $winversion = windows_version($ostype);
3549     my $kvm = $conf->{kvm};
3550     my $nodename = nodename();
3551
3552     my $arch = get_vm_arch($conf);
3553     my $kvm_binary = get_command_for_arch($arch);
3554     my $kvmver = kvm_user_version($kvm_binary);
3555
3556     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3557         $kvmver //= "undefined";
3558         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3559     }
3560
3561     my $add_pve_version = min_version($kvmver, 4, 1);
3562
3563     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3564     my $machine_version = extract_version($machine_type, $kvmver);
3565     $kvm //= 1 if is_native($arch);
3566
3567     $machine_version =~ m/(\d+)\.(\d+)/;
3568     my ($machine_major, $machine_minor) = ($1, $2);
3569
3570     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3571         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3572     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3573         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3574             ." please upgrade node '$nodename'\n"
3575     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3576         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3577         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3578             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3579             ." node '$nodename'\n";
3580     }
3581
3582     # if a specific +pve version is required for a feature, use $version_guard
3583     # instead of min_version to allow machines to be run with the minimum
3584     # required version
3585     my $required_pve_version = 0;
3586     my $version_guard = sub {
3587         my ($major, $minor, $pve) = @_;
3588         return 0 if !min_version($machine_version, $major, $minor, $pve);
3589         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3590         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3591         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3592         return 1;
3593     };
3594
3595     if ($kvm && !defined kvm_version()) {
3596         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3597             ." or enable in BIOS.\n";
3598     }
3599
3600     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3601     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3602     my $use_old_bios_files = undef;
3603     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3604
3605     my $cmd = [];
3606     if ($conf->{affinity}) {
3607         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3608     }
3609
3610     push @$cmd, $kvm_binary;
3611
3612     push @$cmd, '-id', $vmid;
3613
3614     my $vmname = $conf->{name} || "vm$vmid";
3615
3616     push @$cmd, '-name', "$vmname,debug-threads=on";
3617
3618     push @$cmd, '-no-shutdown';
3619
3620     my $use_virtio = 0;
3621
3622     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3623     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3624     push @$cmd, '-mon', "chardev=qmp,mode=control";
3625
3626     if (min_version($machine_version, 2, 12)) {
3627         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3628         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3629     }
3630
3631     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3632
3633     push @$cmd, '-daemonize';
3634
3635     if ($conf->{smbios1}) {
3636         my $smbios_conf = parse_smbios1($conf->{smbios1});
3637         if ($smbios_conf->{base64}) {
3638             # Do not pass base64 flag to qemu
3639             delete $smbios_conf->{base64};
3640             my $smbios_string = "";
3641             foreach my $key (keys %$smbios_conf) {
3642                 my $value;
3643                 if ($key eq "uuid") {
3644                     $value = $smbios_conf->{uuid}
3645                 } else {
3646                     $value = decode_base64($smbios_conf->{$key});
3647                 }
3648                 # qemu accepts any binary data, only commas need escaping by double comma
3649                 $value =~ s/,/,,/g;
3650                 $smbios_string .= "," . $key . "=" . $value if $value;
3651             }
3652             push @$cmd, '-smbios', "type=1" . $smbios_string;
3653         } else {
3654             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3655         }
3656     }
3657
3658     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3659         my ($code_drive_str, $var_drive_str) =
3660             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3661         push $cmd->@*, '-drive', $code_drive_str;
3662         push $cmd->@*, '-drive', $var_drive_str;
3663     }
3664
3665     if ($q35) { # tell QEMU to load q35 config early
3666         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3667         if (min_version($machine_version, 4, 0)) {
3668             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3669         } else {
3670             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3671         }
3672     }
3673
3674     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3675         push @$cmd, $fixups->@*;
3676     }
3677
3678     if ($conf->{vmgenid}) {
3679         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3680     }
3681
3682     # add usb controllers
3683     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3684         $conf, $bridges, $arch, $machine_type, $machine_version);
3685     push @$devices, @usbcontrollers if @usbcontrollers;
3686     my $vga = parse_vga($conf->{vga});
3687
3688     my $qxlnum = vga_conf_has_spice($conf->{vga});
3689     $vga->{type} = 'qxl' if $qxlnum;
3690
3691     if (!$vga->{type}) {
3692         if ($arch eq 'aarch64') {
3693             $vga->{type} = 'virtio';
3694         } elsif (min_version($machine_version, 2, 9)) {
3695             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3696         } else {
3697             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3698         }
3699     }
3700
3701     # enable absolute mouse coordinates (needed by vnc)
3702     my $tablet = $conf->{tablet};
3703     if (!defined($tablet)) {
3704         $tablet = $defaults->{tablet};
3705         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3706         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3707     }
3708
3709     if ($tablet) {
3710         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3711         my $kbd = print_keyboarddevice_full($conf, $arch);
3712         push @$devices, '-device', $kbd if defined($kbd);
3713     }
3714
3715     my $bootorder = device_bootorder($conf);
3716
3717     # host pci device passthrough
3718     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3719         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3720
3721     # usb devices
3722     my $usb_dev_features = {};
3723     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3724
3725     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3726         $conf, $usb_dev_features, $bootorder, $machine_version);
3727     push @$devices, @usbdevices if @usbdevices;
3728
3729     # serial devices
3730     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3731         my $path = $conf->{"serial$i"} or next;
3732         if ($path eq 'socket') {
3733             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3734             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3735             # On aarch64, serial0 is the UART device. QEMU only allows
3736             # connecting UART devices via the '-serial' command line, as
3737             # the device has a fixed slot on the hardware...
3738             if ($arch eq 'aarch64' && $i == 0) {
3739                 push @$devices, '-serial', "chardev:serial$i";
3740             } else {
3741                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3742             }
3743         } else {
3744             die "no such serial device\n" if ! -c $path;
3745             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3746             push @$devices, '-device', "isa-serial,chardev=serial$i";
3747         }
3748     }
3749
3750     # parallel devices
3751     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3752         if (my $path = $conf->{"parallel$i"}) {
3753             die "no such parallel device\n" if ! -c $path;
3754             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3755             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3756             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3757         }
3758     }
3759
3760     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3761         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3762         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3763         push @$devices, @$audio_devs;
3764     }
3765
3766     # Add a TPM only if the VM is not a template,
3767     # to support backing up template VMs even if the TPM disk is write-protected.
3768     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3769
3770     my $sockets = 1;
3771     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3772     $sockets = $conf->{sockets} if  $conf->{sockets};
3773
3774     my $cores = $conf->{cores} || 1;
3775
3776     my $maxcpus = $sockets * $cores;
3777
3778     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3779
3780     my $allowed_vcpus = $cpuinfo->{cpus};
3781
3782     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3783
3784     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3785         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3786         for (my $i = 2; $i <= $vcpus; $i++)  {
3787             my $cpustr = print_cpu_device($conf,$i);
3788             push @$cmd, '-device', $cpustr;
3789         }
3790
3791     } else {
3792
3793         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3794     }
3795     push @$cmd, '-nodefaults';
3796
3797     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3798
3799     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3800
3801     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3802
3803     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3804         push @$devices, '-device', print_vga_device(
3805             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3806
3807         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3808
3809         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3810         push @$cmd,  '-vnc', "unix:$socket,password=on";
3811     } else {
3812         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3813         push @$cmd, '-nographic';
3814     }
3815
3816     # time drift fix
3817     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3818     my $useLocaltime = $conf->{localtime};
3819
3820     if ($winversion >= 5) { # windows
3821         $useLocaltime = 1 if !defined($conf->{localtime});
3822
3823         # use time drift fix when acpi is enabled
3824         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3825             $tdf = 1 if !defined($conf->{tdf});
3826         }
3827     }
3828
3829     if ($winversion >= 6) {
3830         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3831         push @$machineFlags, 'hpet=off';
3832     }
3833
3834     push @$rtcFlags, 'driftfix=slew' if $tdf;
3835
3836     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3837         push @$rtcFlags, "base=$conf->{startdate}";
3838     } elsif ($useLocaltime) {
3839         push @$rtcFlags, 'base=localtime';
3840     }
3841
3842     if ($forcecpu) {
3843         push @$cmd, '-cpu', $forcecpu;
3844     } else {
3845         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3846     }
3847
3848     PVE::QemuServer::Memory::config(
3849         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3850
3851     push @$cmd, '-S' if $conf->{freeze};
3852
3853     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3854
3855     my $guest_agent = parse_guest_agent($conf);
3856
3857     if ($guest_agent->{enabled}) {
3858         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3859         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3860
3861         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3862             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3863             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3864             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3865         } elsif ($guest_agent->{type} eq 'isa') {
3866             push @$devices, '-device', "isa-serial,chardev=qga0";
3867         }
3868     }
3869
3870     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3871     if ($rng && $version_guard->(4, 1, 2)) {
3872         check_rng_source($rng->{source});
3873
3874         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3875         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3876         my $limiter_str = "";
3877         if ($max_bytes) {
3878             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3879         }
3880
3881         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3882         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3883         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3884     }
3885
3886     my $spice_port;
3887
3888     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3889         if ($qxlnum > 1) {
3890             if ($winversion){
3891                 for (my $i = 1; $i < $qxlnum; $i++){
3892                     push @$devices, '-device', print_vga_device(
3893                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3894                 }
3895             } else {
3896                 # assume other OS works like Linux
3897                 my ($ram, $vram) = ("134217728", "67108864");
3898                 if ($vga->{memory}) {
3899                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3900                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3901                 }
3902                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3903                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3904             }
3905         }
3906
3907         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3908
3909         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3910         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3911         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3912
3913         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3914         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3915         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3916
3917         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3918         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3919
3920         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3921         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3922         if ($spice_enhancement->{foldersharing}) {
3923             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3924             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3925         }
3926
3927         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3928         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3929             if $spice_enhancement->{videostreaming};
3930
3931         push @$devices, '-spice', "$spice_opts";
3932     }
3933
3934     # enable balloon by default, unless explicitly disabled
3935     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3936         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3937         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3938         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3939         push @$devices, '-device', $ballooncmd;
3940     }
3941
3942     if ($conf->{watchdog}) {
3943         my $wdopts = parse_watchdog($conf->{watchdog});
3944         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3945         my $watchdog = $wdopts->{model} || 'i6300esb';
3946         push @$devices, '-device', "$watchdog$pciaddr";
3947         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3948     }
3949
3950     my $vollist = [];
3951     my $scsicontroller = {};
3952     my $ahcicontroller = {};
3953     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3954
3955     # Add iscsi initiator name if available
3956     if (my $initiator = get_initiator_name()) {
3957         push @$devices, '-iscsi', "initiator-name=$initiator";
3958     }
3959
3960     PVE::QemuConfig->foreach_volume($conf, sub {
3961         my ($ds, $drive) = @_;
3962
3963         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3964             check_volume_storage_type($storecfg, $drive->{file});
3965             push @$vollist, $drive->{file};
3966         }
3967
3968         # ignore efidisk here, already added in bios/fw handling code above
3969         return if $drive->{interface} eq 'efidisk';
3970         # similar for TPM
3971         return if $drive->{interface} eq 'tpmstate';
3972
3973         $use_virtio = 1 if $ds =~ m/^virtio/;
3974
3975         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3976
3977         if ($drive->{interface} eq 'virtio'){
3978            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3979         }
3980
3981         if ($drive->{interface} eq 'scsi') {
3982
3983             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3984
3985             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3986                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3987
3988             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3989             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3990
3991             my $iothread = '';
3992             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3993                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3994                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3995             } elsif ($drive->{iothread}) {
3996                 log_warn(
3997                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3998                 );
3999             }
4000
4001             my $queues = '';
4002             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4003                 $queues = ",num_queues=$drive->{queues}";
4004             }
4005
4006             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4007                 if !$scsicontroller->{$controller};
4008             $scsicontroller->{$controller}=1;
4009         }
4010
4011         if ($drive->{interface} eq 'sata') {
4012             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4013             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4014             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4015                 if !$ahcicontroller->{$controller};
4016             $ahcicontroller->{$controller}=1;
4017         }
4018
4019         my $pbs_conf = $pbs_backing->{$ds};
4020         my $pbs_name = undef;
4021         if ($pbs_conf) {
4022             $pbs_name = "drive-$ds-pbs";
4023             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4024         }
4025
4026         my $drive_cmd = print_drive_commandline_full(
4027             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4028
4029         # extra protection for templates, but SATA and IDE don't support it..
4030         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4031
4032         push @$devices, '-drive',$drive_cmd;
4033         push @$devices, '-device', print_drivedevice_full(
4034             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4035     });
4036
4037     for (my $i = 0; $i < $MAX_NETS; $i++) {
4038         my $netname = "net$i";
4039
4040         next if !$conf->{$netname};
4041         my $d = parse_net($conf->{$netname});
4042         next if !$d;
4043         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4044
4045         $use_virtio = 1 if $d->{model} eq 'virtio';
4046
4047         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4048
4049         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4050         push @$devices, '-netdev', $netdevfull;
4051
4052         my $netdevicefull = print_netdevice_full(
4053             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4054
4055         push @$devices, '-device', $netdevicefull;
4056     }
4057
4058     if ($conf->{ivshmem}) {
4059         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4060
4061         my $bus;
4062         if ($q35) {
4063             $bus = print_pcie_addr("ivshmem");
4064         } else {
4065             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4066         }
4067
4068         my $ivshmem_name = $ivshmem->{name} // $vmid;
4069         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4070
4071         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4072         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4073             .",size=$ivshmem->{size}M";
4074     }
4075
4076     # pci.4 is nested in pci.1
4077     $bridges->{1} = 1 if $bridges->{4};
4078
4079     if (!$q35) { # add pci bridges
4080         if (min_version($machine_version, 2, 3)) {
4081            $bridges->{1} = 1;
4082            $bridges->{2} = 1;
4083         }
4084         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4085     }
4086
4087     for my $k (sort {$b cmp $a} keys %$bridges) {
4088         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4089
4090         my $k_name = $k;
4091         if ($k == 2 && $legacy_igd) {
4092             $k_name = "$k-igd";
4093         }
4094         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4095         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4096
4097         if ($q35) { # add after -readconfig pve-q35.cfg
4098             splice @$devices, 2, 0, '-device', $devstr;
4099         } else {
4100             unshift @$devices, '-device', $devstr if $k > 0;
4101         }
4102     }
4103
4104     if (!$kvm) {
4105         push @$machineFlags, 'accel=tcg';
4106     }
4107
4108     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4109
4110     my $machine_type_min = $machine_type;
4111     if ($add_pve_version) {
4112         $machine_type_min =~ s/\+pve\d+$//;
4113         $machine_type_min .= "+pve$required_pve_version";
4114     }
4115     push @$machineFlags, "type=${machine_type_min}";
4116
4117     push @$cmd, @$devices;
4118     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4119     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4120     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4121
4122     if (my $vmstate = $conf->{vmstate}) {
4123         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4124         push @$vollist, $vmstate;
4125         push @$cmd, '-loadstate', $statepath;
4126         print "activating and using '$vmstate' as vmstate\n";
4127     }
4128
4129     if (PVE::QemuConfig->is_template($conf)) {
4130         # needed to workaround base volumes being read-only
4131         push @$cmd, '-snapshot';
4132     }
4133
4134     # add custom args
4135     if ($conf->{args}) {
4136         my $aa = PVE::Tools::split_args($conf->{args});
4137         push @$cmd, @$aa;
4138     }
4139
4140     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4141 }
4142
4143 sub check_rng_source {
4144     my ($source) = @_;
4145
4146     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4147     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4148         if ! -e $source;
4149
4150     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4151     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4152         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4153         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4154         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4155             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4156             ." to the host.\n";
4157     }
4158 }
4159
4160 sub spice_port {
4161     my ($vmid) = @_;
4162
4163     my $res = mon_cmd($vmid, 'query-spice');
4164
4165     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4166 }
4167
4168 sub vm_devices_list {
4169     my ($vmid) = @_;
4170
4171     my $res = mon_cmd($vmid, 'query-pci');
4172     my $devices_to_check = [];
4173     my $devices = {};
4174     foreach my $pcibus (@$res) {
4175         push @$devices_to_check, @{$pcibus->{devices}},
4176     }
4177
4178     while (@$devices_to_check) {
4179         my $to_check = [];
4180         for my $d (@$devices_to_check) {
4181             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4182             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4183
4184             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4185             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4186         }
4187         $devices_to_check = $to_check;
4188     }
4189
4190     my $resblock = mon_cmd($vmid, 'query-block');
4191     foreach my $block (@$resblock) {
4192         if($block->{device} =~ m/^drive-(\S+)/){
4193                 $devices->{$1} = 1;
4194         }
4195     }
4196
4197     my $resmice = mon_cmd($vmid, 'query-mice');
4198     foreach my $mice (@$resmice) {
4199         if ($mice->{name} eq 'QEMU HID Tablet') {
4200             $devices->{tablet} = 1;
4201             last;
4202         }
4203     }
4204
4205     # for usb devices there is no query-usb
4206     # but we can iterate over the entries in
4207     # qom-list path=/machine/peripheral
4208     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4209     foreach my $per (@$resperipheral) {
4210         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4211             $devices->{$per->{name}} = 1;
4212         }
4213     }
4214
4215     return $devices;
4216 }
4217
4218 sub vm_deviceplug {
4219     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4220
4221     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4222
4223     my $devices_list = vm_devices_list($vmid);
4224     return 1 if defined($devices_list->{$deviceid});
4225
4226     # add PCI bridge if we need it for the device
4227     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4228
4229     if ($deviceid eq 'tablet') {
4230         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4231     } elsif ($deviceid eq 'keyboard') {
4232         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4233     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4234         my $id = $1;
4235         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4236         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4237     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4238         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4239     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4240         qemu_iothread_add($vmid, $deviceid, $device);
4241
4242         qemu_driveadd($storecfg, $vmid, $device);
4243         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4244
4245         qemu_deviceadd($vmid, $devicefull);
4246         eval { qemu_deviceaddverify($vmid, $deviceid); };
4247         if (my $err = $@) {
4248             eval { qemu_drivedel($vmid, $deviceid); };
4249             warn $@ if $@;
4250             die $err;
4251         }
4252     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4253         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4254         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4255         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4256
4257         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4258
4259         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4260             qemu_iothread_add($vmid, $deviceid, $device);
4261             $devicefull .= ",iothread=iothread-$deviceid";
4262         }
4263
4264         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4265             $devicefull .= ",num_queues=$device->{queues}";
4266         }
4267
4268         qemu_deviceadd($vmid, $devicefull);
4269         qemu_deviceaddverify($vmid, $deviceid);
4270     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4271         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4272         qemu_driveadd($storecfg, $vmid, $device);
4273
4274         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4275         eval { qemu_deviceadd($vmid, $devicefull); };
4276         if (my $err = $@) {
4277             eval { qemu_drivedel($vmid, $deviceid); };
4278             warn $@ if $@;
4279             die $err;
4280         }
4281     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4282         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4283
4284         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4285         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4286         my $use_old_bios_files = undef;
4287         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4288
4289         my $netdevicefull = print_netdevice_full(
4290             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4291         qemu_deviceadd($vmid, $netdevicefull);
4292         eval {
4293             qemu_deviceaddverify($vmid, $deviceid);
4294             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4295         };
4296         if (my $err = $@) {
4297             eval { qemu_netdevdel($vmid, $deviceid); };
4298             warn $@ if $@;
4299             die $err;
4300         }
4301     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4302         my $bridgeid = $2;
4303         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4304         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4305
4306         qemu_deviceadd($vmid, $devicefull);
4307         qemu_deviceaddverify($vmid, $deviceid);
4308     } else {
4309         die "can't hotplug device '$deviceid'\n";
4310     }
4311
4312     return 1;
4313 }
4314
4315 # fixme: this should raise exceptions on error!
4316 sub vm_deviceunplug {
4317     my ($vmid, $conf, $deviceid) = @_;
4318
4319     my $devices_list = vm_devices_list($vmid);
4320     return 1 if !defined($devices_list->{$deviceid});
4321
4322     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4323     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4324
4325     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4326         qemu_devicedel($vmid, $deviceid);
4327     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_devicedelverify($vmid, $deviceid);
4330     } elsif ($deviceid =~ m/^usb\d+$/) {
4331         qemu_devicedel($vmid, $deviceid);
4332         qemu_devicedelverify($vmid, $deviceid);
4333     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4334         my $device = parse_drive($deviceid, $conf->{$deviceid});
4335
4336         qemu_devicedel($vmid, $deviceid);
4337         qemu_devicedelverify($vmid, $deviceid);
4338         qemu_drivedel($vmid, $deviceid);
4339         qemu_iothread_del($vmid, $deviceid, $device);
4340     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4341         qemu_devicedel($vmid, $deviceid);
4342         qemu_devicedelverify($vmid, $deviceid);
4343     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4344         my $device = parse_drive($deviceid, $conf->{$deviceid});
4345
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348         qemu_drivedel($vmid, $deviceid);
4349         qemu_deletescsihw($conf, $vmid, $deviceid);
4350
4351         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4352             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4353     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356         qemu_netdevdel($vmid, $deviceid);
4357     } else {
4358         die "can't unplug device '$deviceid'\n";
4359     }
4360
4361     return 1;
4362 }
4363
4364 sub qemu_spice_usbredir_chardev_add {
4365     my ($vmid, $id) = @_;
4366
4367     mon_cmd($vmid, "chardev-add" , (
4368         id => $id,
4369         backend => {
4370             type => 'spicevmc',
4371             data => {
4372                 type => "usbredir",
4373             },
4374         },
4375     ));
4376 }
4377
4378 sub qemu_iothread_add {
4379     my ($vmid, $deviceid, $device) = @_;
4380
4381     if ($device->{iothread}) {
4382         my $iothreads = vm_iothreads_list($vmid);
4383         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4384     }
4385 }
4386
4387 sub qemu_iothread_del {
4388     my ($vmid, $deviceid, $device) = @_;
4389
4390     if ($device->{iothread}) {
4391         my $iothreads = vm_iothreads_list($vmid);
4392         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4393     }
4394 }
4395
4396 sub qemu_driveadd {
4397     my ($storecfg, $vmid, $device) = @_;
4398
4399     my $kvmver = get_running_qemu_version($vmid);
4400     my $io_uring = min_version($kvmver, 6, 0);
4401     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4402     $drive =~ s/\\/\\\\/g;
4403     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4404
4405     # If the command succeeds qemu prints: "OK"
4406     return 1 if $ret =~ m/OK/s;
4407
4408     die "adding drive failed: $ret\n";
4409 }
4410
4411 sub qemu_drivedel {
4412     my ($vmid, $deviceid) = @_;
4413
4414     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4415     $ret =~ s/^\s+//;
4416
4417     return 1 if $ret eq "";
4418
4419     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4420     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4421
4422     die "deleting drive $deviceid failed : $ret\n";
4423 }
4424
4425 sub qemu_deviceaddverify {
4426     my ($vmid, $deviceid) = @_;
4427
4428     for (my $i = 0; $i <= 5; $i++) {
4429          my $devices_list = vm_devices_list($vmid);
4430          return 1 if defined($devices_list->{$deviceid});
4431          sleep 1;
4432     }
4433
4434     die "error on hotplug device '$deviceid'\n";
4435 }
4436
4437
4438 sub qemu_devicedelverify {
4439     my ($vmid, $deviceid) = @_;
4440
4441     # need to verify that the device is correctly removed as device_del
4442     # is async and empty return is not reliable
4443
4444     for (my $i = 0; $i <= 5; $i++) {
4445          my $devices_list = vm_devices_list($vmid);
4446          return 1 if !defined($devices_list->{$deviceid});
4447          sleep 1;
4448     }
4449
4450     die "error on hot-unplugging device '$deviceid'\n";
4451 }
4452
4453 sub qemu_findorcreatescsihw {
4454     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $scsihwid="$controller_prefix$controller";
4459     my $devices_list = vm_devices_list($vmid);
4460
4461     if (!defined($devices_list->{$scsihwid})) {
4462         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4463     }
4464
4465     return 1;
4466 }
4467
4468 sub qemu_deletescsihw {
4469     my ($conf, $vmid, $opt) = @_;
4470
4471     my $device = parse_drive($opt, $conf->{$opt});
4472
4473     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4474         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4475         return 1;
4476     }
4477
4478     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4479
4480     my $devices_list = vm_devices_list($vmid);
4481     foreach my $opt (keys %{$devices_list}) {
4482         if (is_valid_drivename($opt)) {
4483             my $drive = parse_drive($opt, $conf->{$opt});
4484             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4485                 return 1;
4486             }
4487         }
4488     }
4489
4490     my $scsihwid="scsihw$controller";
4491
4492     vm_deviceunplug($vmid, $conf, $scsihwid);
4493
4494     return 1;
4495 }
4496
4497 sub qemu_add_pci_bridge {
4498     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4499
4500     my $bridges = {};
4501
4502     my $bridgeid;
4503
4504     print_pci_addr($device, $bridges, $arch, $machine_type);
4505
4506     while (my ($k, $v) = each %$bridges) {
4507         $bridgeid = $k;
4508     }
4509     return 1 if !defined($bridgeid) || $bridgeid < 1;
4510
4511     my $bridge = "pci.$bridgeid";
4512     my $devices_list = vm_devices_list($vmid);
4513
4514     if (!defined($devices_list->{$bridge})) {
4515         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4516     }
4517
4518     return 1;
4519 }
4520
4521 sub qemu_set_link_status {
4522     my ($vmid, $device, $up) = @_;
4523
4524     mon_cmd($vmid, "set_link", name => $device,
4525                up => $up ? JSON::true : JSON::false);
4526 }
4527
4528 sub qemu_netdevadd {
4529     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4530
4531     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4532     my %options =  split(/[=,]/, $netdev);
4533
4534     if (defined(my $vhost = $options{vhost})) {
4535         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4536     }
4537
4538     if (defined(my $queues = $options{queues})) {
4539         $options{queues} = $queues + 0;
4540     }
4541
4542     mon_cmd($vmid, "netdev_add",  %options);
4543     return 1;
4544 }
4545
4546 sub qemu_netdevdel {
4547     my ($vmid, $deviceid) = @_;
4548
4549     mon_cmd($vmid, "netdev_del", id => $deviceid);
4550 }
4551
4552 sub qemu_usb_hotplug {
4553     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4554
4555     return if !$device;
4556
4557     # remove the old one first
4558     vm_deviceunplug($vmid, $conf, $deviceid);
4559
4560     # check if xhci controller is necessary and available
4561     my $devicelist = vm_devices_list($vmid);
4562
4563     if (!$devicelist->{xhci}) {
4564         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4565         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4566     }
4567
4568     # add the new one
4569     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4570 }
4571
4572 sub qemu_cpu_hotplug {
4573     my ($vmid, $conf, $vcpus) = @_;
4574
4575     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4576
4577     my $sockets = 1;
4578     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4579     $sockets = $conf->{sockets} if  $conf->{sockets};
4580     my $cores = $conf->{cores} || 1;
4581     my $maxcpus = $sockets * $cores;
4582
4583     $vcpus = $maxcpus if !$vcpus;
4584
4585     die "you can't add more vcpus than maxcpus\n"
4586         if $vcpus > $maxcpus;
4587
4588     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4589
4590     if ($vcpus < $currentvcpus) {
4591
4592         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4593
4594             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4595                 qemu_devicedel($vmid, "cpu$i");
4596                 my $retry = 0;
4597                 my $currentrunningvcpus = undef;
4598                 while (1) {
4599                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4601                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4602                     $retry++;
4603                     sleep 1;
4604                 }
4605                 #update conf after each succesfull cpu unplug
4606                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607                 PVE::QemuConfig->write_config($vmid, $conf);
4608             }
4609         } else {
4610             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4611         }
4612
4613         return;
4614     }
4615
4616     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4617     die "vcpus in running vm does not match its configuration\n"
4618         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4619
4620     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4621
4622         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4623             my $cpustr = print_cpu_device($conf, $i);
4624             qemu_deviceadd($vmid, $cpustr);
4625
4626             my $retry = 0;
4627             my $currentrunningvcpus = undef;
4628             while (1) {
4629                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4630                 last if scalar(@{$currentrunningvcpus}) == $i;
4631                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4632                 sleep 1;
4633                 $retry++;
4634             }
4635             #update conf after each succesfull cpu hotplug
4636             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4637             PVE::QemuConfig->write_config($vmid, $conf);
4638         }
4639     } else {
4640
4641         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4642             mon_cmd($vmid, "cpu-add", id => int($i));
4643         }
4644     }
4645 }
4646
4647 sub qemu_block_set_io_throttle {
4648     my ($vmid, $deviceid,
4649         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4650         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4651         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4652         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4653
4654     return if !check_running($vmid) ;
4655
4656     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4657         bps => int($bps),
4658         bps_rd => int($bps_rd),
4659         bps_wr => int($bps_wr),
4660         iops => int($iops),
4661         iops_rd => int($iops_rd),
4662         iops_wr => int($iops_wr),
4663         bps_max => int($bps_max),
4664         bps_rd_max => int($bps_rd_max),
4665         bps_wr_max => int($bps_wr_max),
4666         iops_max => int($iops_max),
4667         iops_rd_max => int($iops_rd_max),
4668         iops_wr_max => int($iops_wr_max),
4669         bps_max_length => int($bps_max_length),
4670         bps_rd_max_length => int($bps_rd_max_length),
4671         bps_wr_max_length => int($bps_wr_max_length),
4672         iops_max_length => int($iops_max_length),
4673         iops_rd_max_length => int($iops_rd_max_length),
4674         iops_wr_max_length => int($iops_wr_max_length),
4675     );
4676
4677 }
4678
4679 sub qemu_block_resize {
4680     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4681
4682     my $running = check_running($vmid);
4683
4684     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4685
4686     return if !$running;
4687
4688     my $padding = (1024 - $size % 1024) % 1024;
4689     $size = $size + $padding;
4690
4691     mon_cmd(
4692         $vmid,
4693         "block_resize",
4694         device => $deviceid,
4695         size => int($size),
4696         timeout => 60,
4697     );
4698 }
4699
4700 sub qemu_volume_snapshot {
4701     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4702
4703     my $running = check_running($vmid);
4704
4705     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4706         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4707     } else {
4708         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4709     }
4710 }
4711
4712 sub qemu_volume_snapshot_delete {
4713     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4714
4715     my $running = check_running($vmid);
4716
4717     if($running) {
4718
4719         $running = undef;
4720         my $conf = PVE::QemuConfig->load_config($vmid);
4721         PVE::QemuConfig->foreach_volume($conf, sub {
4722             my ($ds, $drive) = @_;
4723             $running = 1 if $drive->{file} eq $volid;
4724         });
4725     }
4726
4727     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4728         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4729     } else {
4730         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4731     }
4732 }
4733
4734 sub set_migration_caps {
4735     my ($vmid, $savevm) = @_;
4736
4737     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4738
4739     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4740     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4741
4742     my $cap_ref = [];
4743
4744     my $enabled_cap = {
4745         "auto-converge" => 1,
4746         "xbzrle" => 1,
4747         "x-rdma-pin-all" => 0,
4748         "zero-blocks" => 0,
4749         "compress" => 0,
4750         "dirty-bitmaps" => $dirty_bitmaps,
4751     };
4752
4753     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4754
4755     for my $supported_capability (@$supported_capabilities) {
4756         push @$cap_ref, {
4757             capability => $supported_capability->{capability},
4758             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4759         };
4760     }
4761
4762     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4763 }
4764
4765 sub foreach_volid {
4766     my ($conf, $func, @param) = @_;
4767
4768     my $volhash = {};
4769
4770     my $test_volid = sub {
4771         my ($key, $drive, $snapname, $pending) = @_;
4772
4773         my $volid = $drive->{file};
4774         return if !$volid;
4775
4776         $volhash->{$volid}->{cdrom} //= 1;
4777         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4778
4779         my $replicate = $drive->{replicate} // 1;
4780         $volhash->{$volid}->{replicate} //= 0;
4781         $volhash->{$volid}->{replicate} = 1 if $replicate;
4782
4783         $volhash->{$volid}->{shared} //= 0;
4784         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4785
4786         $volhash->{$volid}->{is_unused} //= 0;
4787         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4788
4789         $volhash->{$volid}->{is_attached} //= 0;
4790         $volhash->{$volid}->{is_attached} = 1
4791             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4792
4793         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4794             if defined($snapname);
4795
4796         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4797
4798         my $size = $drive->{size};
4799         $volhash->{$volid}->{size} //= $size if $size;
4800
4801         $volhash->{$volid}->{is_vmstate} //= 0;
4802         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4803
4804         $volhash->{$volid}->{is_tpmstate} //= 0;
4805         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4806
4807         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4808     };
4809
4810     my $include_opts = {
4811         extra_keys => ['vmstate'],
4812         include_unused => 1,
4813     };
4814
4815     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4816
4817     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4818         if defined($conf->{pending}) && $conf->{pending}->%*;
4819
4820     foreach my $snapname (keys %{$conf->{snapshots}}) {
4821         my $snap = $conf->{snapshots}->{$snapname};
4822         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4823     }
4824
4825     foreach my $volid (keys %$volhash) {
4826         &$func($volid, $volhash->{$volid}, @param);
4827     }
4828 }
4829
4830 my $fast_plug_option = {
4831     'description' => 1,
4832     'hookscript' => 1,
4833     'lock' => 1,
4834     'migrate_downtime' => 1,
4835     'migrate_speed' => 1,
4836     'name' => 1,
4837     'onboot' => 1,
4838     'protection' => 1,
4839     'shares' => 1,
4840     'startup' => 1,
4841     'tags' => 1,
4842     'vmstatestorage' => 1,
4843 };
4844
4845 for my $opt (keys %$confdesc_cloudinit) {
4846     $fast_plug_option->{$opt} = 1;
4847 };
4848
4849 # hotplug changes in [PENDING]
4850 # $selection hash can be used to only apply specified options, for
4851 # example: { cores => 1 } (only apply changed 'cores')
4852 # $errors ref is used to return error messages
4853 sub vmconfig_hotplug_pending {
4854     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4855
4856     my $defaults = load_defaults();
4857     my $arch = get_vm_arch($conf);
4858     my $machine_type = get_vm_machine($conf, undef, $arch);
4859
4860     # commit values which do not have any impact on running VM first
4861     # Note: those option cannot raise errors, we we do not care about
4862     # $selection and always apply them.
4863
4864     my $add_error = sub {
4865         my ($opt, $msg) = @_;
4866         $errors->{$opt} = "hotplug problem - $msg";
4867     };
4868
4869     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4870
4871     my $cloudinit_record_changed = sub {
4872         my ($conf, $opt, $old, $new) = @_;
4873         return if !$cloudinit_pending_properties->{$opt};
4874
4875         my $ci = ($conf->{cloudinit} //= {});
4876
4877         my $recorded = $ci->{$opt};
4878         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4879
4880         if (defined($new)) {
4881             if (defined($old)) {
4882                 # an existing value is being modified
4883                 if (defined($recorded)) {
4884                     # the value was already not in sync
4885                     if ($new eq $recorded) {
4886                         # a value is being reverted to the cloud-init state:
4887                         delete $ci->{$opt};
4888                         delete $added{$opt};
4889                     } else {
4890                         # the value was changed multiple times, do nothing
4891                     }
4892                 } elsif ($added{$opt}) {
4893                     # the value had been marked as added and is being changed, do nothing
4894                 } else {
4895                     # the value is new, record it:
4896                     $ci->{$opt} = $old;
4897                 }
4898             } else {
4899                 # a new value is being added
4900                 if (defined($recorded)) {
4901                     # it was already not in sync
4902                     if ($new eq $recorded) {
4903                         # a value is being reverted to the cloud-init state:
4904                         delete $ci->{$opt};
4905                         delete $added{$opt};
4906                     } else {
4907                         # the value had temporarily been removed, do nothing
4908                     }
4909                 } elsif ($added{$opt}) {
4910                     # the value had been marked as added already, do nothing
4911                 } else {
4912                     # the value is new, add it
4913                     $added{$opt} = 1;
4914                 }
4915             }
4916         } elsif (!defined($old)) {
4917             # a non-existent value is being removed? ignore...
4918         } else {
4919             # a value is being deleted
4920             if (defined($recorded)) {
4921                 # a value was already recorded, just keep it
4922             } elsif ($added{$opt}) {
4923                 # the value was marked as added, remove it
4924                 delete $added{$opt};
4925             } else {
4926                 # a previously unrecorded value is being removed, record the old value:
4927                 $ci->{$opt} = $old;
4928             }
4929         }
4930
4931         my $added = join(',', sort keys %added);
4932         $ci->{added} = $added if length($added);
4933     };
4934
4935     my $changes = 0;
4936     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4937         if ($fast_plug_option->{$opt}) {
4938             my $new = delete $conf->{pending}->{$opt};
4939             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4940             $conf->{$opt} = $new;
4941             $changes = 1;
4942         }
4943     }
4944
4945     if ($changes) {
4946         PVE::QemuConfig->write_config($vmid, $conf);
4947     }
4948
4949     my $ostype = $conf->{ostype};
4950     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4951     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4952     my $usb_hotplug = $hotplug_features->{usb}
4953         && min_version($version, 7, 1)
4954         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4955
4956     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4957     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4958
4959     foreach my $opt (sort keys %$pending_delete_hash) {
4960         next if $selection && !$selection->{$opt};
4961         my $force = $pending_delete_hash->{$opt}->{force};
4962         eval {
4963             if ($opt eq 'hotplug') {
4964                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4965             } elsif ($opt eq 'tablet') {
4966                 die "skip\n" if !$hotplug_features->{usb};
4967                 if ($defaults->{tablet}) {
4968                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4969                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4970                         if $arch eq 'aarch64';
4971                 } else {
4972                     vm_deviceunplug($vmid, $conf, 'tablet');
4973                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4974                 }
4975             } elsif ($opt =~ m/^usb(\d+)$/) {
4976                 my $index = $1;
4977                 die "skip\n" if !$usb_hotplug;
4978                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4979                 vm_deviceunplug($vmid, $conf, $opt);
4980             } elsif ($opt eq 'vcpus') {
4981                 die "skip\n" if !$hotplug_features->{cpu};
4982                 qemu_cpu_hotplug($vmid, $conf, undef);
4983             } elsif ($opt eq 'balloon') {
4984                 # enable balloon device is not hotpluggable
4985                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4986                 # here we reset the ballooning value to memory
4987                 my $balloon = get_current_memory($conf->{memory});
4988                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4989             } elsif ($fast_plug_option->{$opt}) {
4990                 # do nothing
4991             } elsif ($opt =~ m/^net(\d+)$/) {
4992                 die "skip\n" if !$hotplug_features->{network};
4993                 vm_deviceunplug($vmid, $conf, $opt);
4994             } elsif (is_valid_drivename($opt)) {
4995                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4996                 vm_deviceunplug($vmid, $conf, $opt);
4997                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4998             } elsif ($opt =~ m/^memory$/) {
4999                 die "skip\n" if !$hotplug_features->{memory};
5000                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
5001             } elsif ($opt eq 'cpuunits') {
5002                 $cgroup->change_cpu_shares(undef);
5003             } elsif ($opt eq 'cpulimit') {
5004                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5005             } else {
5006                 die "skip\n";
5007             }
5008         };
5009         if (my $err = $@) {
5010             &$add_error($opt, $err) if $err ne "skip\n";
5011         } else {
5012             my $old = delete $conf->{$opt};
5013             $cloudinit_record_changed->($conf, $opt, $old, undef);
5014             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5015         }
5016     }
5017
5018     my $cloudinit_opt;
5019     foreach my $opt (keys %{$conf->{pending}}) {
5020         next if $selection && !$selection->{$opt};
5021         my $value = $conf->{pending}->{$opt};
5022         eval {
5023             if ($opt eq 'hotplug') {
5024                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5025                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5026             } elsif ($opt eq 'tablet') {
5027                 die "skip\n" if !$hotplug_features->{usb};
5028                 if ($value == 1) {
5029                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5030                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5031                         if $arch eq 'aarch64';
5032                 } elsif ($value == 0) {
5033                     vm_deviceunplug($vmid, $conf, 'tablet');
5034                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5035                 }
5036             } elsif ($opt =~ m/^usb(\d+)$/) {
5037                 my $index = $1;
5038                 die "skip\n" if !$usb_hotplug;
5039                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5040                 my $id = $opt;
5041                 if ($d->{host} =~ m/^spice$/i)  {
5042                     $id = "usbredirdev$index";
5043                 }
5044                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5045             } elsif ($opt eq 'vcpus') {
5046                 die "skip\n" if !$hotplug_features->{cpu};
5047                 qemu_cpu_hotplug($vmid, $conf, $value);
5048             } elsif ($opt eq 'balloon') {
5049                 # enable/disable balloning device is not hotpluggable
5050                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5051                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5052                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5053
5054                 # allow manual ballooning if shares is set to zero
5055                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5056                     my $memory = get_current_memory($conf->{memory});
5057                     my $balloon = $conf->{pending}->{balloon} || $memory;
5058                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5059                 }
5060             } elsif ($opt =~ m/^net(\d+)$/) {
5061                 # some changes can be done without hotplug
5062                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5063                                     $vmid, $opt, $value, $arch, $machine_type);
5064             } elsif (is_valid_drivename($opt)) {
5065                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5066                 # some changes can be done without hotplug
5067                 my $drive = parse_drive($opt, $value);
5068                 if (drive_is_cloudinit($drive)) {
5069                     $cloudinit_opt = [$opt, $drive];
5070                     # apply all the other changes first, then generate the cloudinit disk
5071                     die "skip\n";
5072                 }
5073                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5074                                      $vmid, $opt, $value, $arch, $machine_type);
5075             } elsif ($opt =~ m/^memory$/) { #dimms
5076                 die "skip\n" if !$hotplug_features->{memory};
5077                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5078             } elsif ($opt eq 'cpuunits') {
5079                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5080                 $cgroup->change_cpu_shares($new_cpuunits);
5081             } elsif ($opt eq 'cpulimit') {
5082                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5083                 $cgroup->change_cpu_quota($cpulimit, 100000);
5084             } elsif ($opt eq 'agent') {
5085                 vmconfig_update_agent($conf, $opt, $value);
5086             } else {
5087                 die "skip\n";  # skip non-hot-pluggable options
5088             }
5089         };
5090         if (my $err = $@) {
5091             &$add_error($opt, $err) if $err ne "skip\n";
5092         } else {
5093             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5094             $conf->{$opt} = $value;
5095             delete $conf->{pending}->{$opt};
5096         }
5097     }
5098
5099     if (defined($cloudinit_opt)) {
5100         my ($opt, $drive) = @$cloudinit_opt;
5101         my $value = $conf->{pending}->{$opt};
5102         eval {
5103             my $temp = {%$conf, $opt => $value};
5104             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5105             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5106                                  $vmid, $opt, $value, $arch, $machine_type);
5107         };
5108         if (my $err = $@) {
5109             &$add_error($opt, $err) if $err ne "skip\n";
5110         } else {
5111             $conf->{$opt} = $value;
5112             delete $conf->{pending}->{$opt};
5113         }
5114     }
5115
5116     # unplug xhci controller if no usb device is left
5117     if ($usb_hotplug) {
5118         my $has_usb = 0;
5119         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5120             next if !defined($conf->{"usb$i"});
5121             $has_usb = 1;
5122             last;
5123         }
5124         if (!$has_usb) {
5125             vm_deviceunplug($vmid, $conf, 'xhci');
5126         }
5127     }
5128
5129     PVE::QemuConfig->write_config($vmid, $conf);
5130
5131     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5132         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5133     }
5134 }
5135
5136 sub try_deallocate_drive {
5137     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5138
5139     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5140         my $volid = $drive->{file};
5141         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5142             my $sid = PVE::Storage::parse_volume_id($volid);
5143             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5144
5145             # check if the disk is really unused
5146             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5147                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5148             PVE::Storage::vdisk_free($storecfg, $volid);
5149             return 1;
5150         } else {
5151             # If vm is not owner of this disk remove from config
5152             return 1;
5153         }
5154     }
5155
5156     return;
5157 }
5158
5159 sub vmconfig_delete_or_detach_drive {
5160     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5161
5162     my $drive = parse_drive($opt, $conf->{$opt});
5163
5164     my $rpcenv = PVE::RPCEnvironment::get();
5165     my $authuser = $rpcenv->get_user();
5166
5167     if ($force) {
5168         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5169         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5170     } else {
5171         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5172     }
5173 }
5174
5175
5176
5177 sub vmconfig_apply_pending {
5178     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5179
5180     return if !scalar(keys %{$conf->{pending}});
5181
5182     my $add_apply_error = sub {
5183         my ($opt, $msg) = @_;
5184         my $err_msg = "unable to apply pending change $opt : $msg";
5185         $errors->{$opt} = $err_msg;
5186         warn $err_msg;
5187     };
5188
5189     # cold plug
5190
5191     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5192     foreach my $opt (sort keys %$pending_delete_hash) {
5193         my $force = $pending_delete_hash->{$opt}->{force};
5194         eval {
5195             if ($opt =~ m/^unused/) {
5196                 die "internal error";
5197             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5198                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5199             }
5200         };
5201         if (my $err = $@) {
5202             $add_apply_error->($opt, $err);
5203         } else {
5204             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5205             delete $conf->{$opt};
5206         }
5207     }
5208
5209     PVE::QemuConfig->cleanup_pending($conf);
5210
5211     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5212
5213     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5214         next if $opt eq 'delete'; # just to be sure
5215         eval {
5216             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5217                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5218             }
5219         };
5220         if (my $err = $@) {
5221             $add_apply_error->($opt, $err);
5222         } else {
5223
5224             if (is_valid_drivename($opt)) {
5225                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5226                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5227             }
5228
5229             $conf->{$opt} = delete $conf->{pending}->{$opt};
5230         }
5231     }
5232
5233     # write all changes at once to avoid unnecessary i/o
5234     PVE::QemuConfig->write_config($vmid, $conf);
5235     if ($generate_cloudinit) {
5236         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5237             # After successful generation and if there were changes to be applied, update the
5238             # config to drop the {cloudinit} entry.
5239             PVE::QemuConfig->write_config($vmid, $conf);
5240         }
5241     }
5242 }
5243
5244 sub vmconfig_update_net {
5245     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5246
5247     my $newnet = parse_net($value);
5248
5249     if ($conf->{$opt}) {
5250         my $oldnet = parse_net($conf->{$opt});
5251
5252         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5253             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5254             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5255             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5256             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5257
5258             # for non online change, we try to hot-unplug
5259             die "skip\n" if !$hotplug;
5260             vm_deviceunplug($vmid, $conf, $opt);
5261         } else {
5262
5263             die "internal error" if $opt !~ m/net(\d+)/;
5264             my $iface = "tap${vmid}i$1";
5265
5266             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5267                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5268                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5269                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5270                 PVE::Network::tap_unplug($iface);
5271
5272                 if ($have_sdn) {
5273                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5274                 } else {
5275                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5276                 }
5277             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5278                 # Rate can be applied on its own but any change above needs to
5279                 # include the rate in tap_plug since OVS resets everything.
5280                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5281             }
5282
5283             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5284                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5285             }
5286
5287             return 1;
5288         }
5289     }
5290
5291     if ($hotplug) {
5292         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5293     } else {
5294         die "skip\n";
5295     }
5296 }
5297
5298 sub vmconfig_update_agent {
5299     my ($conf, $opt, $value) = @_;
5300
5301     die "skip\n" if !$conf->{$opt};
5302
5303     my $hotplug_options = { fstrim_cloned_disks => 1 };
5304
5305     my $old_agent = parse_guest_agent($conf);
5306     my $agent = parse_guest_agent({$opt => $value});
5307
5308     for my $option (keys %$agent) { # added/changed options
5309         next if defined($hotplug_options->{$option});
5310         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5311     }
5312
5313     for my $option (keys %$old_agent) { # removed options
5314         next if defined($hotplug_options->{$option});
5315         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5316     }
5317
5318     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5319 }
5320
5321 sub vmconfig_update_disk {
5322     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5323
5324     my $drive = parse_drive($opt, $value);
5325
5326     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5327         my $media = $drive->{media} || 'disk';
5328         my $oldmedia = $old_drive->{media} || 'disk';
5329         die "unable to change media type\n" if $media ne $oldmedia;
5330
5331         if (!drive_is_cdrom($old_drive)) {
5332
5333             if ($drive->{file} ne $old_drive->{file}) {
5334
5335                 die "skip\n" if !$hotplug;
5336
5337                 # unplug and register as unused
5338                 vm_deviceunplug($vmid, $conf, $opt);
5339                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5340
5341             } else {
5342                 # update existing disk
5343
5344                 # skip non hotpluggable value
5345                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5346                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5347                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5348                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5349                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5350                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5351                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5352                     die "skip\n";
5353                 }
5354
5355                 # apply throttle
5356                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5357                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5358                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5359                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5360                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5361                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5362                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5363                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5364                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5365                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5366                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5367                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5368                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5369                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5370                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5371                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5372                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5373                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5374
5375                     qemu_block_set_io_throttle(
5376                         $vmid,"drive-$opt",
5377                         ($drive->{mbps} || 0)*1024*1024,
5378                         ($drive->{mbps_rd} || 0)*1024*1024,
5379                         ($drive->{mbps_wr} || 0)*1024*1024,
5380                          $drive->{iops} || 0,
5381                          $drive->{iops_rd} || 0,
5382                          $drive->{iops_wr} || 0,
5383                         ($drive->{mbps_max} || 0)*1024*1024,
5384                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5385                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5386                          $drive->{iops_max} || 0,
5387                          $drive->{iops_rd_max} || 0,
5388                          $drive->{iops_wr_max} || 0,
5389                          $drive->{bps_max_length} || 1,
5390                          $drive->{bps_rd_max_length} || 1,
5391                          $drive->{bps_wr_max_length} || 1,
5392                          $drive->{iops_max_length} || 1,
5393                          $drive->{iops_rd_max_length} || 1,
5394                          $drive->{iops_wr_max_length} || 1,
5395                     );
5396
5397                 }
5398
5399                 return 1;
5400             }
5401
5402         } else { # cdrom
5403
5404             if ($drive->{file} eq 'none') {
5405                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5406                 if (drive_is_cloudinit($old_drive)) {
5407                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5408                 }
5409             } else {
5410                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5411
5412                 # force eject if locked
5413                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5414
5415                 if ($path) {
5416                     mon_cmd($vmid, "blockdev-change-medium",
5417                         id => "$opt", filename => "$path");
5418                 }
5419             }
5420
5421             return 1;
5422         }
5423     }
5424
5425     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5426     # hotplug new disks
5427     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5428     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5429 }
5430
5431 sub vmconfig_update_cloudinit_drive {
5432     my ($storecfg, $conf, $vmid) = @_;
5433
5434     my $cloudinit_ds = undef;
5435     my $cloudinit_drive = undef;
5436
5437     PVE::QemuConfig->foreach_volume($conf, sub {
5438         my ($ds, $drive) = @_;
5439         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5440             $cloudinit_ds = $ds;
5441             $cloudinit_drive = $drive;
5442         }
5443     });
5444
5445     return if !$cloudinit_drive;
5446
5447     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5448         PVE::QemuConfig->write_config($vmid, $conf);
5449     }
5450
5451     my $running = PVE::QemuServer::check_running($vmid);
5452
5453     if ($running) {
5454         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5455         if ($path) {
5456             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5457             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5458         }
5459     }
5460 }
5461
5462 # called in locked context by incoming migration
5463 sub vm_migrate_get_nbd_disks {
5464     my ($storecfg, $conf, $replicated_volumes) = @_;
5465
5466     my $local_volumes = {};
5467     PVE::QemuConfig->foreach_volume($conf, sub {
5468         my ($ds, $drive) = @_;
5469
5470         return if drive_is_cdrom($drive);
5471         return if $ds eq 'tpmstate0';
5472
5473         my $volid = $drive->{file};
5474
5475         return if !$volid;
5476
5477         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5478
5479         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5480         return if $scfg->{shared};
5481
5482         my $format = qemu_img_format($scfg, $volname);
5483
5484         # replicated disks re-use existing state via bitmap
5485         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5486         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5487     });
5488     return $local_volumes;
5489 }
5490
5491 # called in locked context by incoming migration
5492 sub vm_migrate_alloc_nbd_disks {
5493     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5494
5495     my $nbd = {};
5496     foreach my $opt (sort keys %$source_volumes) {
5497         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5498
5499         if ($use_existing) {
5500             $nbd->{$opt}->{drivestr} = print_drive($drive);
5501             $nbd->{$opt}->{volid} = $volid;
5502             $nbd->{$opt}->{replicated} = 1;
5503             next;
5504         }
5505
5506         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5507
5508         # order of precedence, filtered by whether storage supports it:
5509         # 1. explicit requested format
5510         # 2. default format of storage
5511         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5512         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5513
5514         my $size = $drive->{size} / 1024;
5515         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5516         my $newdrive = $drive;
5517         $newdrive->{format} = $format;
5518         $newdrive->{file} = $newvolid;
5519         my $drivestr = print_drive($newdrive);
5520         $nbd->{$opt}->{drivestr} = $drivestr;
5521         $nbd->{$opt}->{volid} = $newvolid;
5522     }
5523
5524     return $nbd;
5525 }
5526
5527 # see vm_start_nolock for parameters, additionally:
5528 # migrate_opts:
5529 #   storagemap = parsed storage map for allocating NBD disks
5530 sub vm_start {
5531     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5532
5533     return PVE::QemuConfig->lock_config($vmid, sub {
5534         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5535
5536         die "you can't start a vm if it's a template\n"
5537             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5538
5539         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5540         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5541
5542         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5543
5544         if ($has_backup_lock && $running) {
5545             # a backup is currently running, attempt to start the guest in the
5546             # existing QEMU instance
5547             return vm_resume($vmid);
5548         }
5549
5550         PVE::QemuConfig->check_lock($conf)
5551             if !($params->{skiplock} || $has_suspended_lock);
5552
5553         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5554
5555         die "VM $vmid already running\n" if $running;
5556
5557         if (my $storagemap = $migrate_opts->{storagemap}) {
5558             my $replicated = $migrate_opts->{replicated_volumes};
5559             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5560             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5561
5562             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5563                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5564             }
5565         }
5566
5567         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5568     });
5569 }
5570
5571
5572 # params:
5573 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5574 #   skiplock => 0/1, skip checking for config lock
5575 #   skiptemplate => 0/1, skip checking whether VM is template
5576 #   forcemachine => to force QEMU machine (rollback/migration)
5577 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5578 #   timeout => in seconds
5579 #   paused => start VM in paused state (backup)
5580 #   resume => resume from hibernation
5581 #   pbs-backing => {
5582 #      sata0 => {
5583 #         repository
5584 #         snapshot
5585 #         keyfile
5586 #         archive
5587 #      },
5588 #      virtio2 => ...
5589 #   }
5590 # migrate_opts:
5591 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5592 #   migratedfrom => source node
5593 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5594 #   network => CIDR of migration network
5595 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5596 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5597 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5598 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5599 #       contained in config
5600 sub vm_start_nolock {
5601     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5602
5603     my $statefile = $params->{statefile};
5604     my $resume = $params->{resume};
5605
5606     my $migratedfrom = $migrate_opts->{migratedfrom};
5607     my $migration_type = $migrate_opts->{type};
5608
5609     my $res = {};
5610
5611     # clean up leftover reboot request files
5612     eval { clear_reboot_request($vmid); };
5613     warn $@ if $@;
5614
5615     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5616         vmconfig_apply_pending($vmid, $conf, $storecfg);
5617         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5618     }
5619
5620     # don't regenerate the ISO if the VM is started as part of a live migration
5621     # this way we can reuse the old ISO with the correct config
5622     if (!$migratedfrom) {
5623         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5624             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5625             # $conf->{cloudinit}, so we could just not do this?
5626             # But we do it above, so for now let's be consistent.
5627             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5628         }
5629     }
5630
5631     # override offline migrated volumes, conf is out of date still
5632     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5633         for my $key (sort keys $offline_volumes->%*) {
5634             my $parsed = parse_drive($key, $conf->{$key});
5635             $parsed->{file} = $offline_volumes->{$key};
5636             $conf->{$key} = print_drive($parsed);
5637         }
5638     }
5639
5640     my $defaults = load_defaults();
5641
5642     # set environment variable useful inside network script
5643     # for remote migration the config is available on the target node!
5644     if (!$migrate_opts->{remote_node}) {
5645         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5646     }
5647
5648     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5649
5650     my $forcemachine = $params->{forcemachine};
5651     my $forcecpu = $params->{forcecpu};
5652     if ($resume) {
5653         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5654         $forcemachine = $conf->{runningmachine};
5655         $forcecpu = $conf->{runningcpu};
5656         print "Resuming suspended VM\n";
5657     }
5658
5659     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5660         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5661
5662     my $migration_ip;
5663     my $get_migration_ip = sub {
5664         my ($nodename) = @_;
5665
5666         return $migration_ip if defined($migration_ip);
5667
5668         my $cidr = $migrate_opts->{network};
5669
5670         if (!defined($cidr)) {
5671             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5672             $cidr = $dc_conf->{migration}->{network};
5673         }
5674
5675         if (defined($cidr)) {
5676             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5677
5678             die "could not get IP: no address configured on local " .
5679                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5680
5681             die "could not get IP: multiple addresses configured on local " .
5682                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5683
5684             $migration_ip = @$ips[0];
5685         }
5686
5687         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5688             if !defined($migration_ip);
5689
5690         return $migration_ip;
5691     };
5692
5693     if ($statefile) {
5694         if ($statefile eq 'tcp') {
5695             my $migrate = $res->{migrate} = { proto => 'tcp' };
5696             $migrate->{addr} = "localhost";
5697             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5698             my $nodename = nodename();
5699
5700             if (!defined($migration_type)) {
5701                 if (defined($datacenterconf->{migration}->{type})) {
5702                     $migration_type = $datacenterconf->{migration}->{type};
5703                 } else {
5704                     $migration_type = 'secure';
5705                 }
5706             }
5707
5708             if ($migration_type eq 'insecure') {
5709                 $migrate->{addr} = $get_migration_ip->($nodename);
5710                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5711             }
5712
5713             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5714             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5715             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5716             push @$cmd, '-incoming', $migrate->{uri};
5717             push @$cmd, '-S';
5718
5719         } elsif ($statefile eq 'unix') {
5720             # should be default for secure migrations as a ssh TCP forward
5721             # tunnel is not deterministic reliable ready and fails regurarly
5722             # to set up in time, so use UNIX socket forwards
5723             my $migrate = $res->{migrate} = { proto => 'unix' };
5724             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5725             unlink $migrate->{addr};
5726
5727             $migrate->{uri} = "unix:$migrate->{addr}";
5728             push @$cmd, '-incoming', $migrate->{uri};
5729             push @$cmd, '-S';
5730
5731         } elsif (-e $statefile) {
5732             push @$cmd, '-loadstate', $statefile;
5733         } else {
5734             my $statepath = PVE::Storage::path($storecfg, $statefile);
5735             push @$vollist, $statefile;
5736             push @$cmd, '-loadstate', $statepath;
5737         }
5738     } elsif ($params->{paused}) {
5739         push @$cmd, '-S';
5740     }
5741
5742     my $memory = get_current_memory($conf->{memory});
5743     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5744
5745     my $pci_reserve_list = [];
5746     for my $device (values $pci_devices->%*) {
5747         next if $device->{mdev}; # we don't reserve for mdev devices
5748         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5749     }
5750
5751     # reserve all PCI IDs before actually doing anything with them
5752     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5753
5754     eval {
5755         my $uuid;
5756         for my $id (sort keys %$pci_devices) {
5757             my $d = $pci_devices->{$id};
5758             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5759
5760             my $chosen_mdev;
5761             for my $dev ($d->{ids}->@*) {
5762                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5763                 if ($d->{mdev}) {
5764                     warn $@ if $@;
5765                     $chosen_mdev = $info;
5766                     last if $chosen_mdev; # if successful, we're done
5767                 } else {
5768                     die $@ if $@;
5769                 }
5770             }
5771
5772             next if !$d->{mdev};
5773             die "could not create mediated device\n" if !defined($chosen_mdev);
5774
5775             # nvidia grid needs the uuid of the mdev as qemu parameter
5776             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5777                 if (defined($conf->{smbios1})) {
5778                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5779                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5780                 }
5781                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5782             }
5783         }
5784         push @$cmd, '-uuid', $uuid if defined($uuid);
5785     };
5786     if (my $err = $@) {
5787         eval { cleanup_pci_devices($vmid, $conf) };
5788         warn $@ if $@;
5789         die $err;
5790     }
5791
5792     PVE::Storage::activate_volumes($storecfg, $vollist);
5793
5794
5795     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5796     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5797     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5798     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5799     # timeout should be more than enough here...
5800     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5801
5802     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5803
5804     my %run_params = (
5805         timeout => $statefile ? undef : $start_timeout,
5806         umask => 0077,
5807         noerr => 1,
5808     );
5809
5810     # when migrating, prefix QEMU output so other side can pick up any
5811     # errors that might occur and show the user
5812     if ($migratedfrom) {
5813         $run_params{quiet} = 1;
5814         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5815     }
5816
5817     my %systemd_properties = (
5818         Slice => 'qemu.slice',
5819         KillMode => 'process',
5820         SendSIGKILL => 0,
5821         TimeoutStopUSec => ULONG_MAX, # infinity
5822     );
5823
5824     if (PVE::CGroup::cgroup_mode() == 2) {
5825         $systemd_properties{CPUWeight} = $cpuunits;
5826     } else {
5827         $systemd_properties{CPUShares} = $cpuunits;
5828     }
5829
5830     if (my $cpulimit = $conf->{cpulimit}) {
5831         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5832     }
5833     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5834
5835     my $run_qemu = sub {
5836         PVE::Tools::run_fork sub {
5837             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5838
5839             my $tpmpid;
5840             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5841                 # start the TPM emulator so QEMU can connect on start
5842                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5843             }
5844
5845             my $exitcode = run_command($cmd, %run_params);
5846             if ($exitcode) {
5847                 if ($tpmpid) {
5848                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5849                     kill 'TERM', $tpmpid;
5850                 }
5851                 die "QEMU exited with code $exitcode\n";
5852             }
5853         };
5854     };
5855
5856     if ($conf->{hugepages}) {
5857
5858         my $code = sub {
5859             my $hotplug_features =
5860                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5861             my $hugepages_topology =
5862                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5863
5864             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5865
5866             PVE::QemuServer::Memory::hugepages_mount();
5867             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5868
5869             eval { $run_qemu->() };
5870             if (my $err = $@) {
5871                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5872                     if !$conf->{keephugepages};
5873                 die $err;
5874             }
5875
5876             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5877                 if !$conf->{keephugepages};
5878         };
5879         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5880
5881     } else {
5882         eval { $run_qemu->() };
5883     }
5884
5885     if (my $err = $@) {
5886         # deactivate volumes if start fails
5887         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5888         warn $@ if $@;
5889         eval { cleanup_pci_devices($vmid, $conf) };
5890         warn $@ if $@;
5891
5892         die "start failed: $err";
5893     }
5894
5895     # re-reserve all PCI IDs now that we can know the actual VM PID
5896     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5897     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5898     warn $@ if $@;
5899
5900     if (defined($res->{migrate})) {
5901         print "migration listens on $res->{migrate}->{uri}\n";
5902     } elsif ($statefile) {
5903         eval { mon_cmd($vmid, "cont"); };
5904         warn $@ if $@;
5905     }
5906
5907     #start nbd server for storage migration
5908     if (my $nbd = $migrate_opts->{nbd}) {
5909         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5910
5911         my $migrate_storage_uri;
5912         # nbd_protocol_version > 0 for unix socket support
5913         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5914             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5915             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5916             $migrate_storage_uri = "nbd:unix:$socket_path";
5917             $res->{migrate}->{unix_sockets} = [$socket_path];
5918         } else {
5919             my $nodename = nodename();
5920             my $localip = $get_migration_ip->($nodename);
5921             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5922             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5923
5924             mon_cmd($vmid, "nbd-server-start", addr => {
5925                 type => 'inet',
5926                 data => {
5927                     host => "${localip}",
5928                     port => "${storage_migrate_port}",
5929                 },
5930             });
5931             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5932             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5933         }
5934
5935         my $block_info = mon_cmd($vmid, "query-block");
5936         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5937
5938         foreach my $opt (sort keys %$nbd) {
5939             my $drivestr = $nbd->{$opt}->{drivestr};
5940             my $volid = $nbd->{$opt}->{volid};
5941
5942             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5943
5944             mon_cmd(
5945                 $vmid,
5946                 "block-export-add",
5947                 id => "drive-$opt",
5948                 'node-name' => $block_node,
5949                 writable => JSON::true,
5950                 type => "nbd",
5951                 name => "drive-$opt", # NBD export name
5952             );
5953
5954             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5955             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5956             print "re-using replicated volume: $opt - $volid\n"
5957                 if $nbd->{$opt}->{replicated};
5958
5959             $res->{drives}->{$opt} = $nbd->{$opt};
5960             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5961         }
5962     }
5963
5964     if ($migratedfrom) {
5965         eval {
5966             set_migration_caps($vmid);
5967         };
5968         warn $@ if $@;
5969
5970         if ($spice_port) {
5971             print "spice listens on port $spice_port\n";
5972             $res->{spice_port} = $spice_port;
5973             if ($migrate_opts->{spice_ticket}) {
5974                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5975                     $migrate_opts->{spice_ticket});
5976                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5977             }
5978         }
5979
5980     } else {
5981         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5982             if !$statefile && $conf->{balloon};
5983
5984         foreach my $opt (keys %$conf) {
5985             next if $opt !~  m/^net\d+$/;
5986             my $nicconf = parse_net($conf->{$opt});
5987             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5988         }
5989         add_nets_bridge_fdb($conf, $vmid);
5990     }
5991
5992    if (!defined($conf->{balloon}) || $conf->{balloon}) {
5993         eval {
5994             mon_cmd(
5995                 $vmid,
5996                 'qom-set',
5997                 path => "machine/peripheral/balloon0",
5998                 property => "guest-stats-polling-interval",
5999                 value => 2
6000             );
6001         };
6002         log_warn("could not set polling interval for ballooning - $@") if $@;
6003     }
6004
6005     if ($resume) {
6006         print "Resumed VM, removing state\n";
6007         if (my $vmstate = $conf->{vmstate}) {
6008             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6009             PVE::Storage::vdisk_free($storecfg, $vmstate);
6010         }
6011         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6012         PVE::QemuConfig->write_config($vmid, $conf);
6013     }
6014
6015     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6016
6017     return $res;
6018 }
6019
6020 sub vm_commandline {
6021     my ($storecfg, $vmid, $snapname) = @_;
6022
6023     my $conf = PVE::QemuConfig->load_config($vmid);
6024
6025     my ($forcemachine, $forcecpu);
6026     if ($snapname) {
6027         my $snapshot = $conf->{snapshots}->{$snapname};
6028         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6029
6030         # check for machine or CPU overrides in snapshot
6031         $forcemachine = $snapshot->{runningmachine};
6032         $forcecpu = $snapshot->{runningcpu};
6033
6034         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6035
6036         $conf = $snapshot;
6037     }
6038
6039     my $defaults = load_defaults();
6040
6041     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6042
6043     return PVE::Tools::cmd2string($cmd);
6044 }
6045
6046 sub vm_reset {
6047     my ($vmid, $skiplock) = @_;
6048
6049     PVE::QemuConfig->lock_config($vmid, sub {
6050
6051         my $conf = PVE::QemuConfig->load_config($vmid);
6052
6053         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6054
6055         mon_cmd($vmid, "system_reset");
6056     });
6057 }
6058
6059 sub get_vm_volumes {
6060     my ($conf) = @_;
6061
6062     my $vollist = [];
6063     foreach_volid($conf, sub {
6064         my ($volid, $attr) = @_;
6065
6066         return if $volid =~ m|^/|;
6067
6068         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6069         return if !$sid;
6070
6071         push @$vollist, $volid;
6072     });
6073
6074     return $vollist;
6075 }
6076
6077 sub cleanup_pci_devices {
6078     my ($vmid, $conf) = @_;
6079
6080     foreach my $key (keys %$conf) {
6081         next if $key !~ m/^hostpci(\d+)$/;
6082         my $hostpciindex = $1;
6083         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6084         my $d = parse_hostpci($conf->{$key});
6085         if ($d->{mdev}) {
6086             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6087             # don't want to break ABI just for this two liner
6088             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6089
6090             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6091             # out when we do it first. so wait for 10 seconds and then try it
6092             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6093                 sleep 10;
6094             }
6095
6096             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6097         }
6098     }
6099     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6100 }
6101
6102 sub vm_stop_cleanup {
6103     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6104
6105     eval {
6106
6107         if (!$keepActive) {
6108             my $vollist = get_vm_volumes($conf);
6109             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6110
6111             if (my $tpmdrive = $conf->{tpmstate0}) {
6112                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6113                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6114                 if ($storeid) {
6115                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6116                 }
6117             }
6118         }
6119
6120         foreach my $ext (qw(mon qmp pid vnc qga)) {
6121             unlink "/var/run/qemu-server/${vmid}.$ext";
6122         }
6123
6124         if ($conf->{ivshmem}) {
6125             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6126             # just delete it for now, VMs which have this already open do not
6127             # are affected, but new VMs will get a separated one. If this
6128             # becomes an issue we either add some sort of ref-counting or just
6129             # add a "don't delete on stop" flag to the ivshmem format.
6130             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6131         }
6132
6133         cleanup_pci_devices($vmid, $conf);
6134
6135         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6136     };
6137     warn $@ if $@; # avoid errors - just warn
6138 }
6139
6140 # call only in locked context
6141 sub _do_vm_stop {
6142     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6143
6144     my $pid = check_running($vmid, $nocheck);
6145     return if !$pid;
6146
6147     my $conf;
6148     if (!$nocheck) {
6149         $conf = PVE::QemuConfig->load_config($vmid);
6150         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6151         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6152             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6153             $timeout = $opts->{down} if $opts->{down};
6154         }
6155         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6156     }
6157
6158     eval {
6159         if ($shutdown) {
6160             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6161                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6162             } else {
6163                 mon_cmd($vmid, "system_powerdown");
6164             }
6165         } else {
6166             mon_cmd($vmid, "quit");
6167         }
6168     };
6169     my $err = $@;
6170
6171     if (!$err) {
6172         $timeout = 60 if !defined($timeout);
6173
6174         my $count = 0;
6175         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6176             $count++;
6177             sleep 1;
6178         }
6179
6180         if ($count >= $timeout) {
6181             if ($force) {
6182                 warn "VM still running - terminating now with SIGTERM\n";
6183                 kill 15, $pid;
6184             } else {
6185                 die "VM quit/powerdown failed - got timeout\n";
6186             }
6187         } else {
6188             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6189             return;
6190         }
6191     } else {
6192        if (!check_running($vmid, $nocheck)) {
6193             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6194             return;
6195        }
6196        if ($force) {
6197             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6198             kill 15, $pid;
6199         } else {
6200             die "VM quit/powerdown failed\n";
6201         }
6202     }
6203
6204     # wait again
6205     $timeout = 10;
6206
6207     my $count = 0;
6208     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6209         $count++;
6210         sleep 1;
6211     }
6212
6213     if ($count >= $timeout) {
6214         warn "VM still running - terminating now with SIGKILL\n";
6215         kill 9, $pid;
6216         sleep 1;
6217     }
6218
6219     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6220 }
6221
6222 # Note: use $nocheck to skip tests if VM configuration file exists.
6223 # We need that when migration VMs to other nodes (files already moved)
6224 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6225 sub vm_stop {
6226     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6227
6228     $force = 1 if !defined($force) && !$shutdown;
6229
6230     if ($migratedfrom){
6231         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6232         kill 15, $pid if $pid;
6233         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6234         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6235         return;
6236     }
6237
6238     PVE::QemuConfig->lock_config($vmid, sub {
6239         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6240    });
6241 }
6242
6243 sub vm_reboot {
6244     my ($vmid, $timeout) = @_;
6245
6246     PVE::QemuConfig->lock_config($vmid, sub {
6247         eval {
6248
6249             # only reboot if running, as qmeventd starts it again on a stop event
6250             return if !check_running($vmid);
6251
6252             create_reboot_request($vmid);
6253
6254             my $storecfg = PVE::Storage::config();
6255             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6256
6257         };
6258         if (my $err = $@) {
6259             # avoid that the next normal shutdown will be confused for a reboot
6260             clear_reboot_request($vmid);
6261             die $err;
6262         }
6263    });
6264 }
6265
6266 # note: if using the statestorage parameter, the caller has to check privileges
6267 sub vm_suspend {
6268     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6269
6270     my $conf;
6271     my $path;
6272     my $storecfg;
6273     my $vmstate;
6274
6275     PVE::QemuConfig->lock_config($vmid, sub {
6276
6277         $conf = PVE::QemuConfig->load_config($vmid);
6278
6279         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6280         PVE::QemuConfig->check_lock($conf)
6281             if !($skiplock || $is_backing_up);
6282
6283         die "cannot suspend to disk during backup\n"
6284             if $is_backing_up && $includestate;
6285
6286         if ($includestate) {
6287             $conf->{lock} = 'suspending';
6288             my $date = strftime("%Y-%m-%d", localtime(time()));
6289             $storecfg = PVE::Storage::config();
6290             if (!$statestorage) {
6291                 $statestorage = find_vmstate_storage($conf, $storecfg);
6292                 # check permissions for the storage
6293                 my $rpcenv = PVE::RPCEnvironment::get();
6294                 if ($rpcenv->{type} ne 'cli') {
6295                     my $authuser = $rpcenv->get_user();
6296                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6297                 }
6298             }
6299
6300
6301             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6302                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6303             $path = PVE::Storage::path($storecfg, $vmstate);
6304             PVE::QemuConfig->write_config($vmid, $conf);
6305         } else {
6306             mon_cmd($vmid, "stop");
6307         }
6308     });
6309
6310     if ($includestate) {
6311         # save vm state
6312         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6313
6314         eval {
6315             set_migration_caps($vmid, 1);
6316             mon_cmd($vmid, "savevm-start", statefile => $path);
6317             for(;;) {
6318                 my $state = mon_cmd($vmid, "query-savevm");
6319                 if (!$state->{status}) {
6320                     die "savevm not active\n";
6321                 } elsif ($state->{status} eq 'active') {
6322                     sleep(1);
6323                     next;
6324                 } elsif ($state->{status} eq 'completed') {
6325                     print "State saved, quitting\n";
6326                     last;
6327                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6328                     die "query-savevm failed with error '$state->{error}'\n"
6329                 } else {
6330                     die "query-savevm returned status '$state->{status}'\n";
6331                 }
6332             }
6333         };
6334         my $err = $@;
6335
6336         PVE::QemuConfig->lock_config($vmid, sub {
6337             $conf = PVE::QemuConfig->load_config($vmid);
6338             if ($err) {
6339                 # cleanup, but leave suspending lock, to indicate something went wrong
6340                 eval {
6341                     mon_cmd($vmid, "savevm-end");
6342                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6343                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6344                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6345                     PVE::QemuConfig->write_config($vmid, $conf);
6346                 };
6347                 warn $@ if $@;
6348                 die $err;
6349             }
6350
6351             die "lock changed unexpectedly\n"
6352                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6353
6354             mon_cmd($vmid, "quit");
6355             $conf->{lock} = 'suspended';
6356             PVE::QemuConfig->write_config($vmid, $conf);
6357         });
6358     }
6359 }
6360
6361 # $nocheck is set when called as part of a migration - in this context the
6362 # location of the config file (source or target node) is not deterministic,
6363 # since migration cannot wait for pmxcfs to process the rename
6364 sub vm_resume {
6365     my ($vmid, $skiplock, $nocheck) = @_;
6366
6367     PVE::QemuConfig->lock_config($vmid, sub {
6368         my $res = mon_cmd($vmid, 'query-status');
6369         my $resume_cmd = 'cont';
6370         my $reset = 0;
6371         my $conf;
6372         if ($nocheck) {
6373             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6374             if ($@) {
6375                 my $vmlist = PVE::Cluster::get_vmlist();
6376                 if (exists($vmlist->{ids}->{$vmid})) {
6377                     my $node = $vmlist->{ids}->{$vmid}->{node};
6378                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6379                 }
6380                 if (!$conf) {
6381                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6382                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6383                 }
6384             }
6385         } else {
6386             $conf = PVE::QemuConfig->load_config($vmid);
6387         }
6388
6389         if ($res->{status}) {
6390             return if $res->{status} eq 'running'; # job done, go home
6391             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6392             $reset = 1 if $res->{status} eq 'shutdown';
6393         }
6394
6395         if (!$nocheck) {
6396             PVE::QemuConfig->check_lock($conf)
6397                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6398         }
6399
6400         if ($reset) {
6401             # required if a VM shuts down during a backup and we get a resume
6402             # request before the backup finishes for example
6403             mon_cmd($vmid, "system_reset");
6404         }
6405
6406         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6407
6408         mon_cmd($vmid, $resume_cmd);
6409     });
6410 }
6411
6412 sub vm_sendkey {
6413     my ($vmid, $skiplock, $key) = @_;
6414
6415     PVE::QemuConfig->lock_config($vmid, sub {
6416
6417         my $conf = PVE::QemuConfig->load_config($vmid);
6418
6419         # there is no qmp command, so we use the human monitor command
6420         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6421         die $res if $res ne '';
6422     });
6423 }
6424
6425 sub check_bridge_access {
6426     my ($rpcenv, $authuser, $conf) = @_;
6427
6428     return 1 if $authuser eq 'root@pam';
6429
6430     for my $opt (sort keys $conf->%*) {
6431         next if $opt !~ m/^net\d+$/;
6432         my $net = parse_net($conf->{$opt});
6433         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6434         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6435     }
6436     return 1;
6437 };
6438
6439 sub check_mapping_access {
6440     my ($rpcenv, $user, $conf) = @_;
6441
6442     for my $opt (keys $conf->%*) {
6443         if ($opt =~ m/^usb\d+$/) {
6444             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6445             if (my $host = $device->{host}) {
6446                 die "only root can set '$opt' config for real devices\n"
6447                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6448             } elsif ($device->{mapping}) {
6449                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6450             } else {
6451                 die "either 'host' or 'mapping' must be set.\n";
6452             }
6453         } elsif ($opt =~ m/^hostpci\d+$/) {
6454             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6455             if ($device->{host}) {
6456                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6457             } elsif ($device->{mapping}) {
6458                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6459             } else {
6460                 die "either 'host' or 'mapping' must be set.\n";
6461             }
6462        }
6463    }
6464 };
6465
6466 sub check_restore_permissions {
6467     my ($rpcenv, $user, $conf) = @_;
6468
6469     check_bridge_access($rpcenv, $user, $conf);
6470     check_mapping_access($rpcenv, $user, $conf);
6471 }
6472 # vzdump restore implementaion
6473
6474 sub tar_archive_read_firstfile {
6475     my $archive = shift;
6476
6477     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6478
6479     # try to detect archive type first
6480     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6481         die "unable to open file '$archive'\n";
6482     my $firstfile = <$fh>;
6483     kill 15, $pid;
6484     close $fh;
6485
6486     die "ERROR: archive contaions no data\n" if !$firstfile;
6487     chomp $firstfile;
6488
6489     return $firstfile;
6490 }
6491
6492 sub tar_restore_cleanup {
6493     my ($storecfg, $statfile) = @_;
6494
6495     print STDERR "starting cleanup\n";
6496
6497     if (my $fd = IO::File->new($statfile, "r")) {
6498         while (defined(my $line = <$fd>)) {
6499             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6500                 my $volid = $2;
6501                 eval {
6502                     if ($volid =~ m|^/|) {
6503                         unlink $volid || die 'unlink failed\n';
6504                     } else {
6505                         PVE::Storage::vdisk_free($storecfg, $volid);
6506                     }
6507                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6508                 };
6509                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6510             } else {
6511                 print STDERR "unable to parse line in statfile - $line";
6512             }
6513         }
6514         $fd->close();
6515     }
6516 }
6517
6518 sub restore_file_archive {
6519     my ($archive, $vmid, $user, $opts) = @_;
6520
6521     return restore_vma_archive($archive, $vmid, $user, $opts)
6522         if $archive eq '-';
6523
6524     my $info = PVE::Storage::archive_info($archive);
6525     my $format = $opts->{format} // $info->{format};
6526     my $comp = $info->{compression};
6527
6528     # try to detect archive format
6529     if ($format eq 'tar') {
6530         return restore_tar_archive($archive, $vmid, $user, $opts);
6531     } else {
6532         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6533     }
6534 }
6535
6536 # hepler to remove disks that will not be used after restore
6537 my $restore_cleanup_oldconf = sub {
6538     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6539
6540     my $kept_disks = {};
6541
6542     PVE::QemuConfig->foreach_volume($oldconf, sub {
6543         my ($ds, $drive) = @_;
6544
6545         return if drive_is_cdrom($drive, 1);
6546
6547         my $volid = $drive->{file};
6548         return if !$volid || $volid =~ m|^/|;
6549
6550         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6551         return if !$path || !$owner || ($owner != $vmid);
6552
6553         # Note: only delete disk we want to restore
6554         # other volumes will become unused
6555         if ($virtdev_hash->{$ds}) {
6556             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6557             if (my $err = $@) {
6558                 warn $err;
6559             }
6560         } else {
6561             $kept_disks->{$volid} = 1;
6562         }
6563     });
6564
6565     # after the restore we have no snapshots anymore
6566     for my $snapname (keys $oldconf->{snapshots}->%*) {
6567         my $snap = $oldconf->{snapshots}->{$snapname};
6568         if ($snap->{vmstate}) {
6569             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6570             if (my $err = $@) {
6571                 warn $err;
6572             }
6573         }
6574
6575         for my $volid (keys $kept_disks->%*) {
6576             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6577             warn $@ if $@;
6578         }
6579     }
6580 };
6581
6582 # Helper to parse vzdump backup device hints
6583 #
6584 # $rpcenv: Environment, used to ckeck storage permissions
6585 # $user: User ID, to check storage permissions
6586 # $storecfg: Storage configuration
6587 # $fh: the file handle for reading the configuration
6588 # $devinfo: should contain device sizes for all backu-up'ed devices
6589 # $options: backup options (pool, default storage)
6590 #
6591 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6592 my $parse_backup_hints = sub {
6593     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6594
6595     my $check_storage = sub { # assert if an image can be allocate
6596         my ($storeid, $scfg) = @_;
6597         die "Content type 'images' is not available on storage '$storeid'\n"
6598             if !$scfg->{content}->{images};
6599         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6600             if $user ne 'root@pam';
6601     };
6602
6603     my $virtdev_hash = {};
6604     while (defined(my $line = <$fh>)) {
6605         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6606             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6607             die "archive does not contain data for drive '$virtdev'\n"
6608                 if !$devinfo->{$devname};
6609
6610             if (defined($options->{storage})) {
6611                 $storeid = $options->{storage} || 'local';
6612             } elsif (!$storeid) {
6613                 $storeid = 'local';
6614             }
6615             $format = 'raw' if !$format;
6616             $devinfo->{$devname}->{devname} = $devname;
6617             $devinfo->{$devname}->{virtdev} = $virtdev;
6618             $devinfo->{$devname}->{format} = $format;
6619             $devinfo->{$devname}->{storeid} = $storeid;
6620
6621             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6622             $check_storage->($storeid, $scfg); # permission and content type check
6623
6624             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6625         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6626             my $virtdev = $1;
6627             my $drive = parse_drive($virtdev, $2);
6628
6629             if (drive_is_cloudinit($drive)) {
6630                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6631                 $storeid = $options->{storage} if defined ($options->{storage});
6632                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6633                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6634
6635                 $check_storage->($storeid, $scfg); # permission and content type check
6636
6637                 $virtdev_hash->{$virtdev} = {
6638                     format => $format,
6639                     storeid => $storeid,
6640                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6641                     is_cloudinit => 1,
6642                 };
6643             }
6644         }
6645     }
6646
6647     return $virtdev_hash;
6648 };
6649
6650 # Helper to allocate and activate all volumes required for a restore
6651 #
6652 # $storecfg: Storage configuration
6653 # $virtdev_hash: as returned by parse_backup_hints()
6654 #
6655 # Returns: { $virtdev => $volid }
6656 my $restore_allocate_devices = sub {
6657     my ($storecfg, $virtdev_hash, $vmid) = @_;
6658
6659     my $map = {};
6660     foreach my $virtdev (sort keys %$virtdev_hash) {
6661         my $d = $virtdev_hash->{$virtdev};
6662         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6663         my $storeid = $d->{storeid};
6664         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6665
6666         # test if requested format is supported
6667         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6668         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6669         $d->{format} = $defFormat if !$supported;
6670
6671         my $name;
6672         if ($d->{is_cloudinit}) {
6673             $name = "vm-$vmid-cloudinit";
6674             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6675             if ($scfg->{path}) {
6676                 $name .= ".$d->{format}";
6677             }
6678         }
6679
6680         my $volid = PVE::Storage::vdisk_alloc(
6681             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6682
6683         print STDERR "new volume ID is '$volid'\n";
6684         $d->{volid} = $volid;
6685
6686         PVE::Storage::activate_volumes($storecfg, [$volid]);
6687
6688         $map->{$virtdev} = $volid;
6689     }
6690
6691     return $map;
6692 };
6693
6694 sub restore_update_config_line {
6695     my ($cookie, $map, $line, $unique) = @_;
6696
6697     return '' if $line =~ m/^\#qmdump\#/;
6698     return '' if $line =~ m/^\#vzdump\#/;
6699     return '' if $line =~ m/^lock:/;
6700     return '' if $line =~ m/^unused\d+:/;
6701     return '' if $line =~ m/^parent:/;
6702
6703     my $res = '';
6704
6705     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6706     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6707         # try to convert old 1.X settings
6708         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6709         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6710             my ($model, $macaddr) = split(/\=/, $devconfig);
6711             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6712             my $net = {
6713                 model => $model,
6714                 bridge => "vmbr$ind",
6715                 macaddr => $macaddr,
6716             };
6717             my $netstr = print_net($net);
6718
6719             $res .= "net$cookie->{netcount}: $netstr\n";
6720             $cookie->{netcount}++;
6721         }
6722     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6723         my ($id, $netstr) = ($1, $2);
6724         my $net = parse_net($netstr);
6725         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6726         $netstr = print_net($net);
6727         $res .= "$id: $netstr\n";
6728     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6729         my $virtdev = $1;
6730         my $value = $3;
6731         my $di = parse_drive($virtdev, $value);
6732         if (defined($di->{backup}) && !$di->{backup}) {
6733             $res .= "#$line";
6734         } elsif ($map->{$virtdev}) {
6735             delete $di->{format}; # format can change on restore
6736             $di->{file} = $map->{$virtdev};
6737             $value = print_drive($di);
6738             $res .= "$virtdev: $value\n";
6739         } else {
6740             $res .= $line;
6741         }
6742     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6743         my $vmgenid = $1;
6744         if ($vmgenid ne '0') {
6745             # always generate a new vmgenid if there was a valid one setup
6746             $vmgenid = generate_uuid();
6747         }
6748         $res .= "vmgenid: $vmgenid\n";
6749     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6750         my ($uuid, $uuid_str);
6751         UUID::generate($uuid);
6752         UUID::unparse($uuid, $uuid_str);
6753         my $smbios1 = parse_smbios1($2);
6754         $smbios1->{uuid} = $uuid_str;
6755         $res .= $1.print_smbios1($smbios1)."\n";
6756     } else {
6757         $res .= $line;
6758     }
6759
6760     return $res;
6761 }
6762
6763 my $restore_deactivate_volumes = sub {
6764     my ($storecfg, $virtdev_hash) = @_;
6765
6766     my $vollist = [];
6767     for my $dev (values $virtdev_hash->%*) {
6768         push $vollist->@*, $dev->{volid} if $dev->{volid};
6769     }
6770
6771     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6772     print STDERR $@ if $@;
6773 };
6774
6775 my $restore_destroy_volumes = sub {
6776     my ($storecfg, $virtdev_hash) = @_;
6777
6778     for my $dev (values $virtdev_hash->%*) {
6779         my $volid = $dev->{volid} or next;
6780         eval {
6781             PVE::Storage::vdisk_free($storecfg, $volid);
6782             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6783         };
6784         print STDERR "unable to cleanup '$volid' - $@" if $@;
6785     }
6786 };
6787
6788 sub restore_merge_config {
6789     my ($filename, $backup_conf_raw, $override_conf) = @_;
6790
6791     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6792     for my $key (keys $override_conf->%*) {
6793         $backup_conf->{$key} = $override_conf->{$key};
6794     }
6795
6796     return $backup_conf;
6797 }
6798
6799 sub scan_volids {
6800     my ($cfg, $vmid) = @_;
6801
6802     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6803
6804     my $volid_hash = {};
6805     foreach my $storeid (keys %$info) {
6806         foreach my $item (@{$info->{$storeid}}) {
6807             next if !($item->{volid} && $item->{size});
6808             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6809             $volid_hash->{$item->{volid}} = $item;
6810         }
6811     }
6812
6813     return $volid_hash;
6814 }
6815
6816 sub update_disk_config {
6817     my ($vmid, $conf, $volid_hash) = @_;
6818
6819     my $changes;
6820     my $prefix = "VM $vmid";
6821
6822     # used and unused disks
6823     my $referenced = {};
6824
6825     # Note: it is allowed to define multiple storages with same path (alias), so
6826     # we need to check both 'volid' and real 'path' (two different volid can point
6827     # to the same path).
6828
6829     my $referencedpath = {};
6830
6831     # update size info
6832     PVE::QemuConfig->foreach_volume($conf, sub {
6833         my ($opt, $drive) = @_;
6834
6835         my $volid = $drive->{file};
6836         return if !$volid;
6837         my $volume = $volid_hash->{$volid};
6838
6839         # mark volid as "in-use" for next step
6840         $referenced->{$volid} = 1;
6841         if ($volume && (my $path = $volume->{path})) {
6842             $referencedpath->{$path} = 1;
6843         }
6844
6845         return if drive_is_cdrom($drive);
6846         return if !$volume;
6847
6848         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6849         if (defined($updated)) {
6850             $changes = 1;
6851             $conf->{$opt} = print_drive($updated);
6852             print "$prefix ($opt): $msg\n";
6853         }
6854     });
6855
6856     # remove 'unusedX' entry if volume is used
6857     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6858         my ($opt, $drive) = @_;
6859
6860         my $volid = $drive->{file};
6861         return if !$volid;
6862
6863         my $path;
6864         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6865         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6866             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6867             $changes = 1;
6868             delete $conf->{$opt};
6869         }
6870
6871         $referenced->{$volid} = 1;
6872         $referencedpath->{$path} = 1 if $path;
6873     });
6874
6875     foreach my $volid (sort keys %$volid_hash) {
6876         next if $volid =~ m/vm-$vmid-state-/;
6877         next if $referenced->{$volid};
6878         my $path = $volid_hash->{$volid}->{path};
6879         next if !$path; # just to be sure
6880         next if $referencedpath->{$path};
6881         $changes = 1;
6882         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6883         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6884         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6885     }
6886
6887     return $changes;
6888 }
6889
6890 sub rescan {
6891     my ($vmid, $nolock, $dryrun) = @_;
6892
6893     my $cfg = PVE::Storage::config();
6894
6895     print "rescan volumes...\n";
6896     my $volid_hash = scan_volids($cfg, $vmid);
6897
6898     my $updatefn =  sub {
6899         my ($vmid) = @_;
6900
6901         my $conf = PVE::QemuConfig->load_config($vmid);
6902
6903         PVE::QemuConfig->check_lock($conf);
6904
6905         my $vm_volids = {};
6906         foreach my $volid (keys %$volid_hash) {
6907             my $info = $volid_hash->{$volid};
6908             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6909         }
6910
6911         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6912
6913         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6914     };
6915
6916     if (defined($vmid)) {
6917         if ($nolock) {
6918             &$updatefn($vmid);
6919         } else {
6920             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6921         }
6922     } else {
6923         my $vmlist = config_list();
6924         foreach my $vmid (keys %$vmlist) {
6925             if ($nolock) {
6926                 &$updatefn($vmid);
6927             } else {
6928                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6929             }
6930         }
6931     }
6932 }
6933
6934 sub restore_proxmox_backup_archive {
6935     my ($archive, $vmid, $user, $options) = @_;
6936
6937     my $storecfg = PVE::Storage::config();
6938
6939     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6940     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6941
6942     my $fingerprint = $scfg->{fingerprint};
6943     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6944
6945     my $repo = PVE::PBSClient::get_repository($scfg);
6946     my $namespace = $scfg->{namespace};
6947
6948     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6949     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6950     local $ENV{PBS_PASSWORD} = $password;
6951     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6952
6953     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6954         PVE::Storage::parse_volname($storecfg, $archive);
6955
6956     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6957
6958     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6959
6960     my $tmpdir = "/var/tmp/vzdumptmp$$";
6961     rmtree $tmpdir;
6962     mkpath $tmpdir;
6963
6964     my $conffile = PVE::QemuConfig->config_file($vmid);
6965      # disable interrupts (always do cleanups)
6966     local $SIG{INT} =
6967         local $SIG{TERM} =
6968         local $SIG{QUIT} =
6969         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6970
6971     # Note: $oldconf is undef if VM does not exists
6972     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6973     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6974     my $new_conf_raw = '';
6975
6976     my $rpcenv = PVE::RPCEnvironment::get();
6977     my $devinfo = {}; # info about drives included in backup
6978     my $virtdev_hash = {}; # info about allocated drives
6979
6980     eval {
6981         # enable interrupts
6982         local $SIG{INT} =
6983             local $SIG{TERM} =
6984             local $SIG{QUIT} =
6985             local $SIG{HUP} =
6986             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6987
6988         my $cfgfn = "$tmpdir/qemu-server.conf";
6989         my $firewall_config_fn = "$tmpdir/fw.conf";
6990         my $index_fn = "$tmpdir/index.json";
6991
6992         my $cmd = "restore";
6993
6994         my $param = [$pbs_backup_name, "index.json", $index_fn];
6995         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6996         my $index = PVE::Tools::file_get_contents($index_fn);
6997         $index = decode_json($index);
6998
6999         foreach my $info (@{$index->{files}}) {
7000             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7001                 my $devname = $1;
7002                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7003                     $devinfo->{$devname}->{size} = $1;
7004                 } else {
7005                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7006                 }
7007             }
7008         }
7009
7010         my $is_qemu_server_backup = scalar(
7011             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7012         );
7013         if (!$is_qemu_server_backup) {
7014             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7015         }
7016         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7017
7018         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7019         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7020
7021         if ($has_firewall_config) {
7022             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7023             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7024
7025             my $pve_firewall_dir = '/etc/pve/firewall';
7026             mkdir $pve_firewall_dir; # make sure the dir exists
7027             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7028         }
7029
7030         my $fh = IO::File->new($cfgfn, "r") ||
7031             die "unable to read qemu-server.conf - $!\n";
7032
7033         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7034
7035         # fixme: rate limit?
7036
7037         # create empty/temp config
7038         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7039
7040         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7041
7042         # allocate volumes
7043         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7044
7045         foreach my $virtdev (sort keys %$virtdev_hash) {
7046             my $d = $virtdev_hash->{$virtdev};
7047             next if $d->{is_cloudinit}; # no need to restore cloudinit
7048
7049             # this fails if storage is unavailable
7050             my $volid = $d->{volid};
7051             my $path = PVE::Storage::path($storecfg, $volid);
7052
7053             # for live-restore we only want to preload the efidisk and TPM state
7054             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7055
7056             my @ns_arg;
7057             if (defined(my $ns = $scfg->{namespace})) {
7058                 @ns_arg = ('--ns', $ns);
7059             }
7060
7061             my $pbs_restore_cmd = [
7062                 '/usr/bin/pbs-restore',
7063                 '--repository', $repo,
7064                 @ns_arg,
7065                 $pbs_backup_name,
7066                 "$d->{devname}.img.fidx",
7067                 $path,
7068                 '--verbose',
7069                 ];
7070
7071             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7072             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7073
7074             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7075                 push @$pbs_restore_cmd, '--skip-zero';
7076             }
7077
7078             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7079             print "restore proxmox backup image: $dbg_cmdstring\n";
7080             run_command($pbs_restore_cmd);
7081         }
7082
7083         $fh->seek(0, 0) || die "seek failed - $!\n";
7084
7085         my $cookie = { netcount => 0 };
7086         while (defined(my $line = <$fh>)) {
7087             $new_conf_raw .= restore_update_config_line(
7088                 $cookie,
7089                 $map,
7090                 $line,
7091                 $options->{unique},
7092             );
7093         }
7094
7095         $fh->close();
7096     };
7097     my $err = $@;
7098
7099     if ($err || !$options->{live}) {
7100         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7101     }
7102
7103     rmtree $tmpdir;
7104
7105     if ($err) {
7106         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7107         die $err;
7108     }
7109
7110     if ($options->{live}) {
7111         # keep lock during live-restore
7112         $new_conf_raw .= "\nlock: create";
7113     }
7114
7115     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7116     check_restore_permissions($rpcenv, $user, $new_conf);
7117     PVE::QemuConfig->write_config($vmid, $new_conf);
7118
7119     eval { rescan($vmid, 1); };
7120     warn $@ if $@;
7121
7122     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7123
7124     if ($options->{live}) {
7125         # enable interrupts
7126         local $SIG{INT} =
7127             local $SIG{TERM} =
7128             local $SIG{QUIT} =
7129             local $SIG{HUP} =
7130             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7131
7132         my $conf = PVE::QemuConfig->load_config($vmid);
7133         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7134
7135         # these special drives are already restored before start
7136         delete $devinfo->{'drive-efidisk0'};
7137         delete $devinfo->{'drive-tpmstate0-backup'};
7138
7139         my $pbs_opts = {
7140             repo => $repo,
7141             keyfile => $keyfile,
7142             snapshot => $pbs_backup_name,
7143             namespace => $namespace,
7144         };
7145         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7146
7147         PVE::QemuConfig->remove_lock($vmid, "create");
7148     }
7149 }
7150
7151 sub pbs_live_restore {
7152     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7153
7154     print "starting VM for live-restore\n";
7155     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7156
7157     my $pbs_backing = {};
7158     for my $ds (keys %$restored_disks) {
7159         $ds =~ m/^drive-(.*)$/;
7160         my $confname = $1;
7161         $pbs_backing->{$confname} = {
7162             repository => $opts->{repo},
7163             snapshot => $opts->{snapshot},
7164             archive => "$ds.img.fidx",
7165         };
7166         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7167         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7168
7169         my $drive = parse_drive($confname, $conf->{$confname});
7170         print "restoring '$ds' to '$drive->{file}'\n";
7171     }
7172
7173     my $drives_streamed = 0;
7174     eval {
7175         # make sure HA doesn't interrupt our restore by stopping the VM
7176         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7177             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7178         }
7179
7180         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7181         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7182         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7183
7184         my $qmeventd_fd = register_qmeventd_handle($vmid);
7185
7186         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7187         # this will effectively collapse the backing image chain consisting of
7188         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7189         # removes itself once all backing images vanish with 'auto-remove=on')
7190         my $jobs = {};
7191         for my $ds (sort keys %$restored_disks) {
7192             my $job_id = "restore-$ds";
7193             mon_cmd($vmid, 'block-stream',
7194                 'job-id' => $job_id,
7195                 device => "$ds",
7196             );
7197             $jobs->{$job_id} = {};
7198         }
7199
7200         mon_cmd($vmid, 'cont');
7201         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7202
7203         print "restore-drive jobs finished successfully, removing all tracking block devices"
7204             ." to disconnect from Proxmox Backup Server\n";
7205
7206         for my $ds (sort keys %$restored_disks) {
7207             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7208         }
7209
7210         close($qmeventd_fd);
7211     };
7212
7213     my $err = $@;
7214
7215     if ($err) {
7216         warn "An error occurred during live-restore: $err\n";
7217         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7218         die "live-restore failed\n";
7219     }
7220 }
7221
7222 sub restore_vma_archive {
7223     my ($archive, $vmid, $user, $opts, $comp) = @_;
7224
7225     my $readfrom = $archive;
7226
7227     my $cfg = PVE::Storage::config();
7228     my $commands = [];
7229     my $bwlimit = $opts->{bwlimit};
7230
7231     my $dbg_cmdstring = '';
7232     my $add_pipe = sub {
7233         my ($cmd) = @_;
7234         push @$commands, $cmd;
7235         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7236         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7237         $readfrom = '-';
7238     };
7239
7240     my $input = undef;
7241     if ($archive eq '-') {
7242         $input = '<&STDIN';
7243     } else {
7244         # If we use a backup from a PVE defined storage we also consider that
7245         # storage's rate limit:
7246         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7247         if (defined($volid)) {
7248             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7249             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7250             if ($readlimit) {
7251                 print STDERR "applying read rate limit: $readlimit\n";
7252                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7253                 $add_pipe->($cstream);
7254             }
7255         }
7256     }
7257
7258     if ($comp) {
7259         my $info = PVE::Storage::decompressor_info('vma', $comp);
7260         my $cmd = $info->{decompressor};
7261         push @$cmd, $readfrom;
7262         $add_pipe->($cmd);
7263     }
7264
7265     my $tmpdir = "/var/tmp/vzdumptmp$$";
7266     rmtree $tmpdir;
7267
7268     # disable interrupts (always do cleanups)
7269     local $SIG{INT} =
7270         local $SIG{TERM} =
7271         local $SIG{QUIT} =
7272         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7273
7274     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7275     POSIX::mkfifo($mapfifo, 0600);
7276     my $fifofh;
7277     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7278
7279     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7280
7281     my $devinfo = {}; # info about drives included in backup
7282     my $virtdev_hash = {}; # info about allocated drives
7283
7284     my $rpcenv = PVE::RPCEnvironment::get();
7285
7286     my $conffile = PVE::QemuConfig->config_file($vmid);
7287
7288     # Note: $oldconf is undef if VM does not exist
7289     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7290     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7291     my $new_conf_raw = '';
7292
7293     my %storage_limits;
7294
7295     my $print_devmap = sub {
7296         my $cfgfn = "$tmpdir/qemu-server.conf";
7297
7298         # we can read the config - that is already extracted
7299         my $fh = IO::File->new($cfgfn, "r") ||
7300             die "unable to read qemu-server.conf - $!\n";
7301
7302         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7303         if (-f $fwcfgfn) {
7304             my $pve_firewall_dir = '/etc/pve/firewall';
7305             mkdir $pve_firewall_dir; # make sure the dir exists
7306             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7307         }
7308
7309         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7310
7311         foreach my $info (values %{$virtdev_hash}) {
7312             my $storeid = $info->{storeid};
7313             next if defined($storage_limits{$storeid});
7314
7315             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7316             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7317             $storage_limits{$storeid} = $limit * 1024;
7318         }
7319
7320         foreach my $devname (keys %$devinfo) {
7321             die "found no device mapping information for device '$devname'\n"
7322                 if !$devinfo->{$devname}->{virtdev};
7323         }
7324
7325         # create empty/temp config
7326         if ($oldconf) {
7327             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7328             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7329         }
7330
7331         # allocate volumes
7332         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7333
7334         # print restore information to $fifofh
7335         foreach my $virtdev (sort keys %$virtdev_hash) {
7336             my $d = $virtdev_hash->{$virtdev};
7337             next if $d->{is_cloudinit}; # no need to restore cloudinit
7338
7339             my $storeid = $d->{storeid};
7340             my $volid = $d->{volid};
7341
7342             my $map_opts = '';
7343             if (my $limit = $storage_limits{$storeid}) {
7344                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7345             }
7346
7347             my $write_zeros = 1;
7348             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7349                 $write_zeros = 0;
7350             }
7351
7352             my $path = PVE::Storage::path($cfg, $volid);
7353
7354             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7355
7356             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7357         }
7358
7359         $fh->seek(0, 0) || die "seek failed - $!\n";
7360
7361         my $cookie = { netcount => 0 };
7362         while (defined(my $line = <$fh>)) {
7363             $new_conf_raw .= restore_update_config_line(
7364                 $cookie,
7365                 $map,
7366                 $line,
7367                 $opts->{unique},
7368             );
7369         }
7370
7371         $fh->close();
7372     };
7373
7374     my $oldtimeout;
7375
7376     eval {
7377         # enable interrupts
7378         local $SIG{INT} =
7379             local $SIG{TERM} =
7380             local $SIG{QUIT} =
7381             local $SIG{HUP} =
7382             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7383         local $SIG{ALRM} = sub { die "got timeout\n"; };
7384
7385         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7386
7387         my $parser = sub {
7388             my $line = shift;
7389
7390             print "$line\n";
7391
7392             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7393                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7394                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7395             } elsif ($line =~ m/^CTIME: /) {
7396                 # we correctly received the vma config, so we can disable
7397                 # the timeout now for disk allocation
7398                 alarm($oldtimeout || 0);
7399                 $oldtimeout = undef;
7400                 &$print_devmap();
7401                 print $fifofh "done\n";
7402                 close($fifofh);
7403                 $fifofh = undef;
7404             }
7405         };
7406
7407         print "restore vma archive: $dbg_cmdstring\n";
7408         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7409     };
7410     my $err = $@;
7411
7412     alarm($oldtimeout) if $oldtimeout;
7413
7414     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7415
7416     close($fifofh) if $fifofh;
7417     unlink $mapfifo;
7418     rmtree $tmpdir;
7419
7420     if ($err) {
7421         $restore_destroy_volumes->($cfg, $virtdev_hash);
7422         die $err;
7423     }
7424
7425     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7426     check_restore_permissions($rpcenv, $user, $new_conf);
7427     PVE::QemuConfig->write_config($vmid, $new_conf);
7428
7429     eval { rescan($vmid, 1); };
7430     warn $@ if $@;
7431
7432     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7433 }
7434
7435 sub restore_tar_archive {
7436     my ($archive, $vmid, $user, $opts) = @_;
7437
7438     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7439         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7440         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7441     }
7442
7443     if ($archive ne '-') {
7444         my $firstfile = tar_archive_read_firstfile($archive);
7445         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7446             if $firstfile ne 'qemu-server.conf';
7447     }
7448
7449     my $storecfg = PVE::Storage::config();
7450
7451     # avoid zombie disks when restoring over an existing VM -> cleanup first
7452     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7453     # skiplock=1 because qmrestore has set the 'create' lock itself already
7454     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7455     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7456
7457     my $tocmd = "/usr/lib/qemu-server/qmextract";
7458
7459     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7460     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7461     $tocmd .= ' --prealloc' if $opts->{prealloc};
7462     $tocmd .= ' --info' if $opts->{info};
7463
7464     # tar option "xf" does not autodetect compression when read from STDIN,
7465     # so we pipe to zcat
7466     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7467         PVE::Tools::shellquote("--to-command=$tocmd");
7468
7469     my $tmpdir = "/var/tmp/vzdumptmp$$";
7470     mkpath $tmpdir;
7471
7472     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7473     local $ENV{VZDUMP_VMID} = $vmid;
7474     local $ENV{VZDUMP_USER} = $user;
7475
7476     my $conffile = PVE::QemuConfig->config_file($vmid);
7477     my $new_conf_raw = '';
7478
7479     # disable interrupts (always do cleanups)
7480     local $SIG{INT} =
7481         local $SIG{TERM} =
7482         local $SIG{QUIT} =
7483         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7484
7485     eval {
7486         # enable interrupts
7487         local $SIG{INT} =
7488             local $SIG{TERM} =
7489             local $SIG{QUIT} =
7490             local $SIG{HUP} =
7491             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7492
7493         if ($archive eq '-') {
7494             print "extracting archive from STDIN\n";
7495             run_command($cmd, input => "<&STDIN");
7496         } else {
7497             print "extracting archive '$archive'\n";
7498             run_command($cmd);
7499         }
7500
7501         return if $opts->{info};
7502
7503         # read new mapping
7504         my $map = {};
7505         my $statfile = "$tmpdir/qmrestore.stat";
7506         if (my $fd = IO::File->new($statfile, "r")) {
7507             while (defined (my $line = <$fd>)) {
7508                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7509                     $map->{$1} = $2 if $1;
7510                 } else {
7511                     print STDERR "unable to parse line in statfile - $line\n";
7512                 }
7513             }
7514             $fd->close();
7515         }
7516
7517         my $confsrc = "$tmpdir/qemu-server.conf";
7518
7519         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7520
7521         my $cookie = { netcount => 0 };
7522         while (defined (my $line = <$srcfd>)) {
7523             $new_conf_raw .= restore_update_config_line(
7524                 $cookie,
7525                 $map,
7526                 $line,
7527                 $opts->{unique},
7528             );
7529         }
7530
7531         $srcfd->close();
7532     };
7533     if (my $err = $@) {
7534         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7535         die $err;
7536     }
7537
7538     rmtree $tmpdir;
7539
7540     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7541
7542     PVE::Cluster::cfs_update(); # make sure we read new file
7543
7544     eval { rescan($vmid, 1); };
7545     warn $@ if $@;
7546 };
7547
7548 sub foreach_storage_used_by_vm {
7549     my ($conf, $func) = @_;
7550
7551     my $sidhash = {};
7552
7553     PVE::QemuConfig->foreach_volume($conf, sub {
7554         my ($ds, $drive) = @_;
7555         return if drive_is_cdrom($drive);
7556
7557         my $volid = $drive->{file};
7558
7559         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7560         $sidhash->{$sid} = $sid if $sid;
7561     });
7562
7563     foreach my $sid (sort keys %$sidhash) {
7564         &$func($sid);
7565     }
7566 }
7567
7568 my $qemu_snap_storage = {
7569     rbd => 1,
7570 };
7571 sub do_snapshots_with_qemu {
7572     my ($storecfg, $volid, $deviceid) = @_;
7573
7574     return if $deviceid =~ m/tpmstate0/;
7575
7576     my $storage_name = PVE::Storage::parse_volume_id($volid);
7577     my $scfg = $storecfg->{ids}->{$storage_name};
7578     die "could not find storage '$storage_name'\n" if !defined($scfg);
7579
7580     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7581         return 1;
7582     }
7583
7584     if ($volid =~ m/\.(qcow2|qed)$/){
7585         return 1;
7586     }
7587
7588     return;
7589 }
7590
7591 sub qga_check_running {
7592     my ($vmid, $nowarn) = @_;
7593
7594     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7595     if ($@) {
7596         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7597         return 0;
7598     }
7599     return 1;
7600 }
7601
7602 sub template_create {
7603     my ($vmid, $conf, $disk) = @_;
7604
7605     my $storecfg = PVE::Storage::config();
7606
7607     PVE::QemuConfig->foreach_volume($conf, sub {
7608         my ($ds, $drive) = @_;
7609
7610         return if drive_is_cdrom($drive);
7611         return if $disk && $ds ne $disk;
7612
7613         my $volid = $drive->{file};
7614         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7615
7616         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7617         $drive->{file} = $voliddst;
7618         $conf->{$ds} = print_drive($drive);
7619         PVE::QemuConfig->write_config($vmid, $conf);
7620     });
7621 }
7622
7623 sub convert_iscsi_path {
7624     my ($path) = @_;
7625
7626     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7627         my $portal = $1;
7628         my $target = $2;
7629         my $lun = $3;
7630
7631         my $initiator_name = get_initiator_name();
7632
7633         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7634                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7635     }
7636
7637     die "cannot convert iscsi path '$path', unkown format\n";
7638 }
7639
7640 sub qemu_img_convert {
7641     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7642
7643     my $storecfg = PVE::Storage::config();
7644     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7645     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7646
7647     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7648
7649     my $cachemode;
7650     my $src_path;
7651     my $src_is_iscsi = 0;
7652     my $src_format;
7653
7654     if ($src_storeid) {
7655         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7656         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7657         $src_format = qemu_img_format($src_scfg, $src_volname);
7658         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7659         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7660         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7661     } elsif (-f $src_volid || -b $src_volid) {
7662         $src_path = $src_volid;
7663         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7664             $src_format = $1;
7665         }
7666     }
7667
7668     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7669
7670     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7671     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7672     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7673     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7674
7675     my $cmd = [];
7676     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7677     push @$cmd, '-l', "snapshot.name=$snapname"
7678         if $snapname && $src_format && $src_format eq "qcow2";
7679     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7680     push @$cmd, '-T', $cachemode if defined($cachemode);
7681     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7682
7683     if ($src_is_iscsi) {
7684         push @$cmd, '--image-opts';
7685         $src_path = convert_iscsi_path($src_path);
7686     } elsif ($src_format) {
7687         push @$cmd, '-f', $src_format;
7688     }
7689
7690     if ($dst_is_iscsi) {
7691         push @$cmd, '--target-image-opts';
7692         $dst_path = convert_iscsi_path($dst_path);
7693     } else {
7694         push @$cmd, '-O', $dst_format;
7695     }
7696
7697     push @$cmd, $src_path;
7698
7699     if (!$dst_is_iscsi && $is_zero_initialized) {
7700         push @$cmd, "zeroinit:$dst_path";
7701     } else {
7702         push @$cmd, $dst_path;
7703     }
7704
7705     my $parser = sub {
7706         my $line = shift;
7707         if($line =~ m/\((\S+)\/100\%\)/){
7708             my $percent = $1;
7709             my $transferred = int($size * $percent / 100);
7710             my $total_h = render_bytes($size, 1);
7711             my $transferred_h = render_bytes($transferred, 1);
7712
7713             print "transferred $transferred_h of $total_h ($percent%)\n";
7714         }
7715
7716     };
7717
7718     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7719     my $err = $@;
7720     die "copy failed: $err" if $err;
7721 }
7722
7723 sub qemu_img_format {
7724     my ($scfg, $volname) = @_;
7725
7726     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7727         return $1;
7728     } else {
7729         return "raw";
7730     }
7731 }
7732
7733 sub qemu_drive_mirror {
7734     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7735
7736     $jobs = {} if !$jobs;
7737
7738     my $qemu_target;
7739     my $format;
7740     $jobs->{"drive-$drive"} = {};
7741
7742     if ($dst_volid =~ /^nbd:/) {
7743         $qemu_target = $dst_volid;
7744         $format = "nbd";
7745     } else {
7746         my $storecfg = PVE::Storage::config();
7747         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7748
7749         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7750
7751         $format = qemu_img_format($dst_scfg, $dst_volname);
7752
7753         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7754
7755         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7756     }
7757
7758     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7759     $opts->{format} = $format if $format;
7760
7761     if (defined($src_bitmap)) {
7762         $opts->{sync} = 'incremental';
7763         $opts->{bitmap} = $src_bitmap;
7764         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7765     }
7766
7767     if (defined($bwlimit)) {
7768         $opts->{speed} = $bwlimit * 1024;
7769         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7770     } else {
7771         print "drive mirror is starting for drive-$drive\n";
7772     }
7773
7774     # if a job already runs for this device we get an error, catch it for cleanup
7775     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7776     if (my $err = $@) {
7777         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7778         warn "$@\n" if $@;
7779         die "mirroring error: $err\n";
7780     }
7781
7782     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7783 }
7784
7785 # $completion can be either
7786 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7787 # 'cancel': wait until all jobs are ready, block-job-cancel them
7788 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7789 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7790 sub qemu_drive_mirror_monitor {
7791     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7792
7793     $completion //= 'complete';
7794     $op //= "mirror";
7795
7796     eval {
7797         my $err_complete = 0;
7798
7799         my $starttime = time ();
7800         while (1) {
7801             die "block job ('$op') timed out\n" if $err_complete > 300;
7802
7803             my $stats = mon_cmd($vmid, "query-block-jobs");
7804             my $ctime = time();
7805
7806             my $running_jobs = {};
7807             for my $stat (@$stats) {
7808                 next if $stat->{type} ne $op;
7809                 $running_jobs->{$stat->{device}} = $stat;
7810             }
7811
7812             my $readycounter = 0;
7813
7814             for my $job_id (sort keys %$jobs) {
7815                 my $job = $running_jobs->{$job_id};
7816
7817                 my $vanished = !defined($job);
7818                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7819                 if($complete || ($vanished && $completion eq 'auto')) {
7820                     print "$job_id: $op-job finished\n";
7821                     delete $jobs->{$job_id};
7822                     next;
7823                 }
7824
7825                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7826
7827                 my $busy = $job->{busy};
7828                 my $ready = $job->{ready};
7829                 if (my $total = $job->{len}) {
7830                     my $transferred = $job->{offset} || 0;
7831                     my $remaining = $total - $transferred;
7832                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7833
7834                     my $duration = $ctime - $starttime;
7835                     my $total_h = render_bytes($total, 1);
7836                     my $transferred_h = render_bytes($transferred, 1);
7837
7838                     my $status = sprintf(
7839                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7840                         render_duration($duration),
7841                     );
7842
7843                     if ($ready) {
7844                         if ($busy) {
7845                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7846                         } else {
7847                             $status .= ", ready";
7848                         }
7849                     }
7850                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7851                     $jobs->{$job_id}->{ready} = $ready;
7852                 }
7853
7854                 $readycounter++ if $job->{ready};
7855             }
7856
7857             last if scalar(keys %$jobs) == 0;
7858
7859             if ($readycounter == scalar(keys %$jobs)) {
7860                 print "all '$op' jobs are ready\n";
7861
7862                 # do the complete later (or has already been done)
7863                 last if $completion eq 'skip' || $completion eq 'auto';
7864
7865                 if ($vmiddst && $vmiddst != $vmid) {
7866                     my $agent_running = $qga && qga_check_running($vmid);
7867                     if ($agent_running) {
7868                         print "freeze filesystem\n";
7869                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7870                         warn $@ if $@;
7871                     } else {
7872                         print "suspend vm\n";
7873                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7874                         warn $@ if $@;
7875                     }
7876
7877                     # if we clone a disk for a new target vm, we don't switch the disk
7878                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7879
7880                     if ($agent_running) {
7881                         print "unfreeze filesystem\n";
7882                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7883                         warn $@ if $@;
7884                     } else {
7885                         print "resume vm\n";
7886                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7887                         warn $@ if $@;
7888                     }
7889
7890                     last;
7891                 } else {
7892
7893                     for my $job_id (sort keys %$jobs) {
7894                         # try to switch the disk if source and destination are on the same guest
7895                         print "$job_id: Completing block job_id...\n";
7896
7897                         my $op;
7898                         if ($completion eq 'complete') {
7899                             $op = 'block-job-complete';
7900                         } elsif ($completion eq 'cancel') {
7901                             $op = 'block-job-cancel';
7902                         } else {
7903                             die "invalid completion value: $completion\n";
7904                         }
7905                         eval { mon_cmd($vmid, $op, device => $job_id) };
7906                         if ($@ =~ m/cannot be completed/) {
7907                             print "$job_id: block job cannot be completed, trying again.\n";
7908                             $err_complete++;
7909                         }else {
7910                             print "$job_id: Completed successfully.\n";
7911                             $jobs->{$job_id}->{complete} = 1;
7912                         }
7913                     }
7914                 }
7915             }
7916             sleep 1;
7917         }
7918     };
7919     my $err = $@;
7920
7921     if ($err) {
7922         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7923         die "block job ($op) error: $err";
7924     }
7925 }
7926
7927 sub qemu_blockjobs_cancel {
7928     my ($vmid, $jobs) = @_;
7929
7930     foreach my $job (keys %$jobs) {
7931         print "$job: Cancelling block job\n";
7932         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7933         $jobs->{$job}->{cancel} = 1;
7934     }
7935
7936     while (1) {
7937         my $stats = mon_cmd($vmid, "query-block-jobs");
7938
7939         my $running_jobs = {};
7940         foreach my $stat (@$stats) {
7941             $running_jobs->{$stat->{device}} = $stat;
7942         }
7943
7944         foreach my $job (keys %$jobs) {
7945
7946             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7947                 print "$job: Done.\n";
7948                 delete $jobs->{$job};
7949             }
7950         }
7951
7952         last if scalar(keys %$jobs) == 0;
7953
7954         sleep 1;
7955     }
7956 }
7957
7958 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7959 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7960 my sub clone_disk_check_io_uring {
7961     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7962
7963     return if !$use_drive_mirror;
7964
7965     # Don't complain when not changing storage.
7966     # Assume if it works for the source, it'll work for the target too.
7967     return if $src_storeid eq $dst_storeid;
7968
7969     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7970     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7971
7972     my $cache_direct = drive_uses_cache_direct($src_drive);
7973
7974     my $src_uses_io_uring;
7975     if ($src_drive->{aio}) {
7976         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
7977     } else {
7978         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
7979     }
7980
7981     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
7982         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
7983 }
7984
7985 sub clone_disk {
7986     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7987
7988     my ($vmid, $running) = $source->@{qw(vmid running)};
7989     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7990
7991     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7992     my ($storage, $format) = $dest->@{qw(storage format)};
7993
7994     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7995
7996     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7997         die "cloning from/to EFI disk requires EFI disk\n"
7998             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7999         die "cloning from/to TPM state requires TPM state\n"
8000             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8001
8002         # This would lead to two device nodes in QEMU pointing to the same backing image!
8003         die "cannot change drive name when cloning disk from/to the same VM\n"
8004             if $use_drive_mirror && $vmid == $newvmid;
8005     }
8006
8007     die "cannot move TPM state while VM is running\n"
8008         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8009
8010     my $newvolid;
8011
8012     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8013     print "$src_drivename " if $src_drivename;
8014     print "($drive->{file})\n";
8015
8016     if (!$full) {
8017         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8018         push @$newvollist, $newvolid;
8019     } else {
8020         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8021         my $storeid = $storage || $src_storeid;
8022
8023         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8024
8025         my $name = undef;
8026         my $size = undef;
8027         if (drive_is_cloudinit($drive)) {
8028             $name = "vm-$newvmid-cloudinit";
8029             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8030             if ($scfg->{path}) {
8031                 $name .= ".$dst_format";
8032             }
8033             $snapname = undef;
8034             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8035         } elsif ($dst_drivename eq 'efidisk0') {
8036             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8037         } elsif ($dst_drivename eq 'tpmstate0') {
8038             $dst_format = 'raw';
8039             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8040         } else {
8041             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8042
8043             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8044         }
8045         $newvolid = PVE::Storage::vdisk_alloc(
8046             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8047         );
8048         push @$newvollist, $newvolid;
8049
8050         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8051
8052         if (drive_is_cloudinit($drive)) {
8053             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8054             # if this is the case, we have to complete any block-jobs still there from
8055             # previous drive-mirrors
8056             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8057                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8058             }
8059             goto no_data_clone;
8060         }
8061
8062         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8063         if ($use_drive_mirror) {
8064             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8065                 $completion, $qga, $bwlimit);
8066         } else {
8067             if ($dst_drivename eq 'efidisk0') {
8068                 # the relevant data on the efidisk may be smaller than the source
8069                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8070                 # that is given by the OVMF_VARS.fd
8071                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8072                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8073
8074                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8075
8076                 # better for Ceph if block size is not too small, see bug #3324
8077                 my $bs = 1024*1024;
8078
8079                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8080
8081                 if ($src_format eq 'qcow2' && $snapname) {
8082                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8083                         if !min_version(kvm_user_version(), 6, 2);
8084                     push $cmd->@*, '-l', $snapname;
8085                 }
8086                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8087                 run_command($cmd);
8088             } else {
8089                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8090             }
8091         }
8092     }
8093
8094 no_data_clone:
8095     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8096
8097     my $disk = dclone($drive);
8098     delete $disk->{format};
8099     $disk->{file} = $newvolid;
8100     $disk->{size} = $size if defined($size);
8101
8102     return $disk;
8103 }
8104
8105 sub get_running_qemu_version {
8106     my ($vmid) = @_;
8107     my $res = mon_cmd($vmid, "query-version");
8108     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8109 }
8110
8111 sub qemu_use_old_bios_files {
8112     my ($machine_type) = @_;
8113
8114     return if !$machine_type;
8115
8116     my $use_old_bios_files = undef;
8117
8118     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8119         $machine_type = $1;
8120         $use_old_bios_files = 1;
8121     } else {
8122         my $version = extract_version($machine_type, kvm_user_version());
8123         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8124         # load new efi bios files on migration. So this hack is required to allow
8125         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8126         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8127         $use_old_bios_files = !min_version($version, 2, 4);
8128     }
8129
8130     return ($use_old_bios_files, $machine_type);
8131 }
8132
8133 sub get_efivars_size {
8134     my ($conf, $efidisk) = @_;
8135
8136     my $arch = get_vm_arch($conf);
8137     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8138     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8139     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8140     return -s $ovmf_vars;
8141 }
8142
8143 sub update_efidisk_size {
8144     my ($conf) = @_;
8145
8146     return if !defined($conf->{efidisk0});
8147
8148     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8149     $disk->{size} = get_efivars_size($conf);
8150     $conf->{efidisk0} = print_drive($disk);
8151
8152     return;
8153 }
8154
8155 sub update_tpmstate_size {
8156     my ($conf) = @_;
8157
8158     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8159     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8160     $conf->{tpmstate0} = print_drive($disk);
8161 }
8162
8163 sub create_efidisk($$$$$$$) {
8164     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8165
8166     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8167
8168     my $vars_size_b = -s $ovmf_vars;
8169     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8170     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8171     PVE::Storage::activate_volumes($storecfg, [$volid]);
8172
8173     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8174     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8175
8176     return ($volid, $size/1024);
8177 }
8178
8179 sub vm_iothreads_list {
8180     my ($vmid) = @_;
8181
8182     my $res = mon_cmd($vmid, 'query-iothreads');
8183
8184     my $iothreads = {};
8185     foreach my $iothread (@$res) {
8186         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8187     }
8188
8189     return $iothreads;
8190 }
8191
8192 sub scsihw_infos {
8193     my ($conf, $drive) = @_;
8194
8195     my $maxdev = 0;
8196
8197     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8198         $maxdev = 7;
8199     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8200         $maxdev = 1;
8201     } else {
8202         $maxdev = 256;
8203     }
8204
8205     my $controller = int($drive->{index} / $maxdev);
8206     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8207         ? "virtioscsi"
8208         : "scsihw";
8209
8210     return ($maxdev, $controller, $controller_prefix);
8211 }
8212
8213 sub resolve_dst_disk_format {
8214         my ($storecfg, $storeid, $src_volname, $format) = @_;
8215         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8216
8217         if (!$format) {
8218             # if no target format is specified, use the source disk format as hint
8219             if ($src_volname) {
8220                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8221                 $format = qemu_img_format($scfg, $src_volname);
8222             } else {
8223                 return $defFormat;
8224             }
8225         }
8226
8227         # test if requested format is supported - else use default
8228         my $supported = grep { $_ eq $format } @$validFormats;
8229         $format = $defFormat if !$supported;
8230         return $format;
8231 }
8232
8233 # NOTE: if this logic changes, please update docs & possibly gui logic
8234 sub find_vmstate_storage {
8235     my ($conf, $storecfg) = @_;
8236
8237     # first, return storage from conf if set
8238     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8239
8240     my ($target, $shared, $local);
8241
8242     foreach_storage_used_by_vm($conf, sub {
8243         my ($sid) = @_;
8244         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8245         my $dst = $scfg->{shared} ? \$shared : \$local;
8246         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8247     });
8248
8249     # second, use shared storage where VM has at least one disk
8250     # third, use local storage where VM has at least one disk
8251     # fall back to local storage
8252     $target = $shared // $local // 'local';
8253
8254     return $target;
8255 }
8256
8257 sub generate_uuid {
8258     my ($uuid, $uuid_str);
8259     UUID::generate($uuid);
8260     UUID::unparse($uuid, $uuid_str);
8261     return $uuid_str;
8262 }
8263
8264 sub generate_smbios1_uuid {
8265     return "uuid=".generate_uuid();
8266 }
8267
8268 sub nbd_stop {
8269     my ($vmid) = @_;
8270
8271     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8272 }
8273
8274 sub create_reboot_request {
8275     my ($vmid) = @_;
8276     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8277         or die "failed to create reboot trigger file: $!\n";
8278     close($fh);
8279 }
8280
8281 sub clear_reboot_request {
8282     my ($vmid) = @_;
8283     my $path = "/run/qemu-server/$vmid.reboot";
8284     my $res = 0;
8285
8286     $res = unlink($path);
8287     die "could not remove reboot request for $vmid: $!"
8288         if !$res && $! != POSIX::ENOENT;
8289
8290     return $res;
8291 }
8292
8293 sub bootorder_from_legacy {
8294     my ($conf, $bootcfg) = @_;
8295
8296     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8297     my $bootindex_hash = {};
8298     my $i = 1;
8299     foreach my $o (split(//, $boot)) {
8300         $bootindex_hash->{$o} = $i*100;
8301         $i++;
8302     }
8303
8304     my $bootorder = {};
8305
8306     PVE::QemuConfig->foreach_volume($conf, sub {
8307         my ($ds, $drive) = @_;
8308
8309         if (drive_is_cdrom ($drive, 1)) {
8310             if ($bootindex_hash->{d}) {
8311                 $bootorder->{$ds} = $bootindex_hash->{d};
8312                 $bootindex_hash->{d} += 1;
8313             }
8314         } elsif ($bootindex_hash->{c}) {
8315             $bootorder->{$ds} = $bootindex_hash->{c}
8316                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8317             $bootindex_hash->{c} += 1;
8318         }
8319     });
8320
8321     if ($bootindex_hash->{n}) {
8322         for (my $i = 0; $i < $MAX_NETS; $i++) {
8323             my $netname = "net$i";
8324             next if !$conf->{$netname};
8325             $bootorder->{$netname} = $bootindex_hash->{n};
8326             $bootindex_hash->{n} += 1;
8327         }
8328     }
8329
8330     return $bootorder;
8331 }
8332
8333 # Generate default device list for 'boot: order=' property. Matches legacy
8334 # default boot order, but with explicit device names. This is important, since
8335 # the fallback for when neither 'order' nor the old format is specified relies
8336 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8337 sub get_default_bootdevices {
8338     my ($conf) = @_;
8339
8340     my @ret = ();
8341
8342     # harddisk
8343     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8344     push @ret, $first if $first;
8345
8346     # cdrom
8347     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8348     push @ret, $first if $first;
8349
8350     # network
8351     for (my $i = 0; $i < $MAX_NETS; $i++) {
8352         my $netname = "net$i";
8353         next if !$conf->{$netname};
8354         push @ret, $netname;
8355         last;
8356     }
8357
8358     return \@ret;
8359 }
8360
8361 sub device_bootorder {
8362     my ($conf) = @_;
8363
8364     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8365
8366     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8367
8368     my $bootorder = {};
8369     if (!defined($boot) || $boot->{legacy}) {
8370         $bootorder = bootorder_from_legacy($conf, $boot);
8371     } elsif ($boot->{order}) {
8372         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8373         for my $dev (PVE::Tools::split_list($boot->{order})) {
8374             $bootorder->{$dev} = $i++;
8375         }
8376     }
8377
8378     return $bootorder;
8379 }
8380
8381 sub register_qmeventd_handle {
8382     my ($vmid) = @_;
8383
8384     my $fh;
8385     my $peer = "/var/run/qmeventd.sock";
8386     my $count = 0;
8387
8388     for (;;) {
8389         $count++;
8390         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8391         last if $fh;
8392         if ($! != EINTR && $! != EAGAIN) {
8393             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8394         }
8395         if ($count > 4) {
8396             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8397               . "after $count retries\n";
8398         }
8399         usleep(25000);
8400     }
8401
8402     # send handshake to mark VM as backing up
8403     print $fh to_json({vzdump => {vmid => "$vmid"}});
8404
8405     # return handle to be closed later when inhibit is no longer required
8406     return $fh;
8407 }
8408
8409 # bash completion helper
8410
8411 sub complete_backup_archives {
8412     my ($cmdname, $pname, $cvalue) = @_;
8413
8414     my $cfg = PVE::Storage::config();
8415
8416     my $storeid;
8417
8418     if ($cvalue =~ m/^([^:]+):/) {
8419         $storeid = $1;
8420     }
8421
8422     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8423
8424     my $res = [];
8425     foreach my $id (keys %$data) {
8426         foreach my $item (@{$data->{$id}}) {
8427             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8428             push @$res, $item->{volid} if defined($item->{volid});
8429         }
8430     }
8431
8432     return $res;
8433 }
8434
8435 my $complete_vmid_full = sub {
8436     my ($running) = @_;
8437
8438     my $idlist = vmstatus();
8439
8440     my $res = [];
8441
8442     foreach my $id (keys %$idlist) {
8443         my $d = $idlist->{$id};
8444         if (defined($running)) {
8445             next if $d->{template};
8446             next if $running && $d->{status} ne 'running';
8447             next if !$running && $d->{status} eq 'running';
8448         }
8449         push @$res, $id;
8450
8451     }
8452     return $res;
8453 };
8454
8455 sub complete_vmid {
8456     return &$complete_vmid_full();
8457 }
8458
8459 sub complete_vmid_stopped {
8460     return &$complete_vmid_full(0);
8461 }
8462
8463 sub complete_vmid_running {
8464     return &$complete_vmid_full(1);
8465 }
8466
8467 sub complete_storage {
8468
8469     my $cfg = PVE::Storage::config();
8470     my $ids = $cfg->{ids};
8471
8472     my $res = [];
8473     foreach my $sid (keys %$ids) {
8474         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8475         next if !$ids->{$sid}->{content}->{images};
8476         push @$res, $sid;
8477     }
8478
8479     return $res;
8480 }
8481
8482 sub complete_migration_storage {
8483     my ($cmd, $param, $current_value, $all_args) = @_;
8484
8485     my $targetnode = @$all_args[1];
8486
8487     my $cfg = PVE::Storage::config();
8488     my $ids = $cfg->{ids};
8489
8490     my $res = [];
8491     foreach my $sid (keys %$ids) {
8492         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8493         next if !$ids->{$sid}->{content}->{images};
8494         push @$res, $sid;
8495     }
8496
8497     return $res;
8498 }
8499
8500 sub vm_is_paused {
8501     my ($vmid, $include_suspended) = @_;
8502     my $qmpstatus = eval {
8503         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8504         mon_cmd($vmid, "query-status");
8505     };
8506     warn "$@\n" if $@;
8507     return $qmpstatus && (
8508         $qmpstatus->{status} eq "paused" ||
8509         $qmpstatus->{status} eq "prelaunch" ||
8510         ($include_suspended && $qmpstatus->{status} eq "suspended")
8511     );
8512 }
8513
8514 sub check_volume_storage_type {
8515     my ($storecfg, $vol) = @_;
8516
8517     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8518     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8519     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8520
8521     die "storage '$storeid' does not support content-type '$vtype'\n"
8522         if !$scfg->{content}->{$vtype};
8523
8524     return 1;
8525 }
8526
8527 sub add_nets_bridge_fdb {
8528     my ($conf, $vmid) = @_;
8529
8530     for my $opt (keys %$conf) {
8531         next if $opt !~ m/^net(\d+)$/;
8532         my $iface = "tap${vmid}i$1";
8533         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8534         my $net = parse_net($conf->{$opt}, 1) or next;
8535
8536         my $mac = $net->{macaddr};
8537         if (!$mac) {
8538             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8539                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8540             next;
8541         }
8542
8543         my $bridge = $net->{bridge};
8544         if (!$bridge) {
8545             log_warn("Interface '$iface' not attached to any bridge.");
8546             next;
8547         }
8548         if ($have_sdn) {
8549             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8550         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8551             PVE::Network::add_bridge_fdb($iface, $mac);
8552         }
8553     }
8554 }
8555
8556 sub del_nets_bridge_fdb {
8557     my ($conf, $vmid) = @_;
8558
8559     for my $opt (keys %$conf) {
8560         next if $opt !~ m/^net(\d+)$/;
8561         my $iface = "tap${vmid}i$1";
8562
8563         my $net = parse_net($conf->{$opt}) or next;
8564         my $mac = $net->{macaddr} or next;
8565
8566         my $bridge = $net->{bridge};
8567         if ($have_sdn) {
8568             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8569         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8570             PVE::Network::del_bridge_fdb($iface, $mac);
8571         }
8572     }
8573 }
8574
8575 1;