PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::HA::Config;
  38 use PVE::Mapping::PCI;
  39 use PVE::Mapping::USB;
  40 use PVE::INotify;
  41 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  42 use PVE::ProcFSTools;
  43 use PVE::PBSClient;
  44 use PVE::RESTEnvironment qw(log_warn);
  45 use PVE::RPCEnvironment;
  46 use PVE::Storage;
  47 use PVE::SysFSTools;
  48 use PVE::Systemd;
  49 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  50
  51 use PVE::QMPClient;
  52 use PVE::QemuConfig;
  53 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  54 use PVE::QemuServer::Cloudinit;
  55 use PVE::QemuServer::CGroup;
  56 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
  57 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  58 use PVE::QemuServer::Machine;
  59 use PVE::QemuServer::Memory qw(get_current_memory);
  60 use PVE::QemuServer::Monitor qw(mon_cmd);
  61 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  62 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  63 use PVE::QemuServer::USB;
  64
  65 my $have_sdn;
  66 eval {
  67     require PVE::Network::SDN::Zones;
  68     require PVE::Network::SDN::Vnets;
  69     $have_sdn = 1;
  70 };
  71
  72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  73 my $OVMF = {
  74     x86_64 => {
  75         '4m-no-smm' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-no-smm-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         '4m' => [
  84             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  86         ],
  87         '4m-ms' => [
  88             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  89             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  90         ],
  91         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  92         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  93         # older backups and snapshot?
  94         default => [
  95             "$EDK2_FW_BASE/OVMF_CODE.fd",
  96             "$EDK2_FW_BASE/OVMF_VARS.fd",
  97         ],
  98     },
  99     aarch64 => {
 100         default => [
 101             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 102             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 103         ],
 104     },
 105 };
 106
 107 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 108
 109 # Note about locking: we use flock on the config file protect against concurent actions.
 110 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 112 # But you can ignore this kind of lock with the --skiplock flag.
 113
 114 cfs_register_file(
 115     '/qemu-server/',
 116     \&parse_vm_config,
 117     \&write_vm_config
 118 );
 119
 120 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 121     description => "Some command save/restore state from this location.",
 122     type => 'string',
 123     maxLength => 128,
 124     optional => 1,
 125 });
 126
 127 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 128         description => "Specifies the QEMU machine type.",
 129         type => 'string',
 130         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 131         maxLength => 40,
 132         optional => 1,
 133 });
 134
 135 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 136 my $nodename_cache;
 137 sub nodename {
 138     $nodename_cache //= PVE::INotify::nodename();
 139     return $nodename_cache;
 140 }
 141
 142 my $watchdog_fmt = {
 143     model => {
 144         default_key => 1,
 145         type => 'string',
 146         enum => [qw(i6300esb ib700)],
 147         description => "Watchdog type to emulate.",
 148         default => 'i6300esb',
 149         optional => 1,
 150     },
 151     action => {
 152         type => 'string',
 153         enum => [qw(reset shutdown poweroff pause debug none)],
 154         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 155         optional => 1,
 156     },
 157 };
 158 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 159
 160 my $agent_fmt = {
 161     enabled => {
 162         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 163         type => 'boolean',
 164         default => 0,
 165         default_key => 1,
 166     },
 167     fstrim_cloned_disks => {
 168         description => "Run fstrim after moving a disk or migrating the VM.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0,
 172     },
 173     'freeze-fs-on-backup' => {
 174         description => "Freeze/thaw guest filesystems on backup for consistency.",
 175         type => 'boolean',
 176         optional => 1,
 177         default => 1,
 178     },
 179     type => {
 180         description => "Select the agent type",
 181         type => 'string',
 182         default => 'virtio',
 183         optional => 1,
 184         enum => [qw(virtio isa)],
 185     },
 186 };
 187
 188 my $vga_fmt = {
 189     type => {
 190         description => "Select the VGA type.",
 191         type => 'string',
 192         default => 'std',
 193         optional => 1,
 194         default_key => 1,
 195         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 196     },
 197     memory => {
 198         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 199         type => 'integer',
 200         optional => 1,
 201         minimum => 4,
 202         maximum => 512,
 203     },
 204     clipboard => {
 205         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 206             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 207         type => 'string',
 208         enum => ['vnc'],
 209         optional => 1,
 210     },
 211 };
 212
 213 my $ivshmem_fmt = {
 214     size => {
 215         type => 'integer',
 216         minimum => 1,
 217         description => "The size of the file in MB.",
 218     },
 219     name => {
 220         type => 'string',
 221         pattern => '[a-zA-Z0-9\-]+',
 222         optional => 1,
 223         format_description => 'string',
 224         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 225     },
 226 };
 227
 228 my $audio_fmt = {
 229     device => {
 230         type => 'string',
 231         enum => [qw(ich9-intel-hda intel-hda AC97)],
 232         description =>  "Configure an audio device."
 233     },
 234     driver =>  {
 235         type => 'string',
 236         enum => ['spice', 'none'],
 237         default => 'spice',
 238         optional => 1,
 239         description => "Driver backend for the audio device."
 240     },
 241 };
 242
 243 my $spice_enhancements_fmt = {
 244     foldersharing => {
 245         type => 'boolean',
 246         optional => 1,
 247         default => '0',
 248         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 249     },
 250     videostreaming =>  {
 251         type => 'string',
 252         enum => ['off', 'all', 'filter'],
 253         default => 'off',
 254         optional => 1,
 255         description => "Enable video streaming. Uses compression for detected video streams."
 256     },
 257 };
 258
 259 my $rng_fmt = {
 260     source => {
 261         type => 'string',
 262         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 263         default_key => 1,
 264         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 265             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 266             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 267             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 268             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 269             ." a hardware RNG from the host.",
 270     },
 271     max_bytes => {
 272         type => 'integer',
 273         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 274             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 275             ." `0` to disable limiting (potentially dangerous!).",
 276         optional => 1,
 277
 278         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 279         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 280         # reading from /dev/urandom
 281         default => 1024,
 282     },
 283     period => {
 284         type => 'integer',
 285         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 286             ." the guest to retrieve another 'max_bytes' of entropy.",
 287         optional => 1,
 288         default => 1000,
 289     },
 290 };
 291
 292 my $meta_info_fmt = {
 293     'ctime' => {
 294         type => 'integer',
 295         description => "The guest creation timestamp as UNIX epoch time",
 296         minimum => 0,
 297         optional => 1,
 298     },
 299     'creation-qemu' => {
 300         type => 'string',
 301         description => "The QEMU (machine) version from the time this VM was created.",
 302         pattern => '\d+(\.\d+)+',
 303         optional => 1,
 304     },
 305 };
 306
 307 my $confdesc = {
 308     onboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Specifies whether a VM will be started during system bootup.",
 312         default => 0,
 313     },
 314     autostart => {
 315         optional => 1,
 316         type => 'boolean',
 317         description => "Automatic restart after crash (currently ignored).",
 318         default => 0,
 319     },
 320     hotplug => {
 321         optional => 1,
 322         type => 'string', format => 'pve-hotplug-features',
 323         description => "Selectively enable hotplug features. This is a comma separated list of"
 324             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 325             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 326             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 327             ." windows > 7.",
 328         default => 'network,disk,usb',
 329     },
 330     reboot => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 334         default => 1,
 335     },
 336     lock => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Lock/unlock the VM.",
 340         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 341     },
 342     cpulimit => {
 343         optional => 1,
 344         type => 'number',
 345         description => "Limit of CPU usage.",
 346         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 347             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 348         minimum => 0,
 349         maximum => 128,
 350         default => 0,
 351     },
 352     cpuunits => {
 353         optional => 1,
 354         type => 'integer',
 355         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 356         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 357             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 358             ." weights of all the other running VMs.",
 359         minimum => 1,
 360         maximum => 262144,
 361         default => 'cgroup v1: 1024, cgroup v2: 100',
 362     },
 363     memory => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Memory properties.",
 367         format => $PVE::QemuServer::Memory::memory_fmt
 368     },
 369     balloon => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 373         minimum => 0,
 374     },
 375     shares => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 379             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 380             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 381         minimum => 0,
 382         maximum => 50000,
 383         default => 1000,
 384     },
 385     keyboard => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Keyboard layout for VNC server. This option is generally not required and"
 389         ." is often better handled from within the guest OS.",
 390         enum => PVE::Tools::kvmkeymaplist(),
 391         default => undef,
 392     },
 393     name => {
 394         optional => 1,
 395         type => 'string', format => 'dns-name',
 396         description => "Set a name for the VM. Only used on the configuration web interface.",
 397     },
 398     scsihw => {
 399         optional => 1,
 400         type => 'string',
 401         description => "SCSI controller model",
 402         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 403         default => 'lsi',
 404     },
 405     description => {
 406         optional => 1,
 407         type => 'string',
 408         description => "Description for the VM. Shown in the web-interface VM's summary."
 409             ." This is saved as comment inside the configuration file.",
 410         maxLength => 1024 * 8,
 411     },
 412     ostype => {
 413         optional => 1,
 414         type => 'string',
 415         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 416         description => "Specify guest operating system.",
 417         verbose_description => <<EODESC,
 418 Specify guest operating system. This is used to enable special
 419 optimization/features for specific operating systems:
 420
 421 [horizontal]
 422 other;; unspecified OS
 423 wxp;; Microsoft Windows XP
 424 w2k;; Microsoft Windows 2000
 425 w2k3;; Microsoft Windows 2003
 426 w2k8;; Microsoft Windows 2008
 427 wvista;; Microsoft Windows Vista
 428 win7;; Microsoft Windows 7
 429 win8;; Microsoft Windows 8/2012/2012r2
 430 win10;; Microsoft Windows 10/2016/2019
 431 win11;; Microsoft Windows 11/2022
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6 - 6.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string', format => 'pve-qm-boot',
 440         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 441             ." key or 'legacy=' is deprecated.",
 442     },
 443     bootdisk => {
 444         optional => 1,
 445         type => 'string', format => 'pve-qm-bootdisk',
 446         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 447         pattern => '(ide|sata|scsi|virtio)\d+',
 448     },
 449     smp => {
 450         optional => 1,
 451         type => 'integer',
 452         description => "The number of CPUs. Please use option -sockets instead.",
 453         minimum => 1,
 454         default => 1,
 455     },
 456     sockets => {
 457         optional => 1,
 458         type => 'integer',
 459         description => "The number of CPU sockets.",
 460         minimum => 1,
 461         default => 1,
 462     },
 463     cores => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "The number of cores per socket.",
 467         minimum => 1,
 468         default => 1,
 469     },
 470     numa => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable NUMA.",
 474         default => 0,
 475     },
 476     hugepages => {
 477         optional => 1,
 478         type => 'string',
 479         description => "Enable/disable hugepages memory.",
 480         enum => [qw(any 2 1024)],
 481     },
 482     keephugepages => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 0,
 486         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 487             ." after VM shutdown and can be used for subsequent starts.",
 488     },
 489     vcpus => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Number of hotplugged vcpus.",
 493         minimum => 1,
 494         default => 0,
 495     },
 496     acpi => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable ACPI.",
 500         default => 1,
 501     },
 502     agent => {
 503         optional => 1,
 504         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 505         type => 'string',
 506         format => $agent_fmt,
 507     },
 508     kvm => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable KVM hardware virtualization.",
 512         default => 1,
 513     },
 514     tdf => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Enable/disable time drift fix.",
 518         default => 0,
 519     },
 520     localtime => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 524             ." the `ostype` indicates a Microsoft Windows OS.",
 525     },
 526     freeze => {
 527         optional => 1,
 528         type => 'boolean',
 529         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 530     },
 531     vga => {
 532         optional => 1,
 533         type => 'string', format => $vga_fmt,
 534         description => "Configure the VGA hardware.",
 535         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 536             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 537             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 538             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 539             ." display server. For win* OS you can select how many independent displays you want,"
 540             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 541             ." using a serial device as terminal.",
 542     },
 543     watchdog => {
 544         optional => 1,
 545         type => 'string', format => 'pve-qm-watchdog',
 546         description => "Create a virtual hardware watchdog device.",
 547         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 548             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 549             ." else the watchdog will reset the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are:"
 556             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 557         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 558         default => 'now',
 559     },
 560     startup =>  get_standard_option('pve-startup-order'),
 561     template => {
 562         optional => 1,
 563         type => 'boolean',
 564         description => "Enable/disable Template.",
 565         default => 0,
 566     },
 567     args => {
 568         optional => 1,
 569         type => 'string',
 570         description => "Arbitrary arguments passed to kvm.",
 571         verbose_description => <<EODESCR,
 572 Arbitrary arguments passed to kvm, for example:
 573
 574 args: -no-reboot -smbios 'type=0,vendor=FOO'
 575
 576 NOTE: this option is for experts only.
 577 EODESCR
 578     },
 579     tablet => {
 580         optional => 1,
 581         type => 'boolean',
 582         default => 1,
 583         description => "Enable/disable the USB tablet device.",
 584         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 585             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 586             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 587             ." may consider disabling this to save some context switches. This is turned off by"
 588             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => 'pve-vm-cpu-conf',
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally"
 630             ." for snapshots.",
 631     },
 632     vmstatestorage => get_standard_option('pve-storage-id', {
 633         description => "Default storage for VM state volumes/files.",
 634         optional => 1,
 635     }),
 636     runningmachine => get_standard_option('pve-qemu-machine', {
 637         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 638             ." for snapshots.",
 639     }),
 640     runningcpu => {
 641         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 642             ." internally for snapshots.",
 643         optional => 1,
 644         type => 'string',
 645         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 646         format_description => 'QEMU -cpu parameter'
 647     },
 648     machine => get_standard_option('pve-qemu-machine'),
 649     arch => {
 650         description => "Virtual processor architecture. Defaults to the host.",
 651         optional => 1,
 652         type => 'string',
 653         enum => [qw(x86_64 aarch64)],
 654     },
 655     smbios1 => {
 656         description => "Specify SMBIOS type 1 fields.",
 657         type => 'string', format => 'pve-qm-smbios1',
 658         maxLength => 512,
 659         optional => 1,
 660     },
 661     protection => {
 662         optional => 1,
 663         type => 'boolean',
 664         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 665             ." remove disk operations.",
 666         default => 0,
 667     },
 668     bios => {
 669         optional => 1,
 670         type => 'string',
 671         enum => [ qw(seabios ovmf) ],
 672         description => "Select BIOS implementation.",
 673         default => 'seabios',
 674     },
 675     vmgenid => {
 676         type => 'string',
 677         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 678         format_description => 'UUID',
 679         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 680             ." to disable explicitly.",
 681         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 682             ." value identifier to the guest OS. This allows to notify the guest operating system"
 683             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 684             ." execution or creation from a template). The guest operating system notices the"
 685             ." change, and is then able to react as appropriate by marking its copies of"
 686             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 687             ."Note that auto-creation only works when done through API/CLI create or update methods"
 688             .", but not when manually editing the config file.",
 689         default => "1 (autogenerated)",
 690         optional => 1,
 691     },
 692     hookscript => {
 693         type => 'string',
 694         format => 'pve-volume-id',
 695         optional => 1,
 696         description => "Script that will be executed during various steps in the vms lifetime.",
 697     },
 698     ivshmem => {
 699         type => 'string',
 700         format => $ivshmem_fmt,
 701         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 702             ." the host.",
 703         optional => 1,
 704     },
 705     audio0 => {
 706         type => 'string',
 707         format => $audio_fmt,
 708         description => "Configure a audio device, useful in combination with QXL/Spice.",
 709         optional => 1
 710     },
 711     spice_enhancements => {
 712         type => 'string',
 713         format => $spice_enhancements_fmt,
 714         description => "Configure additional enhancements for SPICE.",
 715         optional => 1
 716     },
 717     tags => {
 718         type => 'string', format => 'pve-tag-list',
 719         description => 'Tags of the VM. This is only meta information.',
 720         optional => 1,
 721     },
 722     rng0 => {
 723         type => 'string',
 724         format => $rng_fmt,
 725         description => "Configure a VirtIO-based Random Number Generator.",
 726         optional => 1,
 727     },
 728     meta => {
 729         type => 'string',
 730         format => $meta_info_fmt,
 731         description => "Some (read-only) meta-information about this guest.",
 732         optional => 1,
 733     },
 734     affinity => {
 735         type => 'string', format => 'pve-cpuset',
 736         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 737         optional => 1,
 738     },
 739 };
 740
 741 my $cicustom_fmt = {
 742     meta => {
 743         type => 'string',
 744         optional => 1,
 745         description => 'Specify a custom file containing all meta data passed to the VM via"
 746             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 747         format => 'pve-volume-id',
 748         format_description => 'volume',
 749     },
 750     network => {
 751         type => 'string',
 752         optional => 1,
 753         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 754         format => 'pve-volume-id',
 755         format_description => 'volume',
 756     },
 757     user => {
 758         type => 'string',
 759         optional => 1,
 760         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 761         format => 'pve-volume-id',
 762         format_description => 'volume',
 763     },
 764     vendor => {
 765         type => 'string',
 766         optional => 1,
 767         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 768         format => 'pve-volume-id',
 769         format_description => 'volume',
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 773
 774 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 775 my $confdesc_cloudinit = {
 776     citype => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'Specifies the cloud-init configuration format. The default depends on the'
 780             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 781             .' and `configdrive2` for windows.',
 782         enum => ['configdrive2', 'nocloud', 'opennebula'],
 783     },
 784     ciuser => {
 785         optional => 1,
 786         type => 'string',
 787         description => "cloud-init: User name to change ssh keys and password for instead of the"
 788             ." image's configured default user.",
 789     },
 790     cipassword => {
 791         optional => 1,
 792         type => 'string',
 793         description => 'cloud-init: Password to assign the user. Using this is generally not'
 794             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 795             .' support hashed passwords.',
 796     },
 797     ciupgrade => {
 798         optional => 1,
 799         type => 'boolean',
 800         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 801         default => 1,
 802     },
 803     cicustom => {
 804         optional => 1,
 805         type => 'string',
 806         description => 'cloud-init: Specify custom files to replace the automatically generated'
 807             .' ones at start.',
 808         format => 'pve-qm-cicustom',
 809     },
 810     searchdomain => {
 811         optional => 1,
 812         type => 'string',
 813         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 814             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 815             .' are set.',
 816     },
 817     nameserver => {
 818         optional => 1,
 819         type => 'string', format => 'address-list',
 820         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 821             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 822             .' are set.',
 823     },
 824     sshkeys => {
 825         optional => 1,
 826         type => 'string',
 827         format => 'urlencoded',
 828         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 829     },
 830 };
 831
 832 # what about other qemu settings ?
 833 #cpu => 'string',
 834 #machine => 'string',
 835 #fda => 'file',
 836 #fdb => 'file',
 837 #mtdblock => 'file',
 838 #sd => 'file',
 839 #pflash => 'file',
 840 #snapshot => 'bool',
 841 #bootp => 'file',
 842 ##tftp => 'dir',
 843 ##smb => 'dir',
 844 #kernel => 'file',
 845 #append => 'string',
 846 #initrd => 'file',
 847 ##soundhw => 'string',
 848
 849 while (my ($k, $v) = each %$confdesc) {
 850     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 851 }
 852
 853 my $MAX_NETS = 32;
 854 my $MAX_SERIAL_PORTS = 4;
 855 my $MAX_PARALLEL_PORTS = 3;
 856
 857 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 858     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 859 }
 860
 861 my $nic_model_list = [
 862     'e1000',
 863     'e1000-82540em',
 864     'e1000-82544gc',
 865     'e1000-82545em',
 866     'e1000e',
 867     'i82551',
 868     'i82557b',
 869     'i82559er',
 870     'ne2k_isa',
 871     'ne2k_pci',
 872     'pcnet',
 873     'rtl8139',
 874     'virtio',
 875     'vmxnet3',
 876 ];
 877 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 878
 879 my $net_fmt_bridge_descr = <<__EOD__;
 880 Bridge to attach the network device to. The Proxmox VE standard bridge
 881 is called 'vmbr0'.
 882
 883 If you do not specify a bridge, we create a kvm user (NATed) network
 884 device, which provides DHCP and DNS services. The following addresses
 885 are used:
 886
 887  10.0.2.2   Gateway
 888  10.0.2.3   DNS Server
 889  10.0.2.4   SMB Server
 890
 891 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 892 __EOD__
 893
 894 my $net_fmt = {
 895     macaddr  => get_standard_option('mac-addr', {
 896         description => "MAC address. That address must be unique withing your network. This is"
 897             ." automatically generated if not specified.",
 898     }),
 899     model => {
 900         type => 'string',
 901         description => "Network Card Model. The 'virtio' model provides the best performance with"
 902             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 903             ." best to use 'e1000'.",
 904         enum => $nic_model_list,
 905         default_key => 1,
 906     },
 907     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 908     bridge => get_standard_option('pve-bridge-id', {
 909         description => $net_fmt_bridge_descr,
 910         optional => 1,
 911     }),
 912     queues => {
 913         type => 'integer',
 914         minimum => 0, maximum => 64,
 915         description => 'Number of packet queues to be used on the device.',
 916         optional => 1,
 917     },
 918     rate => {
 919         type => 'number',
 920         minimum => 0,
 921         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 922         optional => 1,
 923     },
 924     tag => {
 925         type => 'integer',
 926         minimum => 1, maximum => 4094,
 927         description => 'VLAN tag to apply to packets on this interface.',
 928         optional => 1,
 929     },
 930     trunks => {
 931         type => 'string',
 932         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 933         description => 'VLAN trunks to pass through this interface.',
 934         format_description => 'vlanid[;vlanid...]',
 935         optional => 1,
 936     },
 937     firewall => {
 938         type => 'boolean',
 939         description => 'Whether this interface should be protected by the firewall.',
 940         optional => 1,
 941     },
 942     link_down => {
 943         type => 'boolean',
 944         description => 'Whether this interface should be disconnected (like pulling the plug).',
 945         optional => 1,
 946     },
 947     mtu => {
 948         type => 'integer',
 949         minimum => 1, maximum => 65520,
 950         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 951         optional => 1,
 952     },
 953 };
 954
 955 my $netdesc = {
 956     optional => 1,
 957     type => 'string', format => $net_fmt,
 958     description => "Specify network devices.",
 959 };
 960
 961 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 962
 963 my $ipconfig_fmt = {
 964     ip => {
 965         type => 'string',
 966         format => 'pve-ipv4-config',
 967         format_description => 'IPv4Format/CIDR',
 968         description => 'IPv4 address in CIDR format.',
 969         optional => 1,
 970         default => 'dhcp',
 971     },
 972     gw => {
 973         type => 'string',
 974         format => 'ipv4',
 975         format_description => 'GatewayIPv4',
 976         description => 'Default gateway for IPv4 traffic.',
 977         optional => 1,
 978         requires => 'ip',
 979     },
 980     ip6 => {
 981         type => 'string',
 982         format => 'pve-ipv6-config',
 983         format_description => 'IPv6Format/CIDR',
 984         description => 'IPv6 address in CIDR format.',
 985         optional => 1,
 986         default => 'dhcp',
 987     },
 988     gw6 => {
 989         type => 'string',
 990         format => 'ipv6',
 991         format_description => 'GatewayIPv6',
 992         description => 'Default gateway for IPv6 traffic.',
 993         optional => 1,
 994         requires => 'ip6',
 995     },
 996 };
 997 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 998 my $ipconfigdesc = {
 999     optional => 1,
1000     type => 'string', format => 'pve-qm-ipconfig',
1001     description => <<'EODESCR',
1002 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1003
1004 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1005
1006 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1007 gateway should be provided.
1008 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1009 cloud-init 19.4 or newer.
1010
1011 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1012 dhcp on IPv4.
1013 EODESCR
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1016
1017 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1018     $confdesc->{"net$i"} = $netdesc;
1019     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1020 }
1021
1022 foreach my $key (keys %$confdesc_cloudinit) {
1023     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1024 }
1025
1026 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1027 sub pve_verify_cpuset {
1028     my ($set_text, $noerr) = @_;
1029
1030     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1031
1032     if ($@) {
1033         return if $noerr;
1034         die "unable to parse cpuset option\n";
1035     }
1036
1037     return PVE::CpuSet->new($members)->short_string();
1038 }
1039
1040 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1041 sub verify_volume_id_or_qm_path {
1042     my ($volid, $noerr) = @_;
1043
1044     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1045
1046     return verify_volume_id_or_absolute_path($volid, $noerr);
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1050 sub verify_volume_id_or_absolute_path {
1051     my ($volid, $noerr) = @_;
1052
1053     return $volid if $volid =~ m|^/|;
1054
1055     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1056     if ($@) {
1057         return if $noerr;
1058         die $@;
1059     }
1060     return $volid;
1061 }
1062
1063 my $serialdesc = {
1064         optional => 1,
1065         type => 'string',
1066         pattern => '(/dev/.+|socket)',
1067         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1068         verbose_description =>  <<EODESCR,
1069 Create a serial device inside the VM (n is 0 to 3), and pass through a
1070 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1071 host side (use 'qm terminal' to open a terminal connection).
1072
1073 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1074 use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 my $paralleldesc= {
1081         optional => 1,
1082         type => 'string',
1083         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1084         description =>  "Map host parallel devices (n is 0 to 2).",
1085         verbose_description =>  <<EODESCR,
1086 Map host parallel devices (n is 0 to 2).
1087
1088 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1089 machines - use with special care.
1090
1091 CAUTION: Experimental! User reported problems with this option.
1092 EODESCR
1093 };
1094
1095 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1096     $confdesc->{"parallel$i"} = $paralleldesc;
1097 }
1098
1099 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1100     $confdesc->{"serial$i"} = $serialdesc;
1101 }
1102
1103 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1104     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1105 }
1106
1107 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1108     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1109 }
1110
1111 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1112     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1113 }
1114
1115 my $boot_fmt = {
1116     legacy => {
1117         optional => 1,
1118         default_key => 1,
1119         type => 'string',
1120         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1121                      . " Deprecated, use 'order=' instead.",
1122         pattern => '[acdn]{1,4}',
1123         format_description => "[acdn]{1,4}",
1124
1125         # note: this is also the fallback if boot: is not given at all
1126         default => 'cdn',
1127     },
1128     order => {
1129         optional => 1,
1130         type => 'string',
1131         format => 'pve-qm-bootdev-list',
1132         format_description => "device[;device...]",
1133         description => <<EODESC,
1134 The guest will attempt to boot from devices in the order they appear here.
1135
1136 Disks, optical drives and passed-through storage USB devices will be directly
1137 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1138 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1139
1140 Note that only devices in this list will be marked as bootable and thus loaded
1141 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1142 (e.g. software-raid), you need to specify all of them here.
1143
1144 Overrides the deprecated 'legacy=[acdn]*' value when given.
1145 EODESC
1146     },
1147 };
1148 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1149
1150 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1151 sub verify_bootdev {
1152     my ($dev, $noerr) = @_;
1153
1154     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1155     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1156
1157     my $check = sub {
1158         my ($base) = @_;
1159         return 0 if $dev !~ m/^$base\d+$/;
1160         return 0 if !$confdesc->{$dev};
1161         return 1;
1162     };
1163
1164     return $dev if $check->("net");
1165     return $dev if $check->("usb");
1166     return $dev if $check->("hostpci");
1167
1168     return if $noerr;
1169     die "invalid boot device '$dev'\n";
1170 }
1171
1172 sub print_bootorder {
1173     my ($devs) = @_;
1174     return "" if !@$devs;
1175     my $data = { order => join(';', @$devs) };
1176     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1177 }
1178
1179 my $kvm_api_version = 0;
1180
1181 sub kvm_version {
1182     return $kvm_api_version if $kvm_api_version;
1183
1184     open my $fh, '<', '/dev/kvm' or return;
1185
1186     # 0xae00 => KVM_GET_API_VERSION
1187     $kvm_api_version = ioctl($fh, 0xae00, 0);
1188     close($fh);
1189
1190     return $kvm_api_version;
1191 }
1192
1193 my $kvm_user_version = {};
1194 my $kvm_mtime = {};
1195
1196 sub kvm_user_version {
1197     my ($binary) = @_;
1198
1199     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1200     my $st = stat($binary);
1201
1202     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1203     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1204         $cachedmtime == $st->mtime;
1205
1206     $kvm_user_version->{$binary} = 'unknown';
1207     $kvm_mtime->{$binary} = $st->mtime;
1208
1209     my $code = sub {
1210         my $line = shift;
1211         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1212             $kvm_user_version->{$binary} = $2;
1213         }
1214     };
1215
1216     eval { run_command([$binary, '--version'], outfunc => $code); };
1217     warn $@ if $@;
1218
1219     return $kvm_user_version->{$binary};
1220
1221 }
1222 my sub extract_version {
1223     my ($machine_type, $version) = @_;
1224     $version = kvm_user_version() if !defined($version);
1225     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1226 }
1227
1228 sub kernel_has_vhost_net {
1229     return -c '/dev/vhost-net';
1230 }
1231
1232 sub option_exists {
1233     my $key = shift;
1234     return defined($confdesc->{$key});
1235 }
1236
1237 my $cdrom_path;
1238 sub get_cdrom_path {
1239
1240     return $cdrom_path if defined($cdrom_path);
1241
1242     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1243
1244     if (!defined($cdrom_path)) {
1245         log_warn("no physical CD-ROM available, ignoring");
1246         $cdrom_path = '';
1247     }
1248
1249     return $cdrom_path;
1250 }
1251
1252 sub get_iso_path {
1253     my ($storecfg, $vmid, $cdrom) = @_;
1254
1255     if ($cdrom eq 'cdrom') {
1256         return get_cdrom_path();
1257     } elsif ($cdrom eq 'none') {
1258         return '';
1259     } elsif ($cdrom =~ m|^/|) {
1260         return $cdrom;
1261     } else {
1262         return PVE::Storage::path($storecfg, $cdrom);
1263     }
1264 }
1265
1266 # try to convert old style file names to volume IDs
1267 sub filename_to_volume_id {
1268     my ($vmid, $file, $media) = @_;
1269
1270      if (!($file eq 'none' || $file eq 'cdrom' ||
1271           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272
1273         return if $file =~ m|/|;
1274
1275         if ($media && $media eq 'cdrom') {
1276             $file = "local:iso/$file";
1277         } else {
1278             $file = "local:$vmid/$file";
1279         }
1280     }
1281
1282     return $file;
1283 }
1284
1285 sub verify_media_type {
1286     my ($opt, $vtype, $media) = @_;
1287
1288     return if !$media;
1289
1290     my $etype;
1291     if ($media eq 'disk') {
1292         $etype = 'images';
1293     } elsif ($media eq 'cdrom') {
1294         $etype = 'iso';
1295     } else {
1296         die "internal error";
1297     }
1298
1299     return if ($vtype eq $etype);
1300
1301     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1302 }
1303
1304 sub cleanup_drive_path {
1305     my ($opt, $storecfg, $drive) = @_;
1306
1307     # try to convert filesystem paths to volume IDs
1308
1309     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1310         ($drive->{file} !~ m|^/dev/.+|) &&
1311         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1312         ($drive->{file} !~ m/^\d+$/)) {
1313         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1314         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315             if !$vtype;
1316         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1317         verify_media_type($opt, $vtype, $drive->{media});
1318         $drive->{file} = $volid;
1319     }
1320
1321     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1322 }
1323
1324 sub parse_hotplug_features {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     return $res if $data eq '0';
1330
1331     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1332
1333     foreach my $feature (PVE::Tools::split_list($data)) {
1334         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1335             $res->{$1} = 1;
1336         } else {
1337             die "invalid hotplug feature '$feature'\n";
1338         }
1339     }
1340     return $res;
1341 }
1342
1343 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1344 sub pve_verify_hotplug_features {
1345     my ($value, $noerr) = @_;
1346
1347     return $value if parse_hotplug_features($value);
1348
1349     return if $noerr;
1350
1351     die "unable to parse hotplug option\n";
1352 }
1353
1354 sub assert_clipboard_config {
1355     my ($vga) = @_;
1356
1357     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1358
1359     if (
1360         $vga->{'clipboard'}
1361         && $vga->{'clipboard'} eq 'vnc'
1362         && $vga->{type}
1363         && $vga->{type} !~ $clipboard_regex
1364     ) {
1365         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1366     }
1367 }
1368
1369 sub print_tabletdevice_full {
1370     my ($conf, $arch) = @_;
1371
1372     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1373
1374     # we use uhci for old VMs because tablet driver was buggy in older qemu
1375     my $usbbus;
1376     if ($q35 || $arch eq 'aarch64') {
1377         $usbbus = 'ehci';
1378     } else {
1379         $usbbus = 'uhci';
1380     }
1381
1382     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1383 }
1384
1385 sub print_keyboarddevice_full {
1386     my ($conf, $arch) = @_;
1387
1388     return if $arch ne 'aarch64';
1389
1390     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1391 }
1392
1393 my sub get_drive_id {
1394     my ($drive) = @_;
1395     return "$drive->{interface}$drive->{index}";
1396 }
1397
1398 sub print_drivedevice_full {
1399     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1400
1401     my $device = '';
1402     my $maxdev = 0;
1403
1404     my $drive_id = get_drive_id($drive);
1405     if ($drive->{interface} eq 'virtio') {
1406         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1407         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1408         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1409     } elsif ($drive->{interface} eq 'scsi') {
1410
1411         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1412         my $unit = $drive->{index} % $maxdev;
1413
1414         my $machine_version = extract_version($machine_type, kvm_user_version());
1415         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1416             $drive, $storecfg, $machine_version);
1417
1418         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1419             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1420         } else {
1421             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1422                 .",lun=$drive->{index}";
1423         }
1424         $device .= ",drive=drive-$drive_id,id=$drive_id";
1425
1426         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1427             $device .= ",rotation_rate=1";
1428         }
1429         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1430
1431         # only scsi-hd and scsi-cd support passing vendor and product information
1432         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1433             if (my $vendor = $drive->{vendor}) {
1434                 $device .= ",vendor=$vendor";
1435             }
1436             if (my $product = $drive->{product}) {
1437                 $device .= ",product=$product";
1438             }
1439         }
1440
1441     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1442         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1443         my $controller = int($drive->{index} / $maxdev);
1444         my $unit = $drive->{index} % $maxdev;
1445
1446         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1447         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1448         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1449         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1450         # were before. Move odd ones up by 2 where they don't clash.
1451         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1452             $controller += 2 * ($unit % 2);
1453             $unit = 0;
1454         }
1455
1456         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1457
1458         $device = "ide-$devicetype";
1459         if ($drive->{interface} eq 'ide') {
1460             $device .= ",bus=ide.$controller,unit=$unit";
1461         } else {
1462             $device .= ",bus=ahci$controller.$unit";
1463         }
1464         $device .= ",drive=drive-$drive_id,id=$drive_id";
1465
1466         if ($devicetype eq 'hd') {
1467             if (my $model = $drive->{model}) {
1468                 $model = URI::Escape::uri_unescape($model);
1469                 $device .= ",model=$model";
1470             }
1471             if ($drive->{ssd}) {
1472                 $device .= ",rotation_rate=1";
1473             }
1474         }
1475         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1476     } elsif ($drive->{interface} eq 'usb') {
1477         die "implement me";
1478         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1479     } else {
1480         die "unsupported interface type";
1481     }
1482
1483     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1484
1485     if (my $serial = $drive->{serial}) {
1486         $serial = URI::Escape::uri_unescape($serial);
1487         $device .= ",serial=$serial";
1488     }
1489
1490
1491     return $device;
1492 }
1493
1494 sub get_initiator_name {
1495     my $initiator;
1496
1497     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1498     while (defined(my $line = <$fh>)) {
1499         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1500         $initiator = $1;
1501         last;
1502     }
1503     $fh->close();
1504
1505     return $initiator;
1506 }
1507
1508 my sub storage_allows_io_uring_default {
1509     my ($scfg, $cache_direct) = @_;
1510
1511     # io_uring with cache mode writeback or writethrough on krbd will hang...
1512     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1513
1514     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1515     # sometimes, just plain disable...
1516     return if $scfg && $scfg->{type} eq 'lvm';
1517
1518     # io_uring causes problems when used with CIFS since kernel 5.15
1519     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1520     return if $scfg && $scfg->{type} eq 'cifs';
1521
1522     return 1;
1523 }
1524
1525 my sub drive_uses_cache_direct {
1526     my ($drive, $scfg) = @_;
1527
1528     my $cache_direct = 0;
1529
1530     if (my $cache = $drive->{cache}) {
1531         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1532     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1533         $cache_direct = 1;
1534     }
1535
1536     return $cache_direct;
1537 }
1538
1539 sub print_drive_commandline_full {
1540     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1541
1542     my $path;
1543     my $volid = $drive->{file};
1544     my $format = $drive->{format};
1545     my $drive_id = get_drive_id($drive);
1546
1547     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1548     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1549
1550     if (drive_is_cdrom($drive)) {
1551         $path = get_iso_path($storecfg, $vmid, $volid);
1552         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1553     } else {
1554         if ($storeid) {
1555             $path = PVE::Storage::path($storecfg, $volid);
1556             $format //= qemu_img_format($scfg, $volname);
1557         } else {
1558             $path = $volid;
1559             $format //= "raw";
1560         }
1561    }
1562
1563    my $is_rbd = $path =~ m/^rbd:/;
1564
1565     my $opts = '';
1566     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1567     foreach my $o (@qemu_drive_options) {
1568         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1569     }
1570
1571     # snapshot only accepts on|off
1572     if (defined($drive->{snapshot})) {
1573         my $v = $drive->{snapshot} ? 'on' : 'off';
1574         $opts .= ",snapshot=$v";
1575     }
1576
1577     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1578         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1579     }
1580
1581     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1582         my ($dir, $qmpname) = @$type;
1583         if (my $v = $drive->{"mbps$dir"}) {
1584             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1585         }
1586         if (my $v = $drive->{"mbps${dir}_max"}) {
1587             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1588         }
1589         if (my $v = $drive->{"bps${dir}_max_length"}) {
1590             $opts .= ",throttling.bps$qmpname-max-length=$v";
1591         }
1592         if (my $v = $drive->{"iops${dir}"}) {
1593             $opts .= ",throttling.iops$qmpname=$v";
1594         }
1595         if (my $v = $drive->{"iops${dir}_max"}) {
1596             $opts .= ",throttling.iops$qmpname-max=$v";
1597         }
1598         if (my $v = $drive->{"iops${dir}_max_length"}) {
1599             $opts .= ",throttling.iops$qmpname-max-length=$v";
1600         }
1601     }
1602
1603     if ($pbs_name) {
1604         $format = "rbd" if $is_rbd;
1605         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1606             if !$format;
1607         $opts .= ",format=alloc-track,file.driver=$format";
1608     } elsif ($format) {
1609         $opts .= ",format=$format";
1610     }
1611
1612     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1613
1614     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1615
1616     if (!$drive->{aio}) {
1617         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1618             # io_uring supports all cache modes
1619             $opts .= ",aio=io_uring";
1620         } else {
1621             # aio native works only with O_DIRECT
1622             if($cache_direct) {
1623                 $opts .= ",aio=native";
1624             } else {
1625                 $opts .= ",aio=threads";
1626             }
1627         }
1628     }
1629
1630     if (!drive_is_cdrom($drive)) {
1631         my $detectzeroes;
1632         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1633             $detectzeroes = 'off';
1634         } elsif ($drive->{discard}) {
1635             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1636         } else {
1637             # This used to be our default with discard not being specified:
1638             $detectzeroes = 'on';
1639         }
1640
1641         # note: 'detect-zeroes' works per blockdev and we want it to persist
1642         # after the alloc-track is removed, so put it on 'file' directly
1643         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1644         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1645     }
1646
1647     if ($pbs_name) {
1648         $opts .= ",backing=$pbs_name";
1649         $opts .= ",auto-remove=on";
1650     }
1651
1652     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1653     my $file_param = "file";
1654     if ($pbs_name) {
1655         # non-rbd drivers require the underlying file to be a seperate block
1656         # node, so add a second .file indirection
1657         $file_param .= ".file" if !$is_rbd;
1658         $file_param .= ".filename";
1659     }
1660     my $pathinfo = $path ? "$file_param=$path," : '';
1661
1662     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1663 }
1664
1665 sub print_pbs_blockdev {
1666     my ($pbs_conf, $pbs_name) = @_;
1667     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1668     $blockdev .= ",repository=$pbs_conf->{repository}";
1669     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1670     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1671     $blockdev .= ",archive=$pbs_conf->{archive}";
1672     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1673     return $blockdev;
1674 }
1675
1676 sub print_netdevice_full {
1677     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1678
1679     my $device = $net->{model};
1680     if ($net->{model} eq 'virtio') {
1681          $device = 'virtio-net-pci';
1682      };
1683
1684     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1685     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1686     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1687         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1688         # and out of each queue plus one config interrupt and control vector queue
1689         my $vectors = $net->{queues} * 2 + 2;
1690         $tmpstr .= ",vectors=$vectors,mq=on";
1691         if (min_version($machine_version, 7, 1)) {
1692             $tmpstr .= ",packed=on";
1693         }
1694     }
1695
1696     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1697         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1698     }
1699
1700     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1701
1702     if (my $mtu = $net->{mtu}) {
1703         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1704             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1705             if ($mtu == 1) {
1706                  $mtu = $bridge_mtu;
1707             } elsif ($mtu < 576) {
1708                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709             } elsif ($mtu > $bridge_mtu) {
1710                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1711             }
1712             $tmpstr .= ",host_mtu=$mtu";
1713         } else {
1714             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1715         }
1716     }
1717
1718     if ($use_old_bios_files) {
1719         my $romfile;
1720         if ($device eq 'virtio-net-pci') {
1721             $romfile = 'pxe-virtio.rom';
1722         } elsif ($device eq 'e1000') {
1723             $romfile = 'pxe-e1000.rom';
1724         } elsif ($device eq 'e1000e') {
1725             $romfile = 'pxe-e1000e.rom';
1726         } elsif ($device eq 'ne2k') {
1727             $romfile = 'pxe-ne2k_pci.rom';
1728         } elsif ($device eq 'pcnet') {
1729             $romfile = 'pxe-pcnet.rom';
1730         } elsif ($device eq 'rtl8139') {
1731             $romfile = 'pxe-rtl8139.rom';
1732         }
1733         $tmpstr .= ",romfile=$romfile" if $romfile;
1734     }
1735
1736     return $tmpstr;
1737 }
1738
1739 sub print_netdev_full {
1740     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1741
1742     my $i = '';
1743     if ($netid =~ m/^net(\d+)$/) {
1744         $i = int($1);
1745     }
1746
1747     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749     my $ifname = "tap${vmid}i$i";
1750
1751     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752     die "interface name '$ifname' is too long (max 15 character)\n"
1753         if length($ifname) >= 16;
1754
1755     my $vhostparam = '';
1756     if (is_native_arch($arch)) {
1757         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1758     }
1759
1760     my $vmname = $conf->{name} || "vm$vmid";
1761
1762     my $netdev = "";
1763     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765     if ($net->{bridge}) {
1766         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1768     } else {
1769         $netdev = "type=user,id=$netid,hostname=$vmname";
1770     }
1771
1772     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1773
1774     return $netdev;
1775 }
1776
1777 my $vga_map = {
1778     'cirrus' => 'cirrus-vga',
1779     'std' => 'VGA',
1780     'vmware' => 'vmware-svga',
1781     'virtio' => 'virtio-vga',
1782     'virtio-gl' => 'virtio-vga-gl',
1783 };
1784
1785 sub print_vga_device {
1786     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1787
1788     my $type = $vga_map->{$vga->{type}};
1789     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1790         $type = 'virtio-gpu';
1791     }
1792     my $vgamem_mb = $vga->{memory};
1793
1794     my $max_outputs = '';
1795     if ($qxlnum) {
1796         $type = $id ? 'qxl' : 'qxl-vga';
1797
1798         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1799             # set max outputs so linux can have up to 4 qxl displays with one device
1800             if (min_version($machine_version, 4, 1)) {
1801                 $max_outputs = ",max_outputs=4";
1802             }
1803         }
1804     }
1805
1806     die "no devicetype for $vga->{type}\n" if !$type;
1807
1808     my $memory = "";
1809     if ($vgamem_mb) {
1810         if ($vga->{type} =~ /^virtio/) {
1811             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1812             $memory = ",max_hostmem=$bytes";
1813         } elsif ($qxlnum) {
1814             # from https://www.spice-space.org/multiple-monitors.html
1815             $memory = ",vgamem_mb=$vga->{memory}";
1816             my $ram = $vgamem_mb * 4;
1817             my $vram = $vgamem_mb * 2;
1818             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1819         } else {
1820             $memory = ",vgamem_mb=$vga->{memory}";
1821         }
1822     } elsif ($qxlnum && $id) {
1823         $memory = ",ram_size=67108864,vram_size=33554432";
1824     }
1825
1826     my $edidoff = "";
1827     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1828         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1829     }
1830
1831     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1832     my $vgaid = "vga" . ($id // '');
1833     my $pciaddr;
1834     if ($q35 && $vgaid eq 'vga') {
1835         # the first display uses pcie.0 bus on q35 machines
1836         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1837     } else {
1838         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1839     }
1840
1841     if ($vga->{type} eq 'virtio-gl') {
1842         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1843         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1844             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1845
1846         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1847             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1848     }
1849
1850     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1851 }
1852
1853 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1854 sub parse_net {
1855     my ($data, $disable_mac_autogen) = @_;
1856
1857     my $res = eval { parse_property_string($net_fmt, $data) };
1858     if ($@) {
1859         warn $@;
1860         return;
1861     }
1862     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1863         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1864         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1865     }
1866     return $res;
1867 }
1868
1869 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1870 sub parse_ipconfig {
1871     my ($data) = @_;
1872
1873     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1874     if ($@) {
1875         warn $@;
1876         return;
1877     }
1878
1879     if ($res->{gw} && !$res->{ip}) {
1880         warn 'gateway specified without specifying an IP address';
1881         return;
1882     }
1883     if ($res->{gw6} && !$res->{ip6}) {
1884         warn 'IPv6 gateway specified without specifying an IPv6 address';
1885         return;
1886     }
1887     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1888         warn 'gateway specified together with DHCP';
1889         return;
1890     }
1891     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1892         # gw6 + auto/dhcp
1893         warn "IPv6 gateway specified together with $res->{ip6} address";
1894         return;
1895     }
1896
1897     if (!$res->{ip} && !$res->{ip6}) {
1898         return { ip => 'dhcp', ip6 => 'dhcp' };
1899     }
1900
1901     return $res;
1902 }
1903
1904 sub print_net {
1905     my $net = shift;
1906
1907     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1908 }
1909
1910 sub add_random_macs {
1911     my ($settings) = @_;
1912
1913     foreach my $opt (keys %$settings) {
1914         next if $opt !~ m/^net(\d+)$/;
1915         my $net = parse_net($settings->{$opt});
1916         next if !$net;
1917         $settings->{$opt} = print_net($net);
1918     }
1919 }
1920
1921 sub vm_is_volid_owner {
1922     my ($storecfg, $vmid, $volid) = @_;
1923
1924     if ($volid !~  m|^/|) {
1925         my ($path, $owner);
1926         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1927         if ($owner && ($owner == $vmid)) {
1928             return 1;
1929         }
1930     }
1931
1932     return;
1933 }
1934
1935 sub vmconfig_register_unused_drive {
1936     my ($storecfg, $vmid, $conf, $drive) = @_;
1937
1938     if (drive_is_cloudinit($drive)) {
1939         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1940         warn $@ if $@;
1941         delete $conf->{cloudinit};
1942     } elsif (!drive_is_cdrom($drive)) {
1943         my $volid = $drive->{file};
1944         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1945             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1946         }
1947     }
1948 }
1949
1950 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1951 my $smbios1_fmt = {
1952     uuid => {
1953         type => 'string',
1954         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1955         format_description => 'UUID',
1956         description => "Set SMBIOS1 UUID.",
1957         optional => 1,
1958     },
1959     version => {
1960         type => 'string',
1961         pattern => '[A-Za-z0-9+\/]+={0,2}',
1962         format_description => 'Base64 encoded string',
1963         description => "Set SMBIOS1 version.",
1964         optional => 1,
1965     },
1966     serial => {
1967         type => 'string',
1968         pattern => '[A-Za-z0-9+\/]+={0,2}',
1969         format_description => 'Base64 encoded string',
1970         description => "Set SMBIOS1 serial number.",
1971         optional => 1,
1972     },
1973     manufacturer => {
1974         type => 'string',
1975         pattern => '[A-Za-z0-9+\/]+={0,2}',
1976         format_description => 'Base64 encoded string',
1977         description => "Set SMBIOS1 manufacturer.",
1978         optional => 1,
1979     },
1980     product => {
1981         type => 'string',
1982         pattern => '[A-Za-z0-9+\/]+={0,2}',
1983         format_description => 'Base64 encoded string',
1984         description => "Set SMBIOS1 product ID.",
1985         optional => 1,
1986     },
1987     sku => {
1988         type => 'string',
1989         pattern => '[A-Za-z0-9+\/]+={0,2}',
1990         format_description => 'Base64 encoded string',
1991         description => "Set SMBIOS1 SKU string.",
1992         optional => 1,
1993     },
1994     family => {
1995         type => 'string',
1996         pattern => '[A-Za-z0-9+\/]+={0,2}',
1997         format_description => 'Base64 encoded string',
1998         description => "Set SMBIOS1 family string.",
1999         optional => 1,
2000     },
2001     base64 => {
2002         type => 'boolean',
2003         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2004         optional => 1,
2005     },
2006 };
2007
2008 sub parse_smbios1 {
2009     my ($data) = @_;
2010
2011     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2012     warn $@ if $@;
2013     return $res;
2014 }
2015
2016 sub print_smbios1 {
2017     my ($smbios1) = @_;
2018     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2019 }
2020
2021 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2022
2023 sub parse_watchdog {
2024     my ($value) = @_;
2025
2026     return if !$value;
2027
2028     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2029     warn $@ if $@;
2030     return $res;
2031 }
2032
2033 sub parse_guest_agent {
2034     my ($conf) = @_;
2035
2036     return {} if !defined($conf->{agent});
2037
2038     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2039     warn $@ if $@;
2040
2041     # if the agent is disabled ignore the other potentially set properties
2042     return {} if !$res->{enabled};
2043     return $res;
2044 }
2045
2046 sub get_qga_key {
2047     my ($conf, $key) = @_;
2048     return undef if !defined($conf->{agent});
2049
2050     my $agent = parse_guest_agent($conf);
2051     return $agent->{$key};
2052 }
2053
2054 sub parse_vga {
2055     my ($value) = @_;
2056
2057     return {} if !$value;
2058     my $res = eval { parse_property_string($vga_fmt, $value) };
2059     warn $@ if $@;
2060     return $res;
2061 }
2062
2063 sub parse_rng {
2064     my ($value) = @_;
2065
2066     return if !$value;
2067
2068     my $res = eval { parse_property_string($rng_fmt, $value) };
2069     warn $@ if $@;
2070     return $res;
2071 }
2072
2073 sub parse_meta_info {
2074     my ($value) = @_;
2075
2076     return if !$value;
2077
2078     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2079     warn $@ if $@;
2080     return $res;
2081 }
2082
2083 sub new_meta_info_string {
2084     my () = @_; # for now do not allow to override any value
2085
2086     return PVE::JSONSchema::print_property_string(
2087         {
2088             'creation-qemu' => kvm_user_version(),
2089             ctime => "". int(time()),
2090         },
2091         $meta_info_fmt
2092     );
2093 }
2094
2095 sub qemu_created_version_fixups {
2096     my ($conf, $forcemachine, $kvmver) = @_;
2097
2098     my $meta = parse_meta_info($conf->{meta}) // {};
2099     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2100
2101     # check if we need to apply some handling for VMs that always use the latest machine version but
2102     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2103     # would be required (we do not want to pin machine version for non-windows OS type)
2104     if (
2105         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2106         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2107         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2108         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2109     ) {
2110         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2111         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2112             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2113             # and thus with the predictable interface naming of systemd
2114             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2115         }
2116     }
2117     return;
2118 }
2119
2120 # add JSON properties for create and set function
2121 sub json_config_properties {
2122     my ($prop, $with_disk_alloc) = @_;
2123
2124     my $skip_json_config_opts = {
2125         parent => 1,
2126         snaptime => 1,
2127         vmstate => 1,
2128         runningmachine => 1,
2129         runningcpu => 1,
2130         meta => 1,
2131     };
2132
2133     foreach my $opt (keys %$confdesc) {
2134         next if $skip_json_config_opts->{$opt};
2135
2136         if ($with_disk_alloc && is_valid_drivename($opt)) {
2137             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2138         } else {
2139             $prop->{$opt} = $confdesc->{$opt};
2140         }
2141     }
2142
2143     return $prop;
2144 }
2145
2146 # Properties that we can read from an OVF file
2147 sub json_ovf_properties {
2148     my $prop = {};
2149
2150     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2151         $prop->{$device} = {
2152             type => 'string',
2153             format => 'pve-volume-id-or-absolute-path',
2154             description => "Disk image that gets imported to $device",
2155             optional => 1,
2156         };
2157     }
2158
2159     $prop->{cores} = {
2160         type => 'integer',
2161         description => "The number of CPU cores.",
2162         optional => 1,
2163     };
2164     $prop->{memory} = {
2165         type => 'integer',
2166         description => "Amount of RAM for the VM in MB.",
2167         optional => 1,
2168     };
2169     $prop->{name} = {
2170         type => 'string',
2171         description => "Name of the VM.",
2172         optional => 1,
2173     };
2174
2175     return $prop;
2176 }
2177
2178 # return copy of $confdesc_cloudinit to generate documentation
2179 sub cloudinit_config_properties {
2180
2181     return dclone($confdesc_cloudinit);
2182 }
2183
2184 sub cloudinit_pending_properties {
2185     my $p = {
2186         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2187         name => 1,
2188     };
2189     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2190     return $p;
2191 }
2192
2193 sub check_type {
2194     my ($key, $value) = @_;
2195
2196     die "unknown setting '$key'\n" if !$confdesc->{$key};
2197
2198     my $type = $confdesc->{$key}->{type};
2199
2200     if (!defined($value)) {
2201         die "got undefined value\n";
2202     }
2203
2204     if ($value =~ m/[\n\r]/) {
2205         die "property contains a line feed\n";
2206     }
2207
2208     if ($type eq 'boolean') {
2209         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2210         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2211         die "type check ('boolean') failed - got '$value'\n";
2212     } elsif ($type eq 'integer') {
2213         return int($1) if $value =~ m/^(\d+)$/;
2214         die "type check ('integer') failed - got '$value'\n";
2215     } elsif ($type eq 'number') {
2216         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2217         die "type check ('number') failed - got '$value'\n";
2218     } elsif ($type eq 'string') {
2219         if (my $fmt = $confdesc->{$key}->{format}) {
2220             PVE::JSONSchema::check_format($fmt, $value);
2221             return $value;
2222         }
2223         $value =~ s/^\"(.*)\"$/$1/;
2224         return $value;
2225     } else {
2226         die "internal error"
2227     }
2228 }
2229
2230 sub destroy_vm {
2231     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2232
2233     my $conf = PVE::QemuConfig->load_config($vmid);
2234
2235     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2236         PVE::QemuConfig->check_lock($conf);
2237     }
2238
2239     if ($conf->{template}) {
2240         # check if any base image is still used by a linked clone
2241         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2242                 my ($ds, $drive) = @_;
2243                 return if drive_is_cdrom($drive);
2244
2245                 my $volid = $drive->{file};
2246                 return if !$volid || $volid =~ m|^/|;
2247
2248                 die "base volume '$volid' is still in use by linked cloned\n"
2249                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2250
2251         });
2252     }
2253
2254     my $volids = {};
2255     my $remove_owned_drive = sub {
2256         my ($ds, $drive) = @_;
2257         return if drive_is_cdrom($drive, 1);
2258
2259         my $volid = $drive->{file};
2260         return if !$volid || $volid =~ m|^/|;
2261         return if $volids->{$volid};
2262
2263         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2264         return if !$path || !$owner || ($owner != $vmid);
2265
2266         $volids->{$volid} = 1;
2267         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2268         warn "Could not remove disk '$volid', check manually: $@" if $@;
2269     };
2270
2271     # only remove disks owned by this VM (referenced in the config)
2272     my $include_opts = {
2273         include_unused => 1,
2274         extra_keys => ['vmstate'],
2275     };
2276     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2277
2278     for my $snap (values %{$conf->{snapshots}}) {
2279         next if !defined($snap->{vmstate});
2280         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2281         next if !defined($drive);
2282         $remove_owned_drive->('vmstate', $drive);
2283     }
2284
2285     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2286
2287     if ($purge_unreferenced) { # also remove unreferenced disk
2288         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2289         PVE::Storage::foreach_volid($vmdisks, sub {
2290             my ($volid, $sid, $volname, $d) = @_;
2291             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2292             warn $@ if $@;
2293         });
2294     }
2295
2296     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2297     warn $@ if $@;
2298
2299     if (defined $replacement_conf) {
2300         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2301     } else {
2302         PVE::QemuConfig->destroy_config($vmid);
2303     }
2304 }
2305
2306 sub parse_vm_config {
2307     my ($filename, $raw, $strict) = @_;
2308
2309     return if !defined($raw);
2310
2311     my $res = {
2312         digest => Digest::SHA::sha1_hex($raw),
2313         snapshots => {},
2314         pending => {},
2315         cloudinit => {},
2316     };
2317
2318     my $handle_error = sub {
2319         my ($msg) = @_;
2320
2321         if ($strict) {
2322             die $msg;
2323         } else {
2324             warn $msg;
2325         }
2326     };
2327
2328     $filename =~ m|/qemu-server/(\d+)\.conf$|
2329         || die "got strange filename '$filename'";
2330
2331     my $vmid = $1;
2332
2333     my $conf = $res;
2334     my $descr;
2335     my $finish_description = sub {
2336         if (defined($descr)) {
2337             $descr =~ s/\s+$//;
2338             $conf->{description} = $descr;
2339         }
2340         $descr = undef;
2341     };
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             $finish_description->();
2351             $conf = $res->{$section} = {};
2352             next;
2353         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2354             $section = 'cloudinit';
2355             $finish_description->();
2356             $conf = $res->{$section} = {};
2357             next;
2358
2359         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2360             $section = $1;
2361             $finish_description->();
2362             $conf = $res->{snapshots}->{$section} = {};
2363             next;
2364         }
2365
2366         if ($line =~ m/^\#(.*)$/) {
2367             $descr = '' if !defined($descr);
2368             $descr .= PVE::Tools::decode_text($1) . "\n";
2369             next;
2370         }
2371
2372         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2373             $descr = '' if !defined($descr);
2374             $descr .= PVE::Tools::decode_text($2);
2375         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2376             $conf->{snapstate} = $1;
2377         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2378             my $key = $1;
2379             my $value = $2;
2380             $conf->{$key} = $value;
2381         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2382             my $value = $1;
2383             if ($section eq 'pending') {
2384                 $conf->{delete} = $value; # we parse this later
2385             } else {
2386                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2387             }
2388         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             if ($section eq 'cloudinit') {
2392                 # ignore validation only used for informative purpose
2393                 $conf->{$key} = $value;
2394                 next;
2395             }
2396             eval { $value = check_type($key, $value); };
2397             if ($@) {
2398                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2399             } else {
2400                 $key = 'ide2' if $key eq 'cdrom';
2401                 my $fmt = $confdesc->{$key}->{format};
2402                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2403                     my $v = parse_drive($key, $value);
2404                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2405                         $v->{file} = $volid;
2406                         $value = print_drive($v);
2407                     } else {
2408                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2409                         next;
2410                     }
2411                 }
2412
2413                 $conf->{$key} = $value;
2414             }
2415         } else {
2416             $handle_error->("vm $vmid - unable to parse config: $line\n");
2417         }
2418     }
2419
2420     $finish_description->();
2421     delete $res->{snapstate}; # just to be sure
2422
2423     return $res;
2424 }
2425
2426 sub write_vm_config {
2427     my ($filename, $conf) = @_;
2428
2429     delete $conf->{snapstate}; # just to be sure
2430
2431     if ($conf->{cdrom}) {
2432         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2433         $conf->{ide2} = $conf->{cdrom};
2434         delete $conf->{cdrom};
2435     }
2436
2437     # we do not use 'smp' any longer
2438     if ($conf->{sockets}) {
2439         delete $conf->{smp};
2440     } elsif ($conf->{smp}) {
2441         $conf->{sockets} = $conf->{smp};
2442         delete $conf->{cores};
2443         delete $conf->{smp};
2444     }
2445
2446     my $used_volids = {};
2447
2448     my $cleanup_config = sub {
2449         my ($cref, $pending, $snapname) = @_;
2450
2451         foreach my $key (keys %$cref) {
2452             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2453                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2454             my $value = $cref->{$key};
2455             if ($key eq 'delete') {
2456                 die "propertry 'delete' is only allowed in [PENDING]\n"
2457                     if !$pending;
2458                 # fixme: check syntax?
2459                 next;
2460             }
2461             eval { $value = check_type($key, $value); };
2462             die "unable to parse value of '$key' - $@" if $@;
2463
2464             $cref->{$key} = $value;
2465
2466             if (!$snapname && is_valid_drivename($key)) {
2467                 my $drive = parse_drive($key, $value);
2468                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2469             }
2470         }
2471     };
2472
2473     &$cleanup_config($conf);
2474
2475     &$cleanup_config($conf->{pending}, 1);
2476
2477     foreach my $snapname (keys %{$conf->{snapshots}}) {
2478         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2479         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2480     }
2481
2482     # remove 'unusedX' settings if we re-add a volume
2483     foreach my $key (keys %$conf) {
2484         my $value = $conf->{$key};
2485         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2486             delete $conf->{$key};
2487         }
2488     }
2489
2490     my $generate_raw_config = sub {
2491         my ($conf, $pending) = @_;
2492
2493         my $raw = '';
2494
2495         # add description as comment to top of file
2496         if (defined(my $descr = $conf->{description})) {
2497             if ($descr) {
2498                 foreach my $cl (split(/\n/, $descr)) {
2499                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2500                 }
2501             } else {
2502                 $raw .= "#\n" if $pending;
2503             }
2504         }
2505
2506         foreach my $key (sort keys %$conf) {
2507             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2508             $raw .= "$key: $conf->{$key}\n";
2509         }
2510         return $raw;
2511     };
2512
2513     my $raw = &$generate_raw_config($conf);
2514
2515     if (scalar(keys %{$conf->{pending}})){
2516         $raw .= "\n[PENDING]\n";
2517         $raw .= &$generate_raw_config($conf->{pending}, 1);
2518     }
2519
2520     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2521         $raw .= "\n[special:cloudinit]\n";
2522         $raw .= &$generate_raw_config($conf->{cloudinit});
2523     }
2524
2525     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2526         $raw .= "\n[$snapname]\n";
2527         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2528     }
2529
2530     return $raw;
2531 }
2532
2533 sub load_defaults {
2534
2535     my $res = {};
2536
2537     # we use static defaults from our JSON schema configuration
2538     foreach my $key (keys %$confdesc) {
2539         if (defined(my $default = $confdesc->{$key}->{default})) {
2540             $res->{$key} = $default;
2541         }
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub config_list {
2548     my $vmlist = PVE::Cluster::get_vmlist();
2549     my $res = {};
2550     return $res if !$vmlist || !$vmlist->{ids};
2551     my $ids = $vmlist->{ids};
2552     my $nodename = nodename();
2553
2554     foreach my $vmid (keys %$ids) {
2555         my $d = $ids->{$vmid};
2556         next if !$d->{node} || $d->{node} ne $nodename;
2557         next if !$d->{type} || $d->{type} ne 'qemu';
2558         $res->{$vmid}->{exists} = 1;
2559     }
2560     return $res;
2561 }
2562
2563 # test if VM uses local resources (to prevent migration)
2564 sub check_local_resources {
2565     my ($conf, $noerr) = @_;
2566
2567     my @loc_res = ();
2568     my $mapped_res = [];
2569
2570     my $nodelist = PVE::Cluster::get_nodelist();
2571     my $pci_map = PVE::Mapping::PCI::config();
2572     my $usb_map = PVE::Mapping::USB::config();
2573
2574     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2575
2576     my $add_missing_mapping = sub {
2577         my ($type, $key, $id) = @_;
2578         for my $node (@$nodelist) {
2579             my $entry;
2580             if ($type eq 'pci') {
2581                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2582             } elsif ($type eq 'usb') {
2583                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2584             }
2585             if (!scalar($entry->@*)) {
2586                 push @{$missing_mappings_by_node->{$node}}, $key;
2587             }
2588         }
2589     };
2590
2591     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2592     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2593
2594     push @loc_res, "ivshmem" if $conf->{ivshmem};
2595
2596     foreach my $k (keys %$conf) {
2597         if ($k =~ m/^usb/) {
2598             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2599             next if $entry->{host} =~ m/^spice$/i;
2600             if ($entry->{mapping}) {
2601                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2602                 push @$mapped_res, $k;
2603             }
2604         }
2605         if ($k =~ m/^hostpci/) {
2606             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2607             if ($entry->{mapping}) {
2608                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2609                 push @$mapped_res, $k;
2610             }
2611         }
2612         # sockets are safe: they will recreated be on the target side post-migrate
2613         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2614         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2615     }
2616
2617     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2618
2619     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2620 }
2621
2622 # check if used storages are available on all nodes (use by migrate)
2623 sub check_storage_availability {
2624     my ($storecfg, $conf, $node) = @_;
2625
2626     PVE::QemuConfig->foreach_volume($conf, sub {
2627         my ($ds, $drive) = @_;
2628
2629         my $volid = $drive->{file};
2630         return if !$volid;
2631
2632         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2633         return if !$sid;
2634
2635         # check if storage is available on both nodes
2636         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2637         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2638
2639         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2640
2641         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2642             if !$scfg->{content}->{$vtype};
2643    });
2644 }
2645
2646 # list nodes where all VM images are available (used by has_feature API)
2647 sub shared_nodes {
2648     my ($conf, $storecfg) = @_;
2649
2650     my $nodelist = PVE::Cluster::get_nodelist();
2651     my $nodehash = { map { $_ => 1 } @$nodelist };
2652     my $nodename = nodename();
2653
2654     PVE::QemuConfig->foreach_volume($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         if ($storeid) {
2662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2663             if ($scfg->{disable}) {
2664                 $nodehash = {};
2665             } elsif (my $avail = $scfg->{nodes}) {
2666                 foreach my $node (keys %$nodehash) {
2667                     delete $nodehash->{$node} if !$avail->{$node};
2668                 }
2669             } elsif (!$scfg->{shared}) {
2670                 foreach my $node (keys %$nodehash) {
2671                     delete $nodehash->{$node} if $node ne $nodename
2672                 }
2673             }
2674         }
2675     });
2676
2677     return $nodehash
2678 }
2679
2680 sub check_local_storage_availability {
2681     my ($conf, $storecfg) = @_;
2682
2683     my $nodelist = PVE::Cluster::get_nodelist();
2684     my $nodehash = { map { $_ => {} } @$nodelist };
2685
2686     PVE::QemuConfig->foreach_volume($conf, sub {
2687         my ($ds, $drive) = @_;
2688
2689         my $volid = $drive->{file};
2690         return if !$volid;
2691
2692         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2693         if ($storeid) {
2694             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2695
2696             if ($scfg->{disable}) {
2697                 foreach my $node (keys %$nodehash) {
2698                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2699                 }
2700             } elsif (my $avail = $scfg->{nodes}) {
2701                 foreach my $node (keys %$nodehash) {
2702                     if (!$avail->{$node}) {
2703                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2704                     }
2705                 }
2706             }
2707         }
2708     });
2709
2710     foreach my $node (values %$nodehash) {
2711         if (my $unavail = $node->{unavailable_storages}) {
2712             $node->{unavailable_storages} = [ sort keys %$unavail ];
2713         }
2714     }
2715
2716     return $nodehash
2717 }
2718
2719 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2720 sub check_running {
2721     my ($vmid, $nocheck, $node) = @_;
2722
2723     # $nocheck is set when called during a migration, in which case the config
2724     # file might still or already reside on the *other* node
2725     # - because rename has already happened, and current node is source
2726     # - because rename hasn't happened yet, and current node is target
2727     # - because rename has happened, current node is target, but hasn't yet
2728     # processed it yet
2729     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2730     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2731 }
2732
2733 sub vzlist {
2734
2735     my $vzlist = config_list();
2736
2737     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2738
2739     while (defined(my $de = $fd->read)) {
2740         next if $de !~ m/^(\d+)\.pid$/;
2741         my $vmid = $1;
2742         next if !defined($vzlist->{$vmid});
2743         if (my $pid = check_running($vmid)) {
2744             $vzlist->{$vmid}->{pid} = $pid;
2745         }
2746     }
2747
2748     return $vzlist;
2749 }
2750
2751 our $vmstatus_return_properties = {
2752     vmid => get_standard_option('pve-vmid'),
2753     status => {
2754         description => "QEMU process status.",
2755         type => 'string',
2756         enum => ['stopped', 'running'],
2757     },
2758     maxmem => {
2759         description => "Maximum memory in bytes.",
2760         type => 'integer',
2761         optional => 1,
2762         renderer => 'bytes',
2763     },
2764     maxdisk => {
2765         description => "Root disk size in bytes.",
2766         type => 'integer',
2767         optional => 1,
2768         renderer => 'bytes',
2769     },
2770     name => {
2771         description => "VM name.",
2772         type => 'string',
2773         optional => 1,
2774     },
2775     qmpstatus => {
2776         description => "VM run state from the 'query-status' QMP monitor command.",
2777         type => 'string',
2778         optional => 1,
2779     },
2780     pid => {
2781         description => "PID of running qemu process.",
2782         type => 'integer',
2783         optional => 1,
2784     },
2785     uptime => {
2786         description => "Uptime.",
2787         type => 'integer',
2788         optional => 1,
2789         renderer => 'duration',
2790     },
2791     cpus => {
2792         description => "Maximum usable CPUs.",
2793         type => 'number',
2794         optional => 1,
2795     },
2796     lock => {
2797         description => "The current config lock, if any.",
2798         type => 'string',
2799         optional => 1,
2800     },
2801     tags => {
2802         description  => "The current configured tags, if any",
2803         type => 'string',
2804         optional => 1,
2805     },
2806     'running-machine' => {
2807         description => "The currently running machine type (if running).",
2808         type => 'string',
2809         optional => 1,
2810     },
2811     'running-qemu' => {
2812         description => "The currently running QEMU version (if running).",
2813         type => 'string',
2814         optional => 1,
2815     },
2816 };
2817
2818 my $last_proc_pid_stat;
2819
2820 # get VM status information
2821 # This must be fast and should not block ($full == false)
2822 # We only query KVM using QMP if $full == true (this can be slow)
2823 sub vmstatus {
2824     my ($opt_vmid, $full) = @_;
2825
2826     my $res = {};
2827
2828     my $storecfg = PVE::Storage::config();
2829
2830     my $list = vzlist();
2831     my $defaults = load_defaults();
2832
2833     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2834
2835     my $cpucount = $cpuinfo->{cpus} || 1;
2836
2837     foreach my $vmid (keys %$list) {
2838         next if $opt_vmid && ($vmid ne $opt_vmid);
2839
2840         my $conf = PVE::QemuConfig->load_config($vmid);
2841
2842         my $d = { vmid => int($vmid) };
2843         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2844
2845         # fixme: better status?
2846         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2847
2848         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2849         if (defined($size)) {
2850             $d->{disk} = 0; # no info available
2851             $d->{maxdisk} = $size;
2852         } else {
2853             $d->{disk} = 0;
2854             $d->{maxdisk} = 0;
2855         }
2856
2857         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2858             * ($conf->{cores} || $defaults->{cores});
2859         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2860         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2861
2862         $d->{name} = $conf->{name} || "VM $vmid";
2863         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2864
2865         if ($conf->{balloon}) {
2866             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2867             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2868                 : $defaults->{shares};
2869         }
2870
2871         $d->{uptime} = 0;
2872         $d->{cpu} = 0;
2873         $d->{mem} = 0;
2874
2875         $d->{netout} = 0;
2876         $d->{netin} = 0;
2877
2878         $d->{diskread} = 0;
2879         $d->{diskwrite} = 0;
2880
2881         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2882
2883         $d->{serial} = 1 if conf_has_serial($conf);
2884         $d->{lock} = $conf->{lock} if $conf->{lock};
2885         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2886
2887         $res->{$vmid} = $d;
2888     }
2889
2890     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2891     foreach my $dev (keys %$netdev) {
2892         next if $dev !~ m/^tap([1-9]\d*)i/;
2893         my $vmid = $1;
2894         my $d = $res->{$vmid};
2895         next if !$d;
2896
2897         $d->{netout} += $netdev->{$dev}->{receive};
2898         $d->{netin} += $netdev->{$dev}->{transmit};
2899
2900         if ($full) {
2901             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2902             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2903         }
2904
2905     }
2906
2907     my $ctime = gettimeofday;
2908
2909     foreach my $vmid (keys %$list) {
2910
2911         my $d = $res->{$vmid};
2912         my $pid = $d->{pid};
2913         next if !$pid;
2914
2915         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2916         next if !$pstat; # not running
2917
2918         my $used = $pstat->{utime} + $pstat->{stime};
2919
2920         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2921
2922         if ($pstat->{vsize}) {
2923             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2924         }
2925
2926         my $old = $last_proc_pid_stat->{$pid};
2927         if (!$old) {
2928             $last_proc_pid_stat->{$pid} = {
2929                 time => $ctime,
2930                 used => $used,
2931                 cpu => 0,
2932             };
2933             next;
2934         }
2935
2936         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2937
2938         if ($dtime > 1000) {
2939             my $dutime = $used -  $old->{used};
2940
2941             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2942             $last_proc_pid_stat->{$pid} = {
2943                 time => $ctime,
2944                 used => $used,
2945                 cpu => $d->{cpu},
2946             };
2947         } else {
2948             $d->{cpu} = $old->{cpu};
2949         }
2950     }
2951
2952     return $res if !$full;
2953
2954     my $qmpclient = PVE::QMPClient->new();
2955
2956     my $ballooncb = sub {
2957         my ($vmid, $resp) = @_;
2958
2959         my $info = $resp->{'return'};
2960         return if !$info->{max_mem};
2961
2962         my $d = $res->{$vmid};
2963
2964         # use memory assigned to VM
2965         $d->{maxmem} = $info->{max_mem};
2966         $d->{balloon} = $info->{actual};
2967
2968         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2969             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2970             $d->{freemem} = $info->{free_mem};
2971         }
2972
2973         $d->{ballooninfo} = $info;
2974     };
2975
2976     my $blockstatscb = sub {
2977         my ($vmid, $resp) = @_;
2978         my $data = $resp->{'return'} || [];
2979         my $totalrdbytes = 0;
2980         my $totalwrbytes = 0;
2981
2982         for my $blockstat (@$data) {
2983             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2984             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2985
2986             $blockstat->{device} =~ s/drive-//;
2987             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2988         }
2989         $res->{$vmid}->{diskread} = $totalrdbytes;
2990         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2991     };
2992
2993     my $machinecb = sub {
2994         my ($vmid, $resp) = @_;
2995         my $data = $resp->{'return'} || [];
2996
2997         $res->{$vmid}->{'running-machine'} =
2998             PVE::QemuServer::Machine::current_from_query_machines($data);
2999     };
3000
3001     my $versioncb = sub {
3002         my ($vmid, $resp) = @_;
3003         my $data = $resp->{'return'} // {};
3004         my $version = 'unknown';
3005
3006         if (my $v = $data->{qemu}) {
3007             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3008         }
3009
3010         $res->{$vmid}->{'running-qemu'} = $version;
3011     };
3012
3013     my $statuscb = sub {
3014         my ($vmid, $resp) = @_;
3015
3016         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3017         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3018         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3019         # this fails if ballon driver is not loaded, so this must be
3020         # the last commnand (following command are aborted if this fails).
3021         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3022
3023         my $status = 'unknown';
3024         if (!defined($status = $resp->{'return'}->{status})) {
3025             warn "unable to get VM status\n";
3026             return;
3027         }
3028
3029         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3030     };
3031
3032     foreach my $vmid (keys %$list) {
3033         next if $opt_vmid && ($vmid ne $opt_vmid);
3034         next if !$res->{$vmid}->{pid}; # not running
3035         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3036     }
3037
3038     $qmpclient->queue_execute(undef, 2);
3039
3040     foreach my $vmid (keys %$list) {
3041         next if $opt_vmid && ($vmid ne $opt_vmid);
3042         next if !$res->{$vmid}->{pid}; #not running
3043
3044         # we can't use the $qmpclient since it might have already aborted on
3045         # 'query-balloon', but this might also fail for older versions...
3046         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3047         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3048     }
3049
3050     foreach my $vmid (keys %$list) {
3051         next if $opt_vmid && ($vmid ne $opt_vmid);
3052         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3053     }
3054
3055     return $res;
3056 }
3057
3058 sub conf_has_serial {
3059     my ($conf) = @_;
3060
3061     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3062         if ($conf->{"serial$i"}) {
3063             return 1;
3064         }
3065     }
3066
3067     return 0;
3068 }
3069
3070 sub conf_has_audio {
3071     my ($conf, $id) = @_;
3072
3073     $id //= 0;
3074     my $audio = $conf->{"audio$id"};
3075     return if !defined($audio);
3076
3077     my $audioproperties = parse_property_string($audio_fmt, $audio);
3078     my $audiodriver = $audioproperties->{driver} // 'spice';
3079
3080     return {
3081         dev => $audioproperties->{device},
3082         dev_id => "audiodev$id",
3083         backend => $audiodriver,
3084         backend_id => "$audiodriver-backend${id}",
3085     };
3086 }
3087
3088 sub audio_devs {
3089     my ($audio, $audiopciaddr, $machine_version) = @_;
3090
3091     my $devs = [];
3092
3093     my $id = $audio->{dev_id};
3094     my $audiodev = "";
3095     if (min_version($machine_version, 4, 2)) {
3096         $audiodev = ",audiodev=$audio->{backend_id}";
3097     }
3098
3099     if ($audio->{dev} eq 'AC97') {
3100         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3101     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3102         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3103         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3104         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3105     } else {
3106         die "unkown audio device '$audio->{dev}', implement me!";
3107     }
3108
3109     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3110
3111     return $devs;
3112 }
3113
3114 sub get_tpm_paths {
3115     my ($vmid) = @_;
3116     return {
3117         socket => "/var/run/qemu-server/$vmid.swtpm",
3118         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3119     };
3120 }
3121
3122 sub add_tpm_device {
3123     my ($vmid, $devices, $conf) = @_;
3124
3125     return if !$conf->{tpmstate0};
3126
3127     my $paths = get_tpm_paths($vmid);
3128
3129     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3130     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3131     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3132 }
3133
3134 sub start_swtpm {
3135     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3136
3137     return if !$tpmdrive;
3138
3139     my $state;
3140     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3141     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3142     if ($storeid) {
3143         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3144     } else {
3145         $state = $tpm->{file};
3146     }
3147
3148     my $paths = get_tpm_paths($vmid);
3149
3150     # during migration, we will get state from remote
3151     #
3152     if (!$migration) {
3153         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3154         my $setup_cmd = [
3155             "swtpm_setup",
3156             "--tpmstate",
3157             "file://$state",
3158             "--createek",
3159             "--create-ek-cert",
3160             "--create-platform-cert",
3161             "--lock-nvram",
3162             "--config",
3163             "/etc/swtpm_setup.conf", # do not use XDG configs
3164             "--runas",
3165             "0", # force creation as root, error if not possible
3166             "--not-overwrite", # ignore existing state, do not modify
3167         ];
3168
3169         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3170         # TPM 2.0 supports ECC crypto, use if possible
3171         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3172
3173         run_command($setup_cmd, outfunc => sub {
3174             print "swtpm_setup: $1\n";
3175         });
3176     }
3177
3178     # Used to distinguish different invocations in the log.
3179     my $log_prefix = "[id=" . int(time()) . "] ";
3180
3181     my $emulator_cmd = [
3182         "swtpm",
3183         "socket",
3184         "--tpmstate",
3185         "backend-uri=file://$state,mode=0600",
3186         "--ctrl",
3187         "type=unixio,path=$paths->{socket},mode=0600",
3188         "--pid",
3189         "file=$paths->{pid}",
3190         "--terminate", # terminate on QEMU disconnect
3191         "--daemon",
3192         "--log",
3193         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3194     ];
3195     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3196     run_command($emulator_cmd, outfunc => sub { print $1; });
3197
3198     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3199     while (! -e $paths->{pid}) {
3200         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3201         usleep(50_000);
3202     }
3203
3204     # return untainted PID of swtpm daemon so it can be killed on error
3205     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3206     return $1;
3207 }
3208
3209 sub vga_conf_has_spice {
3210     my ($vga) = @_;
3211
3212     my $vgaconf = parse_vga($vga);
3213     my $vgatype = $vgaconf->{type};
3214     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3215
3216     return $1 || 1;
3217 }
3218
3219 sub get_vm_arch {
3220     my ($conf) = @_;
3221     return $conf->{arch} // get_host_arch();
3222 }
3223
3224 my $default_machines = {
3225     x86_64 => 'pc',
3226     aarch64 => 'virt',
3227 };
3228
3229 sub get_installed_machine_version {
3230     my ($kvmversion) = @_;
3231     $kvmversion = kvm_user_version() if !defined($kvmversion);
3232     $kvmversion =~ m/^(\d+\.\d+)/;
3233     return $1;
3234 }
3235
3236 sub windows_get_pinned_machine_version {
3237     my ($machine, $base_version, $kvmversion) = @_;
3238
3239     my $pin_version = $base_version;
3240     if (!defined($base_version) ||
3241         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3242     ) {
3243         $pin_version = get_installed_machine_version($kvmversion);
3244     }
3245     if (!$machine || $machine eq 'pc') {
3246         $machine = "pc-i440fx-$pin_version";
3247     } elsif ($machine eq 'q35') {
3248         $machine = "pc-q35-$pin_version";
3249     } elsif ($machine eq 'virt') {
3250         $machine = "virt-$pin_version";
3251     } else {
3252         warn "unknown machine type '$machine', not touching that!\n";
3253     }
3254
3255     return $machine;
3256 }
3257
3258 sub get_vm_machine {
3259     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3260
3261     my $machine = $forcemachine || $conf->{machine};
3262
3263     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3264         $kvmversion //= kvm_user_version();
3265         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3266         # layout which confuses windows quite a bit and may result in various regressions..
3267         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3268         if (windows_version($conf->{ostype})) {
3269             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3270         }
3271         $arch //= 'x86_64';
3272         $machine ||= $default_machines->{$arch};
3273         if ($add_pve_version) {
3274             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3275             $machine .= "+pve$pvever";
3276         }
3277     }
3278
3279     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3280         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3281         $machine = $1 if $is_pxe;
3282
3283         # for version-pinned machines that do not include a pve-version (e.g.
3284         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3285         $machine .= '+pve0';
3286
3287         $machine .= '.pxe' if $is_pxe;
3288     }
3289
3290     return $machine;
3291 }
3292
3293 sub get_ovmf_files($$$) {
3294     my ($arch, $efidisk, $smm) = @_;
3295
3296     my $types = $OVMF->{$arch}
3297         or die "no OVMF images known for architecture '$arch'\n";
3298
3299     my $type = 'default';
3300     if ($arch eq 'x86_64') {
3301         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3302             $type = $smm ? "4m" : "4m-no-smm";
3303             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3304         } else {
3305             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3306         }
3307     }
3308
3309     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3310     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3311     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3312
3313     return ($ovmf_code, $ovmf_vars);
3314 }
3315
3316 my $Arch2Qemu = {
3317     aarch64 => '/usr/bin/qemu-system-aarch64',
3318     x86_64 => '/usr/bin/qemu-system-x86_64',
3319 };
3320 sub get_command_for_arch($) {
3321     my ($arch) = @_;
3322     return '/usr/bin/kvm' if is_native_arch($arch);
3323
3324     my $cmd = $Arch2Qemu->{$arch}
3325         or die "don't know how to emulate architecture '$arch'\n";
3326     return $cmd;
3327 }
3328
3329 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3330 # to use in a QEMU command line (-cpu element), first array_intersect the result
3331 # of query_supported_ with query_understood_. This is necessary because:
3332 #
3333 # a) query_understood_ returns flags the host cannot use and
3334 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3335 #    flags, but CPU settings - with most of them being flags. Those settings
3336 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3337 #
3338 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3339 # expensive. If you need the value returned from this, you can get it much
3340 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3341 # $accel being 'kvm' or 'tcg'.
3342 #
3343 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3344 # changes, automatically populating pmxcfs.
3345 #
3346 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3347 # since kvm and tcg machines support different flags
3348 #
3349 sub query_supported_cpu_flags {
3350     my ($arch) = @_;
3351
3352     $arch //= get_host_arch();
3353     my $default_machine = $default_machines->{$arch};
3354
3355     my $flags = {};
3356
3357     # FIXME: Once this is merged, the code below should work for ARM as well:
3358     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3359     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3360         $arch eq "aarch64";
3361
3362     my $kvm_supported = defined(kvm_version());
3363     my $qemu_cmd = get_command_for_arch($arch);
3364     my $fakevmid = -1;
3365     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3366
3367     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3368     my $query_supported_run_qemu = sub {
3369         my ($kvm) = @_;
3370
3371         my $flags = {};
3372         my $cmd = [
3373             $qemu_cmd,
3374             '-machine', $default_machine,
3375             '-display', 'none',
3376             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3377             '-mon', 'chardev=qmp,mode=control',
3378             '-pidfile', $pidfile,
3379             '-S', '-daemonize'
3380         ];
3381
3382         if (!$kvm) {
3383             push @$cmd, '-accel', 'tcg';
3384         }
3385
3386         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3387         die "QEMU flag querying VM exited with code " . $rc if $rc;
3388
3389         eval {
3390             my $cmd_result = mon_cmd(
3391                 $fakevmid,
3392                 'query-cpu-model-expansion',
3393                 type => 'full',
3394                 model => { name => 'host' }
3395             );
3396
3397             my $props = $cmd_result->{model}->{props};
3398             foreach my $prop (keys %$props) {
3399                 next if $props->{$prop} ne '1';
3400                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3401                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3402                 # We only keep those with underscores, to match /proc/cpuinfo
3403                 $prop =~ s/\.|-/_/g;
3404                 $flags->{$prop} = 1;
3405             }
3406         };
3407         my $err = $@;
3408
3409         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3410         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3411
3412         die $err if $err;
3413
3414         return [ sort keys %$flags ];
3415     };
3416
3417     # We need to query QEMU twice, since KVM and TCG have different supported flags
3418     PVE::QemuConfig->lock_config($fakevmid, sub {
3419         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3420         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3421
3422         if ($kvm_supported) {
3423             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3424             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3425         }
3426     });
3427
3428     return $flags;
3429 }
3430
3431 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3432 my $understood_cpu_flag_dir = "/usr/share/kvm";
3433 sub query_understood_cpu_flags {
3434     my $arch = get_host_arch();
3435     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3436
3437     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3438         if ! -e $filepath;
3439
3440     my $raw = file_get_contents($filepath);
3441     $raw =~ s/^\s+|\s+$//g;
3442     my @flags = split(/\s+/, $raw);
3443
3444     return \@flags;
3445 }
3446
3447 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3448 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3449 my sub should_disable_smm {
3450     my ($conf, $vga, $machine) = @_;
3451
3452     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3453
3454     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3455         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3456 }
3457
3458 my sub print_ovmf_drive_commandlines {
3459     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3460
3461     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3462
3463     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3464
3465     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3466     if ($d) {
3467         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3468         my ($path, $format) = $d->@{'file', 'format'};
3469         if ($storeid) {
3470             $path = PVE::Storage::path($storecfg, $d->{file});
3471             if (!defined($format)) {
3472                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3473                 $format = qemu_img_format($scfg, $volname);
3474             }
3475         } elsif (!defined($format)) {
3476             die "efidisk format must be specified\n";
3477         }
3478         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3479         if ($path =~ m/^rbd:/) {
3480             $var_drive_str .= ',cache=writeback';
3481             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3482         }
3483         $var_drive_str .= ",format=$format,file=$path";
3484
3485         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3486         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3487     } else {
3488         log_warn("no efidisk configured! Using temporary efivars disk.");
3489         my $path = "/tmp/$vmid-ovmf.fd";
3490         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3491         $var_drive_str .= ",format=raw,file=$path";
3492         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3493     }
3494
3495     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3496 }
3497
3498 sub config_to_command {
3499     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3500         $pbs_backing) = @_;
3501
3502     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3503     my $devices = [];
3504     my $bridges = {};
3505     my $ostype = $conf->{ostype};
3506     my $winversion = windows_version($ostype);
3507     my $kvm = $conf->{kvm};
3508     my $nodename = nodename();
3509
3510     my $arch = get_vm_arch($conf);
3511     my $kvm_binary = get_command_for_arch($arch);
3512     my $kvmver = kvm_user_version($kvm_binary);
3513
3514     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3515         $kvmver //= "undefined";
3516         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3517     }
3518
3519     my $add_pve_version = min_version($kvmver, 4, 1);
3520
3521     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3522     my $machine_version = extract_version($machine_type, $kvmver);
3523     $kvm //= 1 if is_native_arch($arch);
3524
3525     $machine_version =~ m/(\d+)\.(\d+)/;
3526     my ($machine_major, $machine_minor) = ($1, $2);
3527
3528     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3529         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3530     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3531         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3532             ." please upgrade node '$nodename'\n"
3533     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3534         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3535         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3536             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3537             ." node '$nodename'\n";
3538     }
3539
3540     # if a specific +pve version is required for a feature, use $version_guard
3541     # instead of min_version to allow machines to be run with the minimum
3542     # required version
3543     my $required_pve_version = 0;
3544     my $version_guard = sub {
3545         my ($major, $minor, $pve) = @_;
3546         return 0 if !min_version($machine_version, $major, $minor, $pve);
3547         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3548         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3549         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3550         return 1;
3551     };
3552
3553     if ($kvm && !defined kvm_version()) {
3554         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3555             ." or enable in BIOS.\n";
3556     }
3557
3558     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3559     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3560     my $use_old_bios_files = undef;
3561     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3562
3563     my $cmd = [];
3564     if ($conf->{affinity}) {
3565         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3566     }
3567
3568     push @$cmd, $kvm_binary;
3569
3570     push @$cmd, '-id', $vmid;
3571
3572     my $vmname = $conf->{name} || "vm$vmid";
3573
3574     push @$cmd, '-name', "$vmname,debug-threads=on";
3575
3576     push @$cmd, '-no-shutdown';
3577
3578     my $use_virtio = 0;
3579
3580     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3581     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3582     push @$cmd, '-mon', "chardev=qmp,mode=control";
3583
3584     if (min_version($machine_version, 2, 12)) {
3585         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3586         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3587     }
3588
3589     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3590
3591     push @$cmd, '-daemonize';
3592
3593     if ($conf->{smbios1}) {
3594         my $smbios_conf = parse_smbios1($conf->{smbios1});
3595         if ($smbios_conf->{base64}) {
3596             # Do not pass base64 flag to qemu
3597             delete $smbios_conf->{base64};
3598             my $smbios_string = "";
3599             foreach my $key (keys %$smbios_conf) {
3600                 my $value;
3601                 if ($key eq "uuid") {
3602                     $value = $smbios_conf->{uuid}
3603                 } else {
3604                     $value = decode_base64($smbios_conf->{$key});
3605                 }
3606                 # qemu accepts any binary data, only commas need escaping by double comma
3607                 $value =~ s/,/,,/g;
3608                 $smbios_string .= "," . $key . "=" . $value if $value;
3609             }
3610             push @$cmd, '-smbios', "type=1" . $smbios_string;
3611         } else {
3612             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3613         }
3614     }
3615
3616     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3617         die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3618             if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3619
3620         my ($code_drive_str, $var_drive_str) =
3621             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3622         push $cmd->@*, '-drive', $code_drive_str;
3623         push $cmd->@*, '-drive', $var_drive_str;
3624     }
3625
3626     if ($q35) { # tell QEMU to load q35 config early
3627         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628         if (min_version($machine_version, 4, 0)) {
3629             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3630         } else {
3631             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632         }
3633     }
3634
3635     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3636         push @$cmd, $fixups->@*;
3637     }
3638
3639     if ($conf->{vmgenid}) {
3640         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3641     }
3642
3643     # add usb controllers
3644     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3645         $conf, $bridges, $arch, $machine_type, $machine_version);
3646     push @$devices, @usbcontrollers if @usbcontrollers;
3647     my $vga = parse_vga($conf->{vga});
3648
3649     my $qxlnum = vga_conf_has_spice($conf->{vga});
3650     $vga->{type} = 'qxl' if $qxlnum;
3651
3652     if (!$vga->{type}) {
3653         if ($arch eq 'aarch64') {
3654             $vga->{type} = 'virtio';
3655         } elsif (min_version($machine_version, 2, 9)) {
3656             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657         } else {
3658             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659         }
3660     }
3661
3662     # enable absolute mouse coordinates (needed by vnc)
3663     my $tablet = $conf->{tablet};
3664     if (!defined($tablet)) {
3665         $tablet = $defaults->{tablet};
3666         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668     }
3669
3670     if ($tablet) {
3671         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3672         my $kbd = print_keyboarddevice_full($conf, $arch);
3673         push @$devices, '-device', $kbd if defined($kbd);
3674     }
3675
3676     my $bootorder = device_bootorder($conf);
3677
3678     # host pci device passthrough
3679     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3680         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3681
3682     # usb devices
3683     my $usb_dev_features = {};
3684     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3685
3686     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3687         $conf, $usb_dev_features, $bootorder, $machine_version);
3688     push @$devices, @usbdevices if @usbdevices;
3689
3690     # serial devices
3691     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3692         my $path = $conf->{"serial$i"} or next;
3693         if ($path eq 'socket') {
3694             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696             # On aarch64, serial0 is the UART device. QEMU only allows
3697             # connecting UART devices via the '-serial' command line, as
3698             # the device has a fixed slot on the hardware...
3699             if ($arch eq 'aarch64' && $i == 0) {
3700                 push @$devices, '-serial', "chardev:serial$i";
3701             } else {
3702                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703             }
3704         } else {
3705             die "no such serial device\n" if ! -c $path;
3706             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3707             push @$devices, '-device', "isa-serial,chardev=serial$i";
3708         }
3709     }
3710
3711     # parallel devices
3712     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3713         if (my $path = $conf->{"parallel$i"}) {
3714             die "no such parallel device\n" if ! -c $path;
3715             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3716             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718         }
3719     }
3720
3721     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3722         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3723         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3724         push @$devices, @$audio_devs;
3725     }
3726
3727     # Add a TPM only if the VM is not a template,
3728     # to support backing up template VMs even if the TPM disk is write-protected.
3729     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3730
3731     my $sockets = 1;
3732     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3733     $sockets = $conf->{sockets} if  $conf->{sockets};
3734
3735     my $cores = $conf->{cores} || 1;
3736
3737     my $maxcpus = $sockets * $cores;
3738
3739     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3740
3741     my $allowed_vcpus = $cpuinfo->{cpus};
3742
3743     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3744
3745     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3746         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747         for (my $i = 2; $i <= $vcpus; $i++)  {
3748             my $cpustr = print_cpu_device($conf, $arch, $i);
3749             push @$cmd, '-device', $cpustr;
3750         }
3751
3752     } else {
3753
3754         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755     }
3756     push @$cmd, '-nodefaults';
3757
3758     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3759
3760     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3761
3762     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3763
3764     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3765         push @$devices, '-device', print_vga_device(
3766             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3767
3768         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3769
3770         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3771         push @$cmd,  '-vnc', "unix:$socket,password=on";
3772     } else {
3773         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3774         push @$cmd, '-nographic';
3775     }
3776
3777     # time drift fix
3778     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3779     my $useLocaltime = $conf->{localtime};
3780
3781     if ($winversion >= 5) { # windows
3782         $useLocaltime = 1 if !defined($conf->{localtime});
3783
3784         # use time drift fix when acpi is enabled
3785         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3786             $tdf = 1 if !defined($conf->{tdf});
3787         }
3788     }
3789
3790     if ($winversion >= 6) {
3791         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3792         push @$machineFlags, 'hpet=off';
3793     }
3794
3795     push @$rtcFlags, 'driftfix=slew' if $tdf;
3796
3797     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3798         push @$rtcFlags, "base=$conf->{startdate}";
3799     } elsif ($useLocaltime) {
3800         push @$rtcFlags, 'base=localtime';
3801     }
3802
3803     if ($forcecpu) {
3804         push @$cmd, '-cpu', $forcecpu;
3805     } else {
3806         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3807     }
3808
3809     PVE::QemuServer::Memory::config(
3810         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3811
3812     push @$cmd, '-S' if $conf->{freeze};
3813
3814     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3815
3816     my $guest_agent = parse_guest_agent($conf);
3817
3818     if ($guest_agent->{enabled}) {
3819         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3820         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3821
3822         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3823             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3824             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3825             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3826         } elsif ($guest_agent->{type} eq 'isa') {
3827             push @$devices, '-device', "isa-serial,chardev=qga0";
3828         }
3829     }
3830
3831     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3832     if ($rng && $version_guard->(4, 1, 2)) {
3833         check_rng_source($rng->{source});
3834
3835         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3836         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3837         my $limiter_str = "";
3838         if ($max_bytes) {
3839             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3840         }
3841
3842         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3843         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3844         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3845     }
3846
3847     my $spice_port;
3848
3849     assert_clipboard_config($vga);
3850     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3851
3852     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3853         if ($qxlnum > 1) {
3854             if ($winversion){
3855                 for (my $i = 1; $i < $qxlnum; $i++){
3856                     push @$devices, '-device', print_vga_device(
3857                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3858                 }
3859             } else {
3860                 # assume other OS works like Linux
3861                 my ($ram, $vram) = ("134217728", "67108864");
3862                 if ($vga->{memory}) {
3863                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3864                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3865                 }
3866                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3867                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3868             }
3869         }
3870
3871         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3872
3873         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3874         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3875             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3876         } else {
3877             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3878         }
3879         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880
3881         if ($is_spice) {
3882             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3883             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3884             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3885
3886             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3887             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3888
3889             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3890             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3891             if ($spice_enhancement->{foldersharing}) {
3892                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3894             }
3895
3896             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898                 if $spice_enhancement->{videostreaming};
3899             push @$devices, '-spice', "$spice_opts";
3900         }
3901     }
3902
3903     # enable balloon by default, unless explicitly disabled
3904     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3905         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3906         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3908         push @$devices, '-device', $ballooncmd;
3909     }
3910
3911     if ($conf->{watchdog}) {
3912         my $wdopts = parse_watchdog($conf->{watchdog});
3913         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3914         my $watchdog = $wdopts->{model} || 'i6300esb';
3915         push @$devices, '-device', "$watchdog$pciaddr";
3916         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3917     }
3918
3919     my $vollist = [];
3920     my $scsicontroller = {};
3921     my $ahcicontroller = {};
3922     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3923
3924     # Add iscsi initiator name if available
3925     if (my $initiator = get_initiator_name()) {
3926         push @$devices, '-iscsi', "initiator-name=$initiator";
3927     }
3928
3929     PVE::QemuConfig->foreach_volume($conf, sub {
3930         my ($ds, $drive) = @_;
3931
3932         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3933             check_volume_storage_type($storecfg, $drive->{file});
3934             push @$vollist, $drive->{file};
3935         }
3936
3937         # ignore efidisk here, already added in bios/fw handling code above
3938         return if $drive->{interface} eq 'efidisk';
3939         # similar for TPM
3940         return if $drive->{interface} eq 'tpmstate';
3941
3942         $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3945
3946         if ($drive->{interface} eq 'virtio'){
3947            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3948         }
3949
3950         if ($drive->{interface} eq 'scsi') {
3951
3952             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3953
3954             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3956
3957             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3959
3960             my $iothread = '';
3961             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3962                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964             } elsif ($drive->{iothread}) {
3965                 log_warn(
3966                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3967                 );
3968             }
3969
3970             my $queues = '';
3971             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3972                 $queues = ",num_queues=$drive->{queues}";
3973             }
3974
3975             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3976                 if !$scsicontroller->{$controller};
3977             $scsicontroller->{$controller}=1;
3978         }
3979
3980         if ($drive->{interface} eq 'sata') {
3981             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3982             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3983             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3984                 if !$ahcicontroller->{$controller};
3985             $ahcicontroller->{$controller}=1;
3986         }
3987
3988         my $pbs_conf = $pbs_backing->{$ds};
3989         my $pbs_name = undef;
3990         if ($pbs_conf) {
3991             $pbs_name = "drive-$ds-pbs";
3992             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3993         }
3994
3995         my $drive_cmd = print_drive_commandline_full(
3996             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3997
3998         # extra protection for templates, but SATA and IDE don't support it..
3999         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4000
4001         push @$devices, '-drive',$drive_cmd;
4002         push @$devices, '-device', print_drivedevice_full(
4003             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4004     });
4005
4006     for (my $i = 0; $i < $MAX_NETS; $i++) {
4007         my $netname = "net$i";
4008
4009         next if !$conf->{$netname};
4010         my $d = parse_net($conf->{$netname});
4011         next if !$d;
4012         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4013
4014         $use_virtio = 1 if $d->{model} eq 'virtio';
4015
4016         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4017
4018         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4019         push @$devices, '-netdev', $netdevfull;
4020
4021         my $netdevicefull = print_netdevice_full(
4022             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4023
4024         push @$devices, '-device', $netdevicefull;
4025     }
4026
4027     if ($conf->{ivshmem}) {
4028         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4029
4030         my $bus;
4031         if ($q35) {
4032             $bus = print_pcie_addr("ivshmem");
4033         } else {
4034             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4035         }
4036
4037         my $ivshmem_name = $ivshmem->{name} // $vmid;
4038         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4039
4040         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4042             .",size=$ivshmem->{size}M";
4043     }
4044
4045     # pci.4 is nested in pci.1
4046     $bridges->{1} = 1 if $bridges->{4};
4047
4048     if (!$q35) { # add pci bridges
4049         if (min_version($machine_version, 2, 3)) {
4050            $bridges->{1} = 1;
4051            $bridges->{2} = 1;
4052         }
4053         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4054     }
4055
4056     for my $k (sort {$b cmp $a} keys %$bridges) {
4057         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4058
4059         my $k_name = $k;
4060         if ($k == 2 && $legacy_igd) {
4061             $k_name = "$k-igd";
4062         }
4063         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4064         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4065
4066         if ($q35) { # add after -readconfig pve-q35.cfg
4067             splice @$devices, 2, 0, '-device', $devstr;
4068         } else {
4069             unshift @$devices, '-device', $devstr if $k > 0;
4070         }
4071     }
4072
4073     if (!$kvm) {
4074         push @$machineFlags, 'accel=tcg';
4075     }
4076
4077     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4078
4079     my $machine_type_min = $machine_type;
4080     if ($add_pve_version) {
4081         $machine_type_min =~ s/\+pve\d+$//;
4082         $machine_type_min .= "+pve$required_pve_version";
4083     }
4084     push @$machineFlags, "type=${machine_type_min}";
4085
4086     push @$cmd, @$devices;
4087     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4088     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4089     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4090
4091     if (my $vmstate = $conf->{vmstate}) {
4092         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4093         push @$vollist, $vmstate;
4094         push @$cmd, '-loadstate', $statepath;
4095         print "activating and using '$vmstate' as vmstate\n";
4096     }
4097
4098     if (PVE::QemuConfig->is_template($conf)) {
4099         # needed to workaround base volumes being read-only
4100         push @$cmd, '-snapshot';
4101     }
4102
4103     # add custom args
4104     if ($conf->{args}) {
4105         my $aa = PVE::Tools::split_args($conf->{args});
4106         push @$cmd, @$aa;
4107     }
4108
4109     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4110 }
4111
4112 sub check_rng_source {
4113     my ($source) = @_;
4114
4115     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4116     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4117         if ! -e $source;
4118
4119     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4120     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4121         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4122         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4123         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4124             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4125             ." to the host.\n";
4126     }
4127 }
4128
4129 sub spice_port {
4130     my ($vmid) = @_;
4131
4132     my $res = mon_cmd($vmid, 'query-spice');
4133
4134     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4135 }
4136
4137 sub vm_devices_list {
4138     my ($vmid) = @_;
4139
4140     my $res = mon_cmd($vmid, 'query-pci');
4141     my $devices_to_check = [];
4142     my $devices = {};
4143     foreach my $pcibus (@$res) {
4144         push @$devices_to_check, @{$pcibus->{devices}},
4145     }
4146
4147     while (@$devices_to_check) {
4148         my $to_check = [];
4149         for my $d (@$devices_to_check) {
4150             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4151             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4152
4153             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4154             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4155         }
4156         $devices_to_check = $to_check;
4157     }
4158
4159     my $resblock = mon_cmd($vmid, 'query-block');
4160     foreach my $block (@$resblock) {
4161         if($block->{device} =~ m/^drive-(\S+)/){
4162                 $devices->{$1} = 1;
4163         }
4164     }
4165
4166     my $resmice = mon_cmd($vmid, 'query-mice');
4167     foreach my $mice (@$resmice) {
4168         if ($mice->{name} eq 'QEMU HID Tablet') {
4169             $devices->{tablet} = 1;
4170             last;
4171         }
4172     }
4173
4174     # for usb devices there is no query-usb
4175     # but we can iterate over the entries in
4176     # qom-list path=/machine/peripheral
4177     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4178     foreach my $per (@$resperipheral) {
4179         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4180             $devices->{$per->{name}} = 1;
4181         }
4182     }
4183
4184     return $devices;
4185 }
4186
4187 sub vm_deviceplug {
4188     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4189
4190     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4191
4192     my $devices_list = vm_devices_list($vmid);
4193     return 1 if defined($devices_list->{$deviceid});
4194
4195     # add PCI bridge if we need it for the device
4196     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4197
4198     if ($deviceid eq 'tablet') {
4199         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4200     } elsif ($deviceid eq 'keyboard') {
4201         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4202     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4203         my $id = $1;
4204         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4205         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4206     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4207         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4208     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4209         qemu_iothread_add($vmid, $deviceid, $device);
4210
4211         qemu_driveadd($storecfg, $vmid, $device);
4212         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4213
4214         qemu_deviceadd($vmid, $devicefull);
4215         eval { qemu_deviceaddverify($vmid, $deviceid); };
4216         if (my $err = $@) {
4217             eval { qemu_drivedel($vmid, $deviceid); };
4218             warn $@ if $@;
4219             die $err;
4220         }
4221     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4222         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4223         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4224         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4225
4226         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4227
4228         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4229             qemu_iothread_add($vmid, $deviceid, $device);
4230             $devicefull .= ",iothread=iothread-$deviceid";
4231         }
4232
4233         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4234             $devicefull .= ",num_queues=$device->{queues}";
4235         }
4236
4237         qemu_deviceadd($vmid, $devicefull);
4238         qemu_deviceaddverify($vmid, $deviceid);
4239     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4240         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4241         qemu_driveadd($storecfg, $vmid, $device);
4242
4243         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4244         eval { qemu_deviceadd($vmid, $devicefull); };
4245         if (my $err = $@) {
4246             eval { qemu_drivedel($vmid, $deviceid); };
4247             warn $@ if $@;
4248             die $err;
4249         }
4250     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4251         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4252
4253         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4254         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4255         my $use_old_bios_files = undef;
4256         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4257
4258         my $netdevicefull = print_netdevice_full(
4259             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4260         qemu_deviceadd($vmid, $netdevicefull);
4261         eval {
4262             qemu_deviceaddverify($vmid, $deviceid);
4263             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4264         };
4265         if (my $err = $@) {
4266             eval { qemu_netdevdel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4271         my $bridgeid = $2;
4272         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4273         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4274
4275         qemu_deviceadd($vmid, $devicefull);
4276         qemu_deviceaddverify($vmid, $deviceid);
4277     } else {
4278         die "can't hotplug device '$deviceid'\n";
4279     }
4280
4281     return 1;
4282 }
4283
4284 # fixme: this should raise exceptions on error!
4285 sub vm_deviceunplug {
4286     my ($vmid, $conf, $deviceid) = @_;
4287
4288     my $devices_list = vm_devices_list($vmid);
4289     return 1 if !defined($devices_list->{$deviceid});
4290
4291     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4292     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4293
4294     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4295         qemu_devicedel($vmid, $deviceid);
4296     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4297         qemu_devicedel($vmid, $deviceid);
4298         qemu_devicedelverify($vmid, $deviceid);
4299     } elsif ($deviceid =~ m/^usb\d+$/) {
4300         qemu_devicedel($vmid, $deviceid);
4301         qemu_devicedelverify($vmid, $deviceid);
4302     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4303         my $device = parse_drive($deviceid, $conf->{$deviceid});
4304
4305         qemu_devicedel($vmid, $deviceid);
4306         qemu_devicedelverify($vmid, $deviceid);
4307         qemu_drivedel($vmid, $deviceid);
4308         qemu_iothread_del($vmid, $deviceid, $device);
4309     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4310         qemu_devicedel($vmid, $deviceid);
4311         qemu_devicedelverify($vmid, $deviceid);
4312     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4313         my $device = parse_drive($deviceid, $conf->{$deviceid});
4314
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_drivedel($vmid, $deviceid);
4318         qemu_deletescsihw($conf, $vmid, $deviceid);
4319
4320         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4321             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4322     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4323         qemu_devicedel($vmid, $deviceid);
4324         qemu_devicedelverify($vmid, $deviceid);
4325         qemu_netdevdel($vmid, $deviceid);
4326     } else {
4327         die "can't unplug device '$deviceid'\n";
4328     }
4329
4330     return 1;
4331 }
4332
4333 sub qemu_spice_usbredir_chardev_add {
4334     my ($vmid, $id) = @_;
4335
4336     mon_cmd($vmid, "chardev-add" , (
4337         id => $id,
4338         backend => {
4339             type => 'spicevmc',
4340             data => {
4341                 type => "usbredir",
4342             },
4343         },
4344     ));
4345 }
4346
4347 sub qemu_iothread_add {
4348     my ($vmid, $deviceid, $device) = @_;
4349
4350     if ($device->{iothread}) {
4351         my $iothreads = vm_iothreads_list($vmid);
4352         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4353     }
4354 }
4355
4356 sub qemu_iothread_del {
4357     my ($vmid, $deviceid, $device) = @_;
4358
4359     if ($device->{iothread}) {
4360         my $iothreads = vm_iothreads_list($vmid);
4361         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4362     }
4363 }
4364
4365 sub qemu_driveadd {
4366     my ($storecfg, $vmid, $device) = @_;
4367
4368     my $kvmver = get_running_qemu_version($vmid);
4369     my $io_uring = min_version($kvmver, 6, 0);
4370     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4371     $drive =~ s/\\/\\\\/g;
4372     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4373
4374     # If the command succeeds qemu prints: "OK"
4375     return 1 if $ret =~ m/OK/s;
4376
4377     die "adding drive failed: $ret\n";
4378 }
4379
4380 sub qemu_drivedel {
4381     my ($vmid, $deviceid) = @_;
4382
4383     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4384     $ret =~ s/^\s+//;
4385
4386     return 1 if $ret eq "";
4387
4388     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4389     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4390
4391     die "deleting drive $deviceid failed : $ret\n";
4392 }
4393
4394 sub qemu_deviceaddverify {
4395     my ($vmid, $deviceid) = @_;
4396
4397     for (my $i = 0; $i <= 5; $i++) {
4398          my $devices_list = vm_devices_list($vmid);
4399          return 1 if defined($devices_list->{$deviceid});
4400          sleep 1;
4401     }
4402
4403     die "error on hotplug device '$deviceid'\n";
4404 }
4405
4406
4407 sub qemu_devicedelverify {
4408     my ($vmid, $deviceid) = @_;
4409
4410     # need to verify that the device is correctly removed as device_del
4411     # is async and empty return is not reliable
4412
4413     for (my $i = 0; $i <= 5; $i++) {
4414          my $devices_list = vm_devices_list($vmid);
4415          return 1 if !defined($devices_list->{$deviceid});
4416          sleep 1;
4417     }
4418
4419     die "error on hot-unplugging device '$deviceid'\n";
4420 }
4421
4422 sub qemu_findorcreatescsihw {
4423     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4424
4425     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4426
4427     my $scsihwid="$controller_prefix$controller";
4428     my $devices_list = vm_devices_list($vmid);
4429
4430     if (!defined($devices_list->{$scsihwid})) {
4431         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4432     }
4433
4434     return 1;
4435 }
4436
4437 sub qemu_deletescsihw {
4438     my ($conf, $vmid, $opt) = @_;
4439
4440     my $device = parse_drive($opt, $conf->{$opt});
4441
4442     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4443         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4444         return 1;
4445     }
4446
4447     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4448
4449     my $devices_list = vm_devices_list($vmid);
4450     foreach my $opt (keys %{$devices_list}) {
4451         if (is_valid_drivename($opt)) {
4452             my $drive = parse_drive($opt, $conf->{$opt});
4453             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4454                 return 1;
4455             }
4456         }
4457     }
4458
4459     my $scsihwid="scsihw$controller";
4460
4461     vm_deviceunplug($vmid, $conf, $scsihwid);
4462
4463     return 1;
4464 }
4465
4466 sub qemu_add_pci_bridge {
4467     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4468
4469     my $bridges = {};
4470
4471     my $bridgeid;
4472
4473     print_pci_addr($device, $bridges, $arch, $machine_type);
4474
4475     while (my ($k, $v) = each %$bridges) {
4476         $bridgeid = $k;
4477     }
4478     return 1 if !defined($bridgeid) || $bridgeid < 1;
4479
4480     my $bridge = "pci.$bridgeid";
4481     my $devices_list = vm_devices_list($vmid);
4482
4483     if (!defined($devices_list->{$bridge})) {
4484         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4485     }
4486
4487     return 1;
4488 }
4489
4490 sub qemu_set_link_status {
4491     my ($vmid, $device, $up) = @_;
4492
4493     mon_cmd($vmid, "set_link", name => $device,
4494                up => $up ? JSON::true : JSON::false);
4495 }
4496
4497 sub qemu_netdevadd {
4498     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4499
4500     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4501     my %options =  split(/[=,]/, $netdev);
4502
4503     if (defined(my $vhost = $options{vhost})) {
4504         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4505     }
4506
4507     if (defined(my $queues = $options{queues})) {
4508         $options{queues} = $queues + 0;
4509     }
4510
4511     mon_cmd($vmid, "netdev_add",  %options);
4512     return 1;
4513 }
4514
4515 sub qemu_netdevdel {
4516     my ($vmid, $deviceid) = @_;
4517
4518     mon_cmd($vmid, "netdev_del", id => $deviceid);
4519 }
4520
4521 sub qemu_usb_hotplug {
4522     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4523
4524     return if !$device;
4525
4526     # remove the old one first
4527     vm_deviceunplug($vmid, $conf, $deviceid);
4528
4529     # check if xhci controller is necessary and available
4530     my $devicelist = vm_devices_list($vmid);
4531
4532     if (!$devicelist->{xhci}) {
4533         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4534         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4535     }
4536
4537     # add the new one
4538     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4539 }
4540
4541 sub qemu_cpu_hotplug {
4542     my ($vmid, $conf, $vcpus) = @_;
4543
4544     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4545
4546     my $sockets = 1;
4547     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4548     $sockets = $conf->{sockets} if  $conf->{sockets};
4549     my $cores = $conf->{cores} || 1;
4550     my $maxcpus = $sockets * $cores;
4551
4552     $vcpus = $maxcpus if !$vcpus;
4553
4554     die "you can't add more vcpus than maxcpus\n"
4555         if $vcpus > $maxcpus;
4556
4557     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4558
4559     if ($vcpus < $currentvcpus) {
4560
4561         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4562
4563             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4564                 qemu_devicedel($vmid, "cpu$i");
4565                 my $retry = 0;
4566                 my $currentrunningvcpus = undef;
4567                 while (1) {
4568                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4569                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4570                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4571                     $retry++;
4572                     sleep 1;
4573                 }
4574                 #update conf after each succesfull cpu unplug
4575                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4576                 PVE::QemuConfig->write_config($vmid, $conf);
4577             }
4578         } else {
4579             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4580         }
4581
4582         return;
4583     }
4584
4585     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4586     die "vcpus in running vm does not match its configuration\n"
4587         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4588
4589     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4590         my $arch = get_vm_arch($conf);
4591
4592         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593             my $cpustr = print_cpu_device($conf, $arch, $i);
4594             qemu_deviceadd($vmid, $cpustr);
4595
4596             my $retry = 0;
4597             my $currentrunningvcpus = undef;
4598             while (1) {
4599                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                 last if scalar(@{$currentrunningvcpus}) == $i;
4601                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4602                 sleep 1;
4603                 $retry++;
4604             }
4605             #update conf after each succesfull cpu hotplug
4606             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607             PVE::QemuConfig->write_config($vmid, $conf);
4608         }
4609     } else {
4610
4611         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612             mon_cmd($vmid, "cpu-add", id => int($i));
4613         }
4614     }
4615 }
4616
4617 sub qemu_block_set_io_throttle {
4618     my ($vmid, $deviceid,
4619         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4623
4624     return if !check_running($vmid) ;
4625
4626     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4627         bps => int($bps),
4628         bps_rd => int($bps_rd),
4629         bps_wr => int($bps_wr),
4630         iops => int($iops),
4631         iops_rd => int($iops_rd),
4632         iops_wr => int($iops_wr),
4633         bps_max => int($bps_max),
4634         bps_rd_max => int($bps_rd_max),
4635         bps_wr_max => int($bps_wr_max),
4636         iops_max => int($iops_max),
4637         iops_rd_max => int($iops_rd_max),
4638         iops_wr_max => int($iops_wr_max),
4639         bps_max_length => int($bps_max_length),
4640         bps_rd_max_length => int($bps_rd_max_length),
4641         bps_wr_max_length => int($bps_wr_max_length),
4642         iops_max_length => int($iops_max_length),
4643         iops_rd_max_length => int($iops_rd_max_length),
4644         iops_wr_max_length => int($iops_wr_max_length),
4645     );
4646
4647 }
4648
4649 sub qemu_block_resize {
4650     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4655
4656     return if !$running;
4657
4658     my $padding = (1024 - $size % 1024) % 1024;
4659     $size = $size + $padding;
4660
4661     mon_cmd(
4662         $vmid,
4663         "block_resize",
4664         device => $deviceid,
4665         size => int($size),
4666         timeout => 60,
4667     );
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686     my $attached_deviceid;
4687
4688     if ($running) {
4689         my $conf = PVE::QemuConfig->load_config($vmid);
4690         PVE::QemuConfig->foreach_volume($conf, sub {
4691             my ($ds, $drive) = @_;
4692             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4693         });
4694     }
4695
4696     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4697         mon_cmd(
4698             $vmid,
4699             'blockdev-snapshot-delete-internal-sync',
4700             device => $attached_deviceid,
4701             name => $snap,
4702         );
4703     } else {
4704         PVE::Storage::volume_snapshot_delete(
4705             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4706     }
4707 }
4708
4709 sub set_migration_caps {
4710     my ($vmid, $savevm) = @_;
4711
4712     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4713
4714     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4715     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4716
4717     my $cap_ref = [];
4718
4719     my $enabled_cap = {
4720         "auto-converge" => 1,
4721         "xbzrle" => 1,
4722         "x-rdma-pin-all" => 0,
4723         "zero-blocks" => 0,
4724         "compress" => 0,
4725         "dirty-bitmaps" => $dirty_bitmaps,
4726     };
4727
4728     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4729
4730     for my $supported_capability (@$supported_capabilities) {
4731         push @$cap_ref, {
4732             capability => $supported_capability->{capability},
4733             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4734         };
4735     }
4736
4737     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4738 }
4739
4740 sub foreach_volid {
4741     my ($conf, $func, @param) = @_;
4742
4743     my $volhash = {};
4744
4745     my $test_volid = sub {
4746         my ($key, $drive, $snapname, $pending) = @_;
4747
4748         my $volid = $drive->{file};
4749         return if !$volid;
4750
4751         $volhash->{$volid}->{cdrom} //= 1;
4752         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4753
4754         my $replicate = $drive->{replicate} // 1;
4755         $volhash->{$volid}->{replicate} //= 0;
4756         $volhash->{$volid}->{replicate} = 1 if $replicate;
4757
4758         $volhash->{$volid}->{shared} //= 0;
4759         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4760
4761         $volhash->{$volid}->{is_unused} //= 0;
4762         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4763
4764         $volhash->{$volid}->{is_attached} //= 0;
4765         $volhash->{$volid}->{is_attached} = 1
4766             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4767
4768         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4769             if defined($snapname);
4770
4771         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4772
4773         my $size = $drive->{size};
4774         $volhash->{$volid}->{size} //= $size if $size;
4775
4776         $volhash->{$volid}->{is_vmstate} //= 0;
4777         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4778
4779         $volhash->{$volid}->{is_tpmstate} //= 0;
4780         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4781
4782         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4783     };
4784
4785     my $include_opts = {
4786         extra_keys => ['vmstate'],
4787         include_unused => 1,
4788     };
4789
4790     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4791
4792     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4793         if defined($conf->{pending}) && $conf->{pending}->%*;
4794
4795     foreach my $snapname (keys %{$conf->{snapshots}}) {
4796         my $snap = $conf->{snapshots}->{$snapname};
4797         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4798     }
4799
4800     foreach my $volid (keys %$volhash) {
4801         &$func($volid, $volhash->{$volid}, @param);
4802     }
4803 }
4804
4805 my $fast_plug_option = {
4806     'description' => 1,
4807     'hookscript' => 1,
4808     'lock' => 1,
4809     'migrate_downtime' => 1,
4810     'migrate_speed' => 1,
4811     'name' => 1,
4812     'onboot' => 1,
4813     'protection' => 1,
4814     'shares' => 1,
4815     'startup' => 1,
4816     'tags' => 1,
4817     'vmstatestorage' => 1,
4818 };
4819
4820 for my $opt (keys %$confdesc_cloudinit) {
4821     $fast_plug_option->{$opt} = 1;
4822 };
4823
4824 # hotplug changes in [PENDING]
4825 # $selection hash can be used to only apply specified options, for
4826 # example: { cores => 1 } (only apply changed 'cores')
4827 # $errors ref is used to return error messages
4828 sub vmconfig_hotplug_pending {
4829     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4830
4831     my $defaults = load_defaults();
4832     my $arch = get_vm_arch($conf);
4833     my $machine_type = get_vm_machine($conf, undef, $arch);
4834
4835     # commit values which do not have any impact on running VM first
4836     # Note: those option cannot raise errors, we we do not care about
4837     # $selection and always apply them.
4838
4839     my $add_error = sub {
4840         my ($opt, $msg) = @_;
4841         $errors->{$opt} = "hotplug problem - $msg";
4842     };
4843
4844     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4845
4846     my $cloudinit_record_changed = sub {
4847         my ($conf, $opt, $old, $new) = @_;
4848         return if !$cloudinit_pending_properties->{$opt};
4849
4850         my $ci = ($conf->{cloudinit} //= {});
4851
4852         my $recorded = $ci->{$opt};
4853         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4854
4855         if (defined($new)) {
4856             if (defined($old)) {
4857                 # an existing value is being modified
4858                 if (defined($recorded)) {
4859                     # the value was already not in sync
4860                     if ($new eq $recorded) {
4861                         # a value is being reverted to the cloud-init state:
4862                         delete $ci->{$opt};
4863                         delete $added{$opt};
4864                     } else {
4865                         # the value was changed multiple times, do nothing
4866                     }
4867                 } elsif ($added{$opt}) {
4868                     # the value had been marked as added and is being changed, do nothing
4869                 } else {
4870                     # the value is new, record it:
4871                     $ci->{$opt} = $old;
4872                 }
4873             } else {
4874                 # a new value is being added
4875                 if (defined($recorded)) {
4876                     # it was already not in sync
4877                     if ($new eq $recorded) {
4878                         # a value is being reverted to the cloud-init state:
4879                         delete $ci->{$opt};
4880                         delete $added{$opt};
4881                     } else {
4882                         # the value had temporarily been removed, do nothing
4883                     }
4884                 } elsif ($added{$opt}) {
4885                     # the value had been marked as added already, do nothing
4886                 } else {
4887                     # the value is new, add it
4888                     $added{$opt} = 1;
4889                 }
4890             }
4891         } elsif (!defined($old)) {
4892             # a non-existent value is being removed? ignore...
4893         } else {
4894             # a value is being deleted
4895             if (defined($recorded)) {
4896                 # a value was already recorded, just keep it
4897             } elsif ($added{$opt}) {
4898                 # the value was marked as added, remove it
4899                 delete $added{$opt};
4900             } else {
4901                 # a previously unrecorded value is being removed, record the old value:
4902                 $ci->{$opt} = $old;
4903             }
4904         }
4905
4906         my $added = join(',', sort keys %added);
4907         $ci->{added} = $added if length($added);
4908     };
4909
4910     my $changes = 0;
4911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4912         if ($fast_plug_option->{$opt}) {
4913             my $new = delete $conf->{pending}->{$opt};
4914             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4915             $conf->{$opt} = $new;
4916             $changes = 1;
4917         }
4918     }
4919
4920     if ($changes) {
4921         PVE::QemuConfig->write_config($vmid, $conf);
4922     }
4923
4924     my $ostype = $conf->{ostype};
4925     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4926     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4927     my $usb_hotplug = $hotplug_features->{usb}
4928         && min_version($version, 7, 1)
4929         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4930
4931     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4932     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4933
4934     foreach my $opt (sort keys %$pending_delete_hash) {
4935         next if $selection && !$selection->{$opt};
4936         my $force = $pending_delete_hash->{$opt}->{force};
4937         eval {
4938             if ($opt eq 'hotplug') {
4939                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4940             } elsif ($opt eq 'tablet') {
4941                 die "skip\n" if !$hotplug_features->{usb};
4942                 if ($defaults->{tablet}) {
4943                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4944                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4945                         if $arch eq 'aarch64';
4946                 } else {
4947                     vm_deviceunplug($vmid, $conf, 'tablet');
4948                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4949                 }
4950             } elsif ($opt =~ m/^usb(\d+)$/) {
4951                 my $index = $1;
4952                 die "skip\n" if !$usb_hotplug;
4953                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif ($opt eq 'vcpus') {
4956                 die "skip\n" if !$hotplug_features->{cpu};
4957                 qemu_cpu_hotplug($vmid, $conf, undef);
4958             } elsif ($opt eq 'balloon') {
4959                 # enable balloon device is not hotpluggable
4960                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4961                 # here we reset the ballooning value to memory
4962                 my $balloon = get_current_memory($conf->{memory});
4963                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4964             } elsif ($fast_plug_option->{$opt}) {
4965                 # do nothing
4966             } elsif ($opt =~ m/^net(\d+)$/) {
4967                 die "skip\n" if !$hotplug_features->{network};
4968                 vm_deviceunplug($vmid, $conf, $opt);
4969                 if($have_sdn) {
4970                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4971                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4972                 }
4973             } elsif (is_valid_drivename($opt)) {
4974                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4975                 vm_deviceunplug($vmid, $conf, $opt);
4976                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4977             } elsif ($opt =~ m/^memory$/) {
4978                 die "skip\n" if !$hotplug_features->{memory};
4979                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4980             } elsif ($opt eq 'cpuunits') {
4981                 $cgroup->change_cpu_shares(undef);
4982             } elsif ($opt eq 'cpulimit') {
4983                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4984             } else {
4985                 die "skip\n";
4986             }
4987         };
4988         if (my $err = $@) {
4989             &$add_error($opt, $err) if $err ne "skip\n";
4990         } else {
4991             my $old = delete $conf->{$opt};
4992             $cloudinit_record_changed->($conf, $opt, $old, undef);
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994         }
4995     }
4996
4997     my $cloudinit_opt;
4998     foreach my $opt (keys %{$conf->{pending}}) {
4999         next if $selection && !$selection->{$opt};
5000         my $value = $conf->{pending}->{$opt};
5001         eval {
5002             if ($opt eq 'hotplug') {
5003                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5004                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5005             } elsif ($opt eq 'tablet') {
5006                 die "skip\n" if !$hotplug_features->{usb};
5007                 if ($value == 1) {
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5010                         if $arch eq 'aarch64';
5011                 } elsif ($value == 0) {
5012                     vm_deviceunplug($vmid, $conf, 'tablet');
5013                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5014                 }
5015             } elsif ($opt =~ m/^usb(\d+)$/) {
5016                 my $index = $1;
5017                 die "skip\n" if !$usb_hotplug;
5018                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5019                 my $id = $opt;
5020                 if ($d->{host} =~ m/^spice$/i)  {
5021                     $id = "usbredirdev$index";
5022                 }
5023                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, $value);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable/disable balloning device is not hotpluggable
5029                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5030                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5031                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5032
5033                 # allow manual ballooning if shares is set to zero
5034                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5035                     my $memory = get_current_memory($conf->{memory});
5036                     my $balloon = $conf->{pending}->{balloon} || $memory;
5037                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5045                 # some changes can be done without hotplug
5046                 my $drive = parse_drive($opt, $value);
5047                 if (drive_is_cloudinit($drive)) {
5048                     $cloudinit_opt = [$opt, $drive];
5049                     # apply all the other changes first, then generate the cloudinit disk
5050                     die "skip\n";
5051                 }
5052                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5053                                      $vmid, $opt, $value, $arch, $machine_type);
5054             } elsif ($opt =~ m/^memory$/) { #dimms
5055                 die "skip\n" if !$hotplug_features->{memory};
5056                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5057             } elsif ($opt eq 'cpuunits') {
5058                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5059                 $cgroup->change_cpu_shares($new_cpuunits);
5060             } elsif ($opt eq 'cpulimit') {
5061                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5062                 $cgroup->change_cpu_quota($cpulimit, 100000);
5063             } elsif ($opt eq 'agent') {
5064                 vmconfig_update_agent($conf, $opt, $value);
5065             } else {
5066                 die "skip\n";  # skip non-hot-pluggable options
5067             }
5068         };
5069         if (my $err = $@) {
5070             &$add_error($opt, $err) if $err ne "skip\n";
5071         } else {
5072             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5073             $conf->{$opt} = $value;
5074             delete $conf->{pending}->{$opt};
5075         }
5076     }
5077
5078     if (defined($cloudinit_opt)) {
5079         my ($opt, $drive) = @$cloudinit_opt;
5080         my $value = $conf->{pending}->{$opt};
5081         eval {
5082             my $temp = {%$conf, $opt => $value};
5083             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5084             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5085                                  $vmid, $opt, $value, $arch, $machine_type);
5086         };
5087         if (my $err = $@) {
5088             &$add_error($opt, $err) if $err ne "skip\n";
5089         } else {
5090             $conf->{$opt} = $value;
5091             delete $conf->{pending}->{$opt};
5092         }
5093     }
5094
5095     # unplug xhci controller if no usb device is left
5096     if ($usb_hotplug) {
5097         my $has_usb = 0;
5098         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5099             next if !defined($conf->{"usb$i"});
5100             $has_usb = 1;
5101             last;
5102         }
5103         if (!$has_usb) {
5104             vm_deviceunplug($vmid, $conf, 'xhci');
5105         }
5106     }
5107
5108     PVE::QemuConfig->write_config($vmid, $conf);
5109
5110     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5111         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5112     }
5113 }
5114
5115 sub try_deallocate_drive {
5116     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5117
5118     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5119         my $volid = $drive->{file};
5120         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5121             my $sid = PVE::Storage::parse_volume_id($volid);
5122             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5123
5124             # check if the disk is really unused
5125             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5126                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5127             PVE::Storage::vdisk_free($storecfg, $volid);
5128             return 1;
5129         } else {
5130             # If vm is not owner of this disk remove from config
5131             return 1;
5132         }
5133     }
5134
5135     return;
5136 }
5137
5138 sub vmconfig_delete_or_detach_drive {
5139     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5140
5141     my $drive = parse_drive($opt, $conf->{$opt});
5142
5143     my $rpcenv = PVE::RPCEnvironment::get();
5144     my $authuser = $rpcenv->get_user();
5145
5146     if ($force) {
5147         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5148         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5149     } else {
5150         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5151     }
5152 }
5153
5154
5155
5156 sub vmconfig_apply_pending {
5157     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5158
5159     return if !scalar(keys %{$conf->{pending}});
5160
5161     my $add_apply_error = sub {
5162         my ($opt, $msg) = @_;
5163         my $err_msg = "unable to apply pending change $opt : $msg";
5164         $errors->{$opt} = $err_msg;
5165         warn $err_msg;
5166     };
5167
5168     # cold plug
5169
5170     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5171     foreach my $opt (sort keys %$pending_delete_hash) {
5172         my $force = $pending_delete_hash->{$opt}->{force};
5173         eval {
5174             if ($opt =~ m/^unused/) {
5175                 die "internal error";
5176             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5177                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5178             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5179                 if($have_sdn) {
5180                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5181                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5182                     warn if $@;
5183                 }
5184             }
5185         };
5186         if (my $err = $@) {
5187             $add_apply_error->($opt, $err);
5188         } else {
5189             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5190             delete $conf->{$opt};
5191         }
5192     }
5193
5194     PVE::QemuConfig->cleanup_pending($conf);
5195
5196     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5197
5198     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5199         next if $opt eq 'delete'; # just to be sure
5200         eval {
5201             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5202                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5203             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5204                 return if !$have_sdn; # return from eval if SDN is not available
5205
5206                 my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5207                 if ($conf->{$opt}) {
5208                     my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5209
5210                     if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (
5211                         safe_string_ne($old_net->{bridge}, $new_net->{bridge}) ||
5212                         safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})
5213                     )) {
5214                         PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5215                     }
5216                 }
5217                 #fixme: reuse ip if mac change && same bridge
5218                 PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5219             }
5220         };
5221         if (my $err = $@) {
5222             $add_apply_error->($opt, $err);
5223         } else {
5224
5225             if (is_valid_drivename($opt)) {
5226                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5227                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5228             }
5229
5230             $conf->{$opt} = delete $conf->{pending}->{$opt};
5231         }
5232     }
5233
5234     # write all changes at once to avoid unnecessary i/o
5235     PVE::QemuConfig->write_config($vmid, $conf);
5236     if ($generate_cloudinit) {
5237         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5238             # After successful generation and if there were changes to be applied, update the
5239             # config to drop the {cloudinit} entry.
5240             PVE::QemuConfig->write_config($vmid, $conf);
5241         }
5242     }
5243 }
5244
5245 sub vmconfig_update_net {
5246     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5247
5248     my $newnet = parse_net($value);
5249
5250     if ($conf->{$opt}) {
5251         my $oldnet = parse_net($conf->{$opt});
5252
5253         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5254             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5255             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5256             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5257             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5258
5259             # for non online change, we try to hot-unplug
5260             die "skip\n" if !$hotplug;
5261             vm_deviceunplug($vmid, $conf, $opt);
5262
5263             if($have_sdn) {
5264                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5265             }
5266
5267         } else {
5268
5269             die "internal error" if $opt !~ m/net(\d+)/;
5270             my $iface = "tap${vmid}i$1";
5271
5272             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5273                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5274                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5275                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5276                 PVE::Network::tap_unplug($iface);
5277
5278                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5279                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5280                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5281                     qemu_set_link_status($vmid, $opt, 0);
5282                 }
5283
5284                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5285                     if ($have_sdn) {
5286                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5287                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5288                     }
5289                 }
5290
5291                 if ($have_sdn) {
5292                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5293                 } else {
5294                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5295                 }
5296
5297                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5298                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5299                     safe_num_ne($oldnet->{tag}, $newnet->{tag})) {
5300                     qemu_set_link_status($vmid, $opt, 1);
5301                 }
5302
5303             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5304                 # Rate can be applied on its own but any change above needs to
5305                 # include the rate in tap_plug since OVS resets everything.
5306                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5307             }
5308
5309             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5310                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5311             }
5312
5313             return 1;
5314         }
5315     }
5316
5317     if ($hotplug) {
5318         if ($have_sdn) {
5319             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5320             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5321         }
5322         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5323     } else {
5324         die "skip\n";
5325     }
5326 }
5327
5328 sub vmconfig_update_agent {
5329     my ($conf, $opt, $value) = @_;
5330
5331     die "skip\n" if !$conf->{$opt};
5332
5333     my $hotplug_options = { fstrim_cloned_disks => 1 };
5334
5335     my $old_agent = parse_guest_agent($conf);
5336     my $agent = parse_guest_agent({$opt => $value});
5337
5338     for my $option (keys %$agent) { # added/changed options
5339         next if defined($hotplug_options->{$option});
5340         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5341     }
5342
5343     for my $option (keys %$old_agent) { # removed options
5344         next if defined($hotplug_options->{$option});
5345         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5346     }
5347
5348     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5349 }
5350
5351 sub vmconfig_update_disk {
5352     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5353
5354     my $drive = parse_drive($opt, $value);
5355
5356     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5357         my $media = $drive->{media} || 'disk';
5358         my $oldmedia = $old_drive->{media} || 'disk';
5359         die "unable to change media type\n" if $media ne $oldmedia;
5360
5361         if (!drive_is_cdrom($old_drive)) {
5362
5363             if ($drive->{file} ne $old_drive->{file}) {
5364
5365                 die "skip\n" if !$hotplug;
5366
5367                 # unplug and register as unused
5368                 vm_deviceunplug($vmid, $conf, $opt);
5369                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5370
5371             } else {
5372                 # update existing disk
5373
5374                 # skip non hotpluggable value
5375                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5376                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5377                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5378                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5379                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5380                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5381                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5382                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5383                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5384                     die "skip\n";
5385                 }
5386
5387                 # apply throttle
5388                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5389                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5390                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5391                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5392                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5393                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5394                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5395                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5396                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5397                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5398                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5399                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5400                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5401                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5402                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5403                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5404                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5405                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5406
5407                     qemu_block_set_io_throttle(
5408                         $vmid,"drive-$opt",
5409                         ($drive->{mbps} || 0)*1024*1024,
5410                         ($drive->{mbps_rd} || 0)*1024*1024,
5411                         ($drive->{mbps_wr} || 0)*1024*1024,
5412                          $drive->{iops} || 0,
5413                          $drive->{iops_rd} || 0,
5414                          $drive->{iops_wr} || 0,
5415                         ($drive->{mbps_max} || 0)*1024*1024,
5416                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5417                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5418                          $drive->{iops_max} || 0,
5419                          $drive->{iops_rd_max} || 0,
5420                          $drive->{iops_wr_max} || 0,
5421                          $drive->{bps_max_length} || 1,
5422                          $drive->{bps_rd_max_length} || 1,
5423                          $drive->{bps_wr_max_length} || 1,
5424                          $drive->{iops_max_length} || 1,
5425                          $drive->{iops_rd_max_length} || 1,
5426                          $drive->{iops_wr_max_length} || 1,
5427                     );
5428
5429                 }
5430
5431                 return 1;
5432             }
5433
5434         } else { # cdrom
5435
5436             if ($drive->{file} eq 'none') {
5437                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5438                 if (drive_is_cloudinit($old_drive)) {
5439                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5440                 }
5441             } else {
5442                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5443
5444                 # force eject if locked
5445                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5446
5447                 if ($path) {
5448                     mon_cmd($vmid, "blockdev-change-medium",
5449                         id => "$opt", filename => "$path");
5450                 }
5451             }
5452
5453             return 1;
5454         }
5455     }
5456
5457     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5458     # hotplug new disks
5459     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5460     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5461 }
5462
5463 sub vmconfig_update_cloudinit_drive {
5464     my ($storecfg, $conf, $vmid) = @_;
5465
5466     my $cloudinit_ds = undef;
5467     my $cloudinit_drive = undef;
5468
5469     PVE::QemuConfig->foreach_volume($conf, sub {
5470         my ($ds, $drive) = @_;
5471         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5472             $cloudinit_ds = $ds;
5473             $cloudinit_drive = $drive;
5474         }
5475     });
5476
5477     return if !$cloudinit_drive;
5478
5479     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5480         PVE::QemuConfig->write_config($vmid, $conf);
5481     }
5482
5483     my $running = PVE::QemuServer::check_running($vmid);
5484
5485     if ($running) {
5486         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5487         if ($path) {
5488             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5489             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5490         }
5491     }
5492 }
5493
5494 # called in locked context by incoming migration
5495 sub vm_migrate_get_nbd_disks {
5496     my ($storecfg, $conf, $replicated_volumes) = @_;
5497
5498     my $local_volumes = {};
5499     PVE::QemuConfig->foreach_volume($conf, sub {
5500         my ($ds, $drive) = @_;
5501
5502         return if drive_is_cdrom($drive);
5503         return if $ds eq 'tpmstate0';
5504
5505         my $volid = $drive->{file};
5506
5507         return if !$volid;
5508
5509         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5510
5511         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5512         return if $scfg->{shared};
5513
5514         my $format = qemu_img_format($scfg, $volname);
5515
5516         # replicated disks re-use existing state via bitmap
5517         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5518         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5519     });
5520     return $local_volumes;
5521 }
5522
5523 # called in locked context by incoming migration
5524 sub vm_migrate_alloc_nbd_disks {
5525     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5526
5527     my $nbd = {};
5528     foreach my $opt (sort keys %$source_volumes) {
5529         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5530
5531         if ($use_existing) {
5532             $nbd->{$opt}->{drivestr} = print_drive($drive);
5533             $nbd->{$opt}->{volid} = $volid;
5534             $nbd->{$opt}->{replicated} = 1;
5535             next;
5536         }
5537
5538         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5539
5540         # order of precedence, filtered by whether storage supports it:
5541         # 1. explicit requested format
5542         # 2. default format of storage
5543         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5544         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5545
5546         my $size = $drive->{size} / 1024;
5547         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5548         my $newdrive = $drive;
5549         $newdrive->{format} = $format;
5550         $newdrive->{file} = $newvolid;
5551         my $drivestr = print_drive($newdrive);
5552         $nbd->{$opt}->{drivestr} = $drivestr;
5553         $nbd->{$opt}->{volid} = $newvolid;
5554     }
5555
5556     return $nbd;
5557 }
5558
5559 # see vm_start_nolock for parameters, additionally:
5560 # migrate_opts:
5561 #   storagemap = parsed storage map for allocating NBD disks
5562 sub vm_start {
5563     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5564
5565     return PVE::QemuConfig->lock_config($vmid, sub {
5566         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5567
5568         die "you can't start a vm if it's a template\n"
5569             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5570
5571         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5572         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5573
5574         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5575
5576         if ($has_backup_lock && $running) {
5577             # a backup is currently running, attempt to start the guest in the
5578             # existing QEMU instance
5579             return vm_resume($vmid);
5580         }
5581
5582         PVE::QemuConfig->check_lock($conf)
5583             if !($params->{skiplock} || $has_suspended_lock);
5584
5585         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5586
5587         die "VM $vmid already running\n" if $running;
5588
5589         if (my $storagemap = $migrate_opts->{storagemap}) {
5590             my $replicated = $migrate_opts->{replicated_volumes};
5591             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5592             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5593
5594             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5595                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5596             }
5597         }
5598
5599         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5600     });
5601 }
5602
5603
5604 # params:
5605 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5606 #   skiplock => 0/1, skip checking for config lock
5607 #   skiptemplate => 0/1, skip checking whether VM is template
5608 #   forcemachine => to force QEMU machine (rollback/migration)
5609 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5610 #   timeout => in seconds
5611 #   paused => start VM in paused state (backup)
5612 #   resume => resume from hibernation
5613 #   pbs-backing => {
5614 #      sata0 => {
5615 #         repository
5616 #         snapshot
5617 #         keyfile
5618 #         archive
5619 #      },
5620 #      virtio2 => ...
5621 #   }
5622 # migrate_opts:
5623 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5624 #   migratedfrom => source node
5625 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5626 #   network => CIDR of migration network
5627 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5628 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5629 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5630 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5631 #       contained in config
5632 sub vm_start_nolock {
5633     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5634
5635     my $statefile = $params->{statefile};
5636     my $resume = $params->{resume};
5637
5638     my $migratedfrom = $migrate_opts->{migratedfrom};
5639     my $migration_type = $migrate_opts->{type};
5640
5641     my $res = {};
5642
5643     # clean up leftover reboot request files
5644     eval { clear_reboot_request($vmid); };
5645     warn $@ if $@;
5646
5647     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5648         vmconfig_apply_pending($vmid, $conf, $storecfg);
5649         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5650     }
5651
5652     # don't regenerate the ISO if the VM is started as part of a live migration
5653     # this way we can reuse the old ISO with the correct config
5654     if (!$migratedfrom) {
5655         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5656             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5657             # $conf->{cloudinit}, so we could just not do this?
5658             # But we do it above, so for now let's be consistent.
5659             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5660         }
5661     }
5662
5663     # override offline migrated volumes, conf is out of date still
5664     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5665         for my $key (sort keys $offline_volumes->%*) {
5666             my $parsed = parse_drive($key, $conf->{$key});
5667             $parsed->{file} = $offline_volumes->{$key};
5668             $conf->{$key} = print_drive($parsed);
5669         }
5670     }
5671
5672     my $defaults = load_defaults();
5673
5674     # set environment variable useful inside network script
5675     # for remote migration the config is available on the target node!
5676     if (!$migrate_opts->{remote_node}) {
5677         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5678     }
5679
5680     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5681
5682     my $forcemachine = $params->{forcemachine};
5683     my $forcecpu = $params->{forcecpu};
5684     if ($resume) {
5685         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5686         $forcemachine = $conf->{runningmachine};
5687         $forcecpu = $conf->{runningcpu};
5688         print "Resuming suspended VM\n";
5689     }
5690
5691     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5692         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5693
5694     my $migration_ip;
5695     my $get_migration_ip = sub {
5696         my ($nodename) = @_;
5697
5698         return $migration_ip if defined($migration_ip);
5699
5700         my $cidr = $migrate_opts->{network};
5701
5702         if (!defined($cidr)) {
5703             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5704             $cidr = $dc_conf->{migration}->{network};
5705         }
5706
5707         if (defined($cidr)) {
5708             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5709
5710             die "could not get IP: no address configured on local " .
5711                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5712
5713             die "could not get IP: multiple addresses configured on local " .
5714                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5715
5716             $migration_ip = @$ips[0];
5717         }
5718
5719         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5720             if !defined($migration_ip);
5721
5722         return $migration_ip;
5723     };
5724
5725     if ($statefile) {
5726         if ($statefile eq 'tcp') {
5727             my $migrate = $res->{migrate} = { proto => 'tcp' };
5728             $migrate->{addr} = "localhost";
5729             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5730             my $nodename = nodename();
5731
5732             if (!defined($migration_type)) {
5733                 if (defined($datacenterconf->{migration}->{type})) {
5734                     $migration_type = $datacenterconf->{migration}->{type};
5735                 } else {
5736                     $migration_type = 'secure';
5737                 }
5738             }
5739
5740             if ($migration_type eq 'insecure') {
5741                 $migrate->{addr} = $get_migration_ip->($nodename);
5742                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5743             }
5744
5745             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5746             # via QMP later
5747             push @$cmd, '-incoming', 'defer';
5748             push @$cmd, '-S';
5749
5750         } elsif ($statefile eq 'unix') {
5751             # should be default for secure migrations as a ssh TCP forward
5752             # tunnel is not deterministic reliable ready and fails regurarly
5753             # to set up in time, so use UNIX socket forwards
5754             my $migrate = $res->{migrate} = { proto => 'unix' };
5755             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5756             unlink $migrate->{addr};
5757
5758             $migrate->{uri} = "unix:$migrate->{addr}";
5759             push @$cmd, '-incoming', $migrate->{uri};
5760             push @$cmd, '-S';
5761
5762         } elsif (-e $statefile) {
5763             push @$cmd, '-loadstate', $statefile;
5764         } else {
5765             my $statepath = PVE::Storage::path($storecfg, $statefile);
5766             push @$vollist, $statefile;
5767             push @$cmd, '-loadstate', $statepath;
5768         }
5769     } elsif ($params->{paused}) {
5770         push @$cmd, '-S';
5771     }
5772
5773     my $memory = get_current_memory($conf->{memory});
5774     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5775
5776     my $pci_reserve_list = [];
5777     for my $device (values $pci_devices->%*) {
5778         next if $device->{mdev}; # we don't reserve for mdev devices
5779         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5780     }
5781
5782     # reserve all PCI IDs before actually doing anything with them
5783     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5784
5785     eval {
5786         my $uuid;
5787         for my $id (sort keys %$pci_devices) {
5788             my $d = $pci_devices->{$id};
5789             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5790
5791             my $chosen_mdev;
5792             for my $dev ($d->{ids}->@*) {
5793                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5794                 if ($d->{mdev}) {
5795                     warn $@ if $@;
5796                     $chosen_mdev = $info;
5797                     last if $chosen_mdev; # if successful, we're done
5798                 } else {
5799                     die $@ if $@;
5800                 }
5801             }
5802
5803             next if !$d->{mdev};
5804             die "could not create mediated device\n" if !defined($chosen_mdev);
5805
5806             # nvidia grid needs the uuid of the mdev as qemu parameter
5807             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5808                 if (defined($conf->{smbios1})) {
5809                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5810                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5811                 }
5812                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5813             }
5814         }
5815         push @$cmd, '-uuid', $uuid if defined($uuid);
5816     };
5817     if (my $err = $@) {
5818         eval { cleanup_pci_devices($vmid, $conf) };
5819         warn $@ if $@;
5820         die $err;
5821     }
5822
5823     PVE::Storage::activate_volumes($storecfg, $vollist);
5824
5825
5826     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5827     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5828     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5829     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5830     # timeout should be more than enough here...
5831     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5832
5833     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5834
5835     my %run_params = (
5836         timeout => $statefile ? undef : $start_timeout,
5837         umask => 0077,
5838         noerr => 1,
5839     );
5840
5841     # when migrating, prefix QEMU output so other side can pick up any
5842     # errors that might occur and show the user
5843     if ($migratedfrom) {
5844         $run_params{quiet} = 1;
5845         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5846     }
5847
5848     my %systemd_properties = (
5849         Slice => 'qemu.slice',
5850         KillMode => 'process',
5851         SendSIGKILL => 0,
5852         TimeoutStopUSec => ULONG_MAX, # infinity
5853     );
5854
5855     if (PVE::CGroup::cgroup_mode() == 2) {
5856         $systemd_properties{CPUWeight} = $cpuunits;
5857     } else {
5858         $systemd_properties{CPUShares} = $cpuunits;
5859     }
5860
5861     if (my $cpulimit = $conf->{cpulimit}) {
5862         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5863     }
5864     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5865
5866     my $run_qemu = sub {
5867         PVE::Tools::run_fork sub {
5868             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5869
5870             my $tpmpid;
5871             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5872                 # start the TPM emulator so QEMU can connect on start
5873                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5874             }
5875
5876             my $exitcode = run_command($cmd, %run_params);
5877             if ($exitcode) {
5878                 if ($tpmpid) {
5879                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5880                     kill 'TERM', $tpmpid;
5881                 }
5882                 die "QEMU exited with code $exitcode\n";
5883             }
5884         };
5885     };
5886
5887     if ($conf->{hugepages}) {
5888
5889         my $code = sub {
5890             my $hotplug_features =
5891                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5892             my $hugepages_topology =
5893                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5894
5895             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5896
5897             PVE::QemuServer::Memory::hugepages_mount();
5898             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5899
5900             eval { $run_qemu->() };
5901             if (my $err = $@) {
5902                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5903                     if !$conf->{keephugepages};
5904                 die $err;
5905             }
5906
5907             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5908                 if !$conf->{keephugepages};
5909         };
5910         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5911
5912     } else {
5913         eval { $run_qemu->() };
5914     }
5915
5916     if (my $err = $@) {
5917         # deactivate volumes if start fails
5918         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5919         warn $@ if $@;
5920         eval { cleanup_pci_devices($vmid, $conf) };
5921         warn $@ if $@;
5922
5923         die "start failed: $err";
5924     }
5925
5926     # re-reserve all PCI IDs now that we can know the actual VM PID
5927     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5928     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5929     warn $@ if $@;
5930
5931     if (defined(my $migrate = $res->{migrate})) {
5932         if ($migrate->{proto} eq 'tcp') {
5933             my $nodename = nodename();
5934             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5935             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5936             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5937             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5938         }
5939         print "migration listens on $migrate->{uri}\n";
5940     } elsif ($statefile) {
5941         eval { mon_cmd($vmid, "cont"); };
5942         warn $@ if $@;
5943     }
5944
5945     #start nbd server for storage migration
5946     if (my $nbd = $migrate_opts->{nbd}) {
5947         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5948
5949         my $migrate_storage_uri;
5950         # nbd_protocol_version > 0 for unix socket support
5951         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5952             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5953             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5954             $migrate_storage_uri = "nbd:unix:$socket_path";
5955             $res->{migrate}->{unix_sockets} = [$socket_path];
5956         } else {
5957             my $nodename = nodename();
5958             my $localip = $get_migration_ip->($nodename);
5959             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5960             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5961
5962             mon_cmd($vmid, "nbd-server-start", addr => {
5963                 type => 'inet',
5964                 data => {
5965                     host => "${localip}",
5966                     port => "${storage_migrate_port}",
5967                 },
5968             });
5969             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5970             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5971         }
5972
5973         my $block_info = mon_cmd($vmid, "query-block");
5974         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5975
5976         foreach my $opt (sort keys %$nbd) {
5977             my $drivestr = $nbd->{$opt}->{drivestr};
5978             my $volid = $nbd->{$opt}->{volid};
5979
5980             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5981
5982             mon_cmd(
5983                 $vmid,
5984                 "block-export-add",
5985                 id => "drive-$opt",
5986                 'node-name' => $block_node,
5987                 writable => JSON::true,
5988                 type => "nbd",
5989                 name => "drive-$opt", # NBD export name
5990             );
5991
5992             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5993             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5994             print "re-using replicated volume: $opt - $volid\n"
5995                 if $nbd->{$opt}->{replicated};
5996
5997             $res->{drives}->{$opt} = $nbd->{$opt};
5998             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5999         }
6000     }
6001
6002     if ($migratedfrom) {
6003         eval {
6004             set_migration_caps($vmid);
6005         };
6006         warn $@ if $@;
6007
6008         if ($spice_port) {
6009             print "spice listens on port $spice_port\n";
6010             $res->{spice_port} = $spice_port;
6011             if ($migrate_opts->{spice_ticket}) {
6012                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6013                     $migrate_opts->{spice_ticket});
6014                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6015             }
6016         }
6017
6018     } else {
6019         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6020             if !$statefile && $conf->{balloon};
6021
6022         foreach my $opt (keys %$conf) {
6023             next if $opt !~  m/^net\d+$/;
6024             my $nicconf = parse_net($conf->{$opt});
6025             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6026         }
6027         add_nets_bridge_fdb($conf, $vmid);
6028     }
6029
6030    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6031         eval {
6032             mon_cmd(
6033                 $vmid,
6034                 'qom-set',
6035                 path => "machine/peripheral/balloon0",
6036                 property => "guest-stats-polling-interval",
6037                 value => 2
6038             );
6039         };
6040         log_warn("could not set polling interval for ballooning - $@") if $@;
6041     }
6042
6043     if ($resume) {
6044         print "Resumed VM, removing state\n";
6045         if (my $vmstate = $conf->{vmstate}) {
6046             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6047             PVE::Storage::vdisk_free($storecfg, $vmstate);
6048         }
6049         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6050         PVE::QemuConfig->write_config($vmid, $conf);
6051     }
6052
6053     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6054
6055     my ($current_machine, $is_deprecated) =
6056         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6057     if ($is_deprecated) {
6058         log_warn(
6059             "current machine version '$current_machine' is deprecated - see the documentation and ".
6060             "change to a newer one",
6061         );
6062     }
6063
6064     return $res;
6065 }
6066
6067 sub vm_commandline {
6068     my ($storecfg, $vmid, $snapname) = @_;
6069
6070     my $conf = PVE::QemuConfig->load_config($vmid);
6071
6072     my ($forcemachine, $forcecpu);
6073     if ($snapname) {
6074         my $snapshot = $conf->{snapshots}->{$snapname};
6075         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6076
6077         # check for machine or CPU overrides in snapshot
6078         $forcemachine = $snapshot->{runningmachine};
6079         $forcecpu = $snapshot->{runningcpu};
6080
6081         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6082
6083         $conf = $snapshot;
6084     }
6085
6086     my $defaults = load_defaults();
6087
6088     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6089
6090     return PVE::Tools::cmd2string($cmd);
6091 }
6092
6093 sub vm_reset {
6094     my ($vmid, $skiplock) = @_;
6095
6096     PVE::QemuConfig->lock_config($vmid, sub {
6097
6098         my $conf = PVE::QemuConfig->load_config($vmid);
6099
6100         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6101
6102         mon_cmd($vmid, "system_reset");
6103     });
6104 }
6105
6106 sub get_vm_volumes {
6107     my ($conf) = @_;
6108
6109     my $vollist = [];
6110     foreach_volid($conf, sub {
6111         my ($volid, $attr) = @_;
6112
6113         return if $volid =~ m|^/|;
6114
6115         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6116         return if !$sid;
6117
6118         push @$vollist, $volid;
6119     });
6120
6121     return $vollist;
6122 }
6123
6124 sub cleanup_pci_devices {
6125     my ($vmid, $conf) = @_;
6126
6127     foreach my $key (keys %$conf) {
6128         next if $key !~ m/^hostpci(\d+)$/;
6129         my $hostpciindex = $1;
6130         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6131         my $d = parse_hostpci($conf->{$key});
6132         if ($d->{mdev}) {
6133             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6134             # don't want to break ABI just for this two liner
6135             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6136
6137             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6138             # out when we do it first. so wait for up to 10 seconds and then try it manually
6139             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6140                 my $count = 0;
6141                 while (-e $dev_sysfs_dir && $count < 10) {
6142                     sleep 1;
6143                     $count++;
6144                 }
6145                 print "waited $count seconds for mediated device driver finishing clean up\n";
6146             }
6147
6148             if (-e $dev_sysfs_dir) {
6149                 print "actively clean up mediated device with UUID $uuid\n";
6150                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6151             }
6152         }
6153     }
6154     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6155 }
6156
6157 sub vm_stop_cleanup {
6158     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6159
6160     eval {
6161
6162         if (!$keepActive) {
6163             my $vollist = get_vm_volumes($conf);
6164             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6165
6166             if (my $tpmdrive = $conf->{tpmstate0}) {
6167                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6168                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6169                 if ($storeid) {
6170                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6171                 }
6172             }
6173         }
6174
6175         foreach my $ext (qw(mon qmp pid vnc qga)) {
6176             unlink "/var/run/qemu-server/${vmid}.$ext";
6177         }
6178
6179         if ($conf->{ivshmem}) {
6180             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6181             # just delete it for now, VMs which have this already open do not
6182             # are affected, but new VMs will get a separated one. If this
6183             # becomes an issue we either add some sort of ref-counting or just
6184             # add a "don't delete on stop" flag to the ivshmem format.
6185             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6186         }
6187
6188         cleanup_pci_devices($vmid, $conf);
6189
6190         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6191     };
6192     warn $@ if $@; # avoid errors - just warn
6193 }
6194
6195 # call only in locked context
6196 sub _do_vm_stop {
6197     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6198
6199     my $pid = check_running($vmid, $nocheck);
6200     return if !$pid;
6201
6202     my $conf;
6203     if (!$nocheck) {
6204         $conf = PVE::QemuConfig->load_config($vmid);
6205         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6206         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6207             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6208             $timeout = $opts->{down} if $opts->{down};
6209         }
6210         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6211     }
6212
6213     eval {
6214         if ($shutdown) {
6215             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6216                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6217             } else {
6218                 mon_cmd($vmid, "system_powerdown");
6219             }
6220         } else {
6221             mon_cmd($vmid, "quit");
6222         }
6223     };
6224     my $err = $@;
6225
6226     if (!$err) {
6227         $timeout = 60 if !defined($timeout);
6228
6229         my $count = 0;
6230         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6231             $count++;
6232             sleep 1;
6233         }
6234
6235         if ($count >= $timeout) {
6236             if ($force) {
6237                 warn "VM still running - terminating now with SIGTERM\n";
6238                 kill 15, $pid;
6239             } else {
6240                 die "VM quit/powerdown failed - got timeout\n";
6241             }
6242         } else {
6243             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6244             return;
6245         }
6246     } else {
6247        if (!check_running($vmid, $nocheck)) {
6248             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6249             return;
6250        }
6251        if ($force) {
6252             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6253             kill 15, $pid;
6254         } else {
6255             die "VM quit/powerdown failed\n";
6256         }
6257     }
6258
6259     # wait again
6260     $timeout = 10;
6261
6262     my $count = 0;
6263     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6264         $count++;
6265         sleep 1;
6266     }
6267
6268     if ($count >= $timeout) {
6269         warn "VM still running - terminating now with SIGKILL\n";
6270         kill 9, $pid;
6271         sleep 1;
6272     }
6273
6274     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6275 }
6276
6277 # Note: use $nocheck to skip tests if VM configuration file exists.
6278 # We need that when migration VMs to other nodes (files already moved)
6279 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6280 sub vm_stop {
6281     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6282
6283     $force = 1 if !defined($force) && !$shutdown;
6284
6285     if ($migratedfrom){
6286         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6287         kill 15, $pid if $pid;
6288         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6289         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6290         return;
6291     }
6292
6293     PVE::QemuConfig->lock_config($vmid, sub {
6294         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6295    });
6296 }
6297
6298 sub vm_reboot {
6299     my ($vmid, $timeout) = @_;
6300
6301     PVE::QemuConfig->lock_config($vmid, sub {
6302         eval {
6303
6304             # only reboot if running, as qmeventd starts it again on a stop event
6305             return if !check_running($vmid);
6306
6307             create_reboot_request($vmid);
6308
6309             my $storecfg = PVE::Storage::config();
6310             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6311
6312         };
6313         if (my $err = $@) {
6314             # avoid that the next normal shutdown will be confused for a reboot
6315             clear_reboot_request($vmid);
6316             die $err;
6317         }
6318    });
6319 }
6320
6321 # note: if using the statestorage parameter, the caller has to check privileges
6322 sub vm_suspend {
6323     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6324
6325     my $conf;
6326     my $path;
6327     my $storecfg;
6328     my $vmstate;
6329
6330     PVE::QemuConfig->lock_config($vmid, sub {
6331
6332         $conf = PVE::QemuConfig->load_config($vmid);
6333
6334         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6335         PVE::QemuConfig->check_lock($conf)
6336             if !($skiplock || $is_backing_up);
6337
6338         die "cannot suspend to disk during backup\n"
6339             if $is_backing_up && $includestate;
6340
6341         if ($includestate) {
6342             $conf->{lock} = 'suspending';
6343             my $date = strftime("%Y-%m-%d", localtime(time()));
6344             $storecfg = PVE::Storage::config();
6345             if (!$statestorage) {
6346                 $statestorage = find_vmstate_storage($conf, $storecfg);
6347                 # check permissions for the storage
6348                 my $rpcenv = PVE::RPCEnvironment::get();
6349                 if ($rpcenv->{type} ne 'cli') {
6350                     my $authuser = $rpcenv->get_user();
6351                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6352                 }
6353             }
6354
6355
6356             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6357                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6358             $path = PVE::Storage::path($storecfg, $vmstate);
6359             PVE::QemuConfig->write_config($vmid, $conf);
6360         } else {
6361             mon_cmd($vmid, "stop");
6362         }
6363     });
6364
6365     if ($includestate) {
6366         # save vm state
6367         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6368
6369         eval {
6370             set_migration_caps($vmid, 1);
6371             mon_cmd($vmid, "savevm-start", statefile => $path);
6372             for(;;) {
6373                 my $state = mon_cmd($vmid, "query-savevm");
6374                 if (!$state->{status}) {
6375                     die "savevm not active\n";
6376                 } elsif ($state->{status} eq 'active') {
6377                     sleep(1);
6378                     next;
6379                 } elsif ($state->{status} eq 'completed') {
6380                     print "State saved, quitting\n";
6381                     last;
6382                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6383                     die "query-savevm failed with error '$state->{error}'\n"
6384                 } else {
6385                     die "query-savevm returned status '$state->{status}'\n";
6386                 }
6387             }
6388         };
6389         my $err = $@;
6390
6391         PVE::QemuConfig->lock_config($vmid, sub {
6392             $conf = PVE::QemuConfig->load_config($vmid);
6393             if ($err) {
6394                 # cleanup, but leave suspending lock, to indicate something went wrong
6395                 eval {
6396                     mon_cmd($vmid, "savevm-end");
6397                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6398                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6399                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6400                     PVE::QemuConfig->write_config($vmid, $conf);
6401                 };
6402                 warn $@ if $@;
6403                 die $err;
6404             }
6405
6406             die "lock changed unexpectedly\n"
6407                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6408
6409             mon_cmd($vmid, "quit");
6410             $conf->{lock} = 'suspended';
6411             PVE::QemuConfig->write_config($vmid, $conf);
6412         });
6413     }
6414 }
6415
6416 # $nocheck is set when called as part of a migration - in this context the
6417 # location of the config file (source or target node) is not deterministic,
6418 # since migration cannot wait for pmxcfs to process the rename
6419 sub vm_resume {
6420     my ($vmid, $skiplock, $nocheck) = @_;
6421
6422     PVE::QemuConfig->lock_config($vmid, sub {
6423         my $res = mon_cmd($vmid, 'query-status');
6424         my $resume_cmd = 'cont';
6425         my $reset = 0;
6426         my $conf;
6427         if ($nocheck) {
6428             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6429             if ($@) {
6430                 my $vmlist = PVE::Cluster::get_vmlist();
6431                 if (exists($vmlist->{ids}->{$vmid})) {
6432                     my $node = $vmlist->{ids}->{$vmid}->{node};
6433                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6434                 }
6435                 if (!$conf) {
6436                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6437                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6438                 }
6439             }
6440         } else {
6441             $conf = PVE::QemuConfig->load_config($vmid);
6442         }
6443
6444         if ($res->{status}) {
6445             return if $res->{status} eq 'running'; # job done, go home
6446             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6447             $reset = 1 if $res->{status} eq 'shutdown';
6448         }
6449
6450         if (!$nocheck) {
6451             PVE::QemuConfig->check_lock($conf)
6452                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6453         }
6454
6455         if ($reset) {
6456             # required if a VM shuts down during a backup and we get a resume
6457             # request before the backup finishes for example
6458             mon_cmd($vmid, "system_reset");
6459         }
6460
6461         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6462
6463         mon_cmd($vmid, $resume_cmd);
6464     });
6465 }
6466
6467 sub vm_sendkey {
6468     my ($vmid, $skiplock, $key) = @_;
6469
6470     PVE::QemuConfig->lock_config($vmid, sub {
6471
6472         my $conf = PVE::QemuConfig->load_config($vmid);
6473
6474         # there is no qmp command, so we use the human monitor command
6475         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6476         die $res if $res ne '';
6477     });
6478 }
6479
6480 sub check_bridge_access {
6481     my ($rpcenv, $authuser, $conf) = @_;
6482
6483     return 1 if $authuser eq 'root@pam';
6484
6485     for my $opt (sort keys $conf->%*) {
6486         next if $opt !~ m/^net\d+$/;
6487         my $net = parse_net($conf->{$opt});
6488         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6489         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6490     }
6491     return 1;
6492 };
6493
6494 sub check_mapping_access {
6495     my ($rpcenv, $user, $conf) = @_;
6496
6497     for my $opt (keys $conf->%*) {
6498         if ($opt =~ m/^usb\d+$/) {
6499             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6500             if (my $host = $device->{host}) {
6501                 die "only root can set '$opt' config for real devices\n"
6502                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6503             } elsif ($device->{mapping}) {
6504                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6505             } else {
6506                 die "either 'host' or 'mapping' must be set.\n";
6507             }
6508         } elsif ($opt =~ m/^hostpci\d+$/) {
6509             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6510             if ($device->{host}) {
6511                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6512             } elsif ($device->{mapping}) {
6513                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6514             } else {
6515                 die "either 'host' or 'mapping' must be set.\n";
6516             }
6517        }
6518    }
6519 };
6520
6521 sub check_restore_permissions {
6522     my ($rpcenv, $user, $conf) = @_;
6523
6524     check_bridge_access($rpcenv, $user, $conf);
6525     check_mapping_access($rpcenv, $user, $conf);
6526 }
6527 # vzdump restore implementaion
6528
6529 sub tar_archive_read_firstfile {
6530     my $archive = shift;
6531
6532     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6533
6534     # try to detect archive type first
6535     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6536         die "unable to open file '$archive'\n";
6537     my $firstfile = <$fh>;
6538     kill 15, $pid;
6539     close $fh;
6540
6541     die "ERROR: archive contaions no data\n" if !$firstfile;
6542     chomp $firstfile;
6543
6544     return $firstfile;
6545 }
6546
6547 sub tar_restore_cleanup {
6548     my ($storecfg, $statfile) = @_;
6549
6550     print STDERR "starting cleanup\n";
6551
6552     if (my $fd = IO::File->new($statfile, "r")) {
6553         while (defined(my $line = <$fd>)) {
6554             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6555                 my $volid = $2;
6556                 eval {
6557                     if ($volid =~ m|^/|) {
6558                         unlink $volid || die 'unlink failed\n';
6559                     } else {
6560                         PVE::Storage::vdisk_free($storecfg, $volid);
6561                     }
6562                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6563                 };
6564                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6565             } else {
6566                 print STDERR "unable to parse line in statfile - $line";
6567             }
6568         }
6569         $fd->close();
6570     }
6571 }
6572
6573 sub restore_file_archive {
6574     my ($archive, $vmid, $user, $opts) = @_;
6575
6576     return restore_vma_archive($archive, $vmid, $user, $opts)
6577         if $archive eq '-';
6578
6579     my $info = PVE::Storage::archive_info($archive);
6580     my $format = $opts->{format} // $info->{format};
6581     my $comp = $info->{compression};
6582
6583     # try to detect archive format
6584     if ($format eq 'tar') {
6585         return restore_tar_archive($archive, $vmid, $user, $opts);
6586     } else {
6587         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6588     }
6589 }
6590
6591 # hepler to remove disks that will not be used after restore
6592 my $restore_cleanup_oldconf = sub {
6593     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6594
6595     my $kept_disks = {};
6596
6597     PVE::QemuConfig->foreach_volume($oldconf, sub {
6598         my ($ds, $drive) = @_;
6599
6600         return if drive_is_cdrom($drive, 1);
6601
6602         my $volid = $drive->{file};
6603         return if !$volid || $volid =~ m|^/|;
6604
6605         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6606         return if !$path || !$owner || ($owner != $vmid);
6607
6608         # Note: only delete disk we want to restore
6609         # other volumes will become unused
6610         if ($virtdev_hash->{$ds}) {
6611             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6612             if (my $err = $@) {
6613                 warn $err;
6614             }
6615         } else {
6616             $kept_disks->{$volid} = 1;
6617         }
6618     });
6619
6620     # after the restore we have no snapshots anymore
6621     for my $snapname (keys $oldconf->{snapshots}->%*) {
6622         my $snap = $oldconf->{snapshots}->{$snapname};
6623         if ($snap->{vmstate}) {
6624             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6625             if (my $err = $@) {
6626                 warn $err;
6627             }
6628         }
6629
6630         for my $volid (keys $kept_disks->%*) {
6631             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6632             warn $@ if $@;
6633         }
6634     }
6635 };
6636
6637 # Helper to parse vzdump backup device hints
6638 #
6639 # $rpcenv: Environment, used to ckeck storage permissions
6640 # $user: User ID, to check storage permissions
6641 # $storecfg: Storage configuration
6642 # $fh: the file handle for reading the configuration
6643 # $devinfo: should contain device sizes for all backu-up'ed devices
6644 # $options: backup options (pool, default storage)
6645 #
6646 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6647 my $parse_backup_hints = sub {
6648     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6649
6650     my $check_storage = sub { # assert if an image can be allocate
6651         my ($storeid, $scfg) = @_;
6652         die "Content type 'images' is not available on storage '$storeid'\n"
6653             if !$scfg->{content}->{images};
6654         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6655             if $user ne 'root@pam';
6656     };
6657
6658     my $virtdev_hash = {};
6659     while (defined(my $line = <$fh>)) {
6660         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6661             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6662             die "archive does not contain data for drive '$virtdev'\n"
6663                 if !$devinfo->{$devname};
6664
6665             if (defined($options->{storage})) {
6666                 $storeid = $options->{storage} || 'local';
6667             } elsif (!$storeid) {
6668                 $storeid = 'local';
6669             }
6670             $format = 'raw' if !$format;
6671             $devinfo->{$devname}->{devname} = $devname;
6672             $devinfo->{$devname}->{virtdev} = $virtdev;
6673             $devinfo->{$devname}->{format} = $format;
6674             $devinfo->{$devname}->{storeid} = $storeid;
6675
6676             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6677             $check_storage->($storeid, $scfg); # permission and content type check
6678
6679             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6680         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6681             my $virtdev = $1;
6682             my $drive = parse_drive($virtdev, $2);
6683
6684             if (drive_is_cloudinit($drive)) {
6685                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6686                 $storeid = $options->{storage} if defined ($options->{storage});
6687                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6688                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6689
6690                 $check_storage->($storeid, $scfg); # permission and content type check
6691
6692                 $virtdev_hash->{$virtdev} = {
6693                     format => $format,
6694                     storeid => $storeid,
6695                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6696                     is_cloudinit => 1,
6697                 };
6698             }
6699         }
6700     }
6701
6702     return $virtdev_hash;
6703 };
6704
6705 # Helper to allocate and activate all volumes required for a restore
6706 #
6707 # $storecfg: Storage configuration
6708 # $virtdev_hash: as returned by parse_backup_hints()
6709 #
6710 # Returns: { $virtdev => $volid }
6711 my $restore_allocate_devices = sub {
6712     my ($storecfg, $virtdev_hash, $vmid) = @_;
6713
6714     my $map = {};
6715     foreach my $virtdev (sort keys %$virtdev_hash) {
6716         my $d = $virtdev_hash->{$virtdev};
6717         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6718         my $storeid = $d->{storeid};
6719         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6720
6721         # test if requested format is supported
6722         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6723         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6724         $d->{format} = $defFormat if !$supported;
6725
6726         my $name;
6727         if ($d->{is_cloudinit}) {
6728             $name = "vm-$vmid-cloudinit";
6729             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6730             if ($scfg->{path}) {
6731                 $name .= ".$d->{format}";
6732             }
6733         }
6734
6735         my $volid = PVE::Storage::vdisk_alloc(
6736             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6737
6738         print STDERR "new volume ID is '$volid'\n";
6739         $d->{volid} = $volid;
6740
6741         PVE::Storage::activate_volumes($storecfg, [$volid]);
6742
6743         $map->{$virtdev} = $volid;
6744     }
6745
6746     return $map;
6747 };
6748
6749 sub restore_update_config_line {
6750     my ($cookie, $map, $line, $unique) = @_;
6751
6752     return '' if $line =~ m/^\#qmdump\#/;
6753     return '' if $line =~ m/^\#vzdump\#/;
6754     return '' if $line =~ m/^lock:/;
6755     return '' if $line =~ m/^unused\d+:/;
6756     return '' if $line =~ m/^parent:/;
6757
6758     my $res = '';
6759
6760     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6761     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6762         # try to convert old 1.X settings
6763         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6764         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6765             my ($model, $macaddr) = split(/\=/, $devconfig);
6766             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6767             my $net = {
6768                 model => $model,
6769                 bridge => "vmbr$ind",
6770                 macaddr => $macaddr,
6771             };
6772             my $netstr = print_net($net);
6773
6774             $res .= "net$cookie->{netcount}: $netstr\n";
6775             $cookie->{netcount}++;
6776         }
6777     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6778         my ($id, $netstr) = ($1, $2);
6779         my $net = parse_net($netstr);
6780         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6781         $netstr = print_net($net);
6782         $res .= "$id: $netstr\n";
6783     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6784         my $virtdev = $1;
6785         my $value = $3;
6786         my $di = parse_drive($virtdev, $value);
6787         if (defined($di->{backup}) && !$di->{backup}) {
6788             $res .= "#$line";
6789         } elsif ($map->{$virtdev}) {
6790             delete $di->{format}; # format can change on restore
6791             $di->{file} = $map->{$virtdev};
6792             $value = print_drive($di);
6793             $res .= "$virtdev: $value\n";
6794         } else {
6795             $res .= $line;
6796         }
6797     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6798         my $vmgenid = $1;
6799         if ($vmgenid ne '0') {
6800             # always generate a new vmgenid if there was a valid one setup
6801             $vmgenid = generate_uuid();
6802         }
6803         $res .= "vmgenid: $vmgenid\n";
6804     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6805         my ($uuid, $uuid_str);
6806         UUID::generate($uuid);
6807         UUID::unparse($uuid, $uuid_str);
6808         my $smbios1 = parse_smbios1($2);
6809         $smbios1->{uuid} = $uuid_str;
6810         $res .= $1.print_smbios1($smbios1)."\n";
6811     } else {
6812         $res .= $line;
6813     }
6814
6815     return $res;
6816 }
6817
6818 my $restore_deactivate_volumes = sub {
6819     my ($storecfg, $virtdev_hash) = @_;
6820
6821     my $vollist = [];
6822     for my $dev (values $virtdev_hash->%*) {
6823         push $vollist->@*, $dev->{volid} if $dev->{volid};
6824     }
6825
6826     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6827     print STDERR $@ if $@;
6828 };
6829
6830 my $restore_destroy_volumes = sub {
6831     my ($storecfg, $virtdev_hash) = @_;
6832
6833     for my $dev (values $virtdev_hash->%*) {
6834         my $volid = $dev->{volid} or next;
6835         eval {
6836             PVE::Storage::vdisk_free($storecfg, $volid);
6837             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6838         };
6839         print STDERR "unable to cleanup '$volid' - $@" if $@;
6840     }
6841 };
6842
6843 sub restore_merge_config {
6844     my ($filename, $backup_conf_raw, $override_conf) = @_;
6845
6846     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6847     for my $key (keys $override_conf->%*) {
6848         $backup_conf->{$key} = $override_conf->{$key};
6849     }
6850
6851     return $backup_conf;
6852 }
6853
6854 sub scan_volids {
6855     my ($cfg, $vmid) = @_;
6856
6857     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6858
6859     my $volid_hash = {};
6860     foreach my $storeid (keys %$info) {
6861         foreach my $item (@{$info->{$storeid}}) {
6862             next if !($item->{volid} && $item->{size});
6863             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6864             $volid_hash->{$item->{volid}} = $item;
6865         }
6866     }
6867
6868     return $volid_hash;
6869 }
6870
6871 sub update_disk_config {
6872     my ($vmid, $conf, $volid_hash) = @_;
6873
6874     my $changes;
6875     my $prefix = "VM $vmid";
6876
6877     # used and unused disks
6878     my $referenced = {};
6879
6880     # Note: it is allowed to define multiple storages with same path (alias), so
6881     # we need to check both 'volid' and real 'path' (two different volid can point
6882     # to the same path).
6883
6884     my $referencedpath = {};
6885
6886     # update size info
6887     PVE::QemuConfig->foreach_volume($conf, sub {
6888         my ($opt, $drive) = @_;
6889
6890         my $volid = $drive->{file};
6891         return if !$volid;
6892         my $volume = $volid_hash->{$volid};
6893
6894         # mark volid as "in-use" for next step
6895         $referenced->{$volid} = 1;
6896         if ($volume && (my $path = $volume->{path})) {
6897             $referencedpath->{$path} = 1;
6898         }
6899
6900         return if drive_is_cdrom($drive);
6901         return if !$volume;
6902
6903         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6904         if (defined($updated)) {
6905             $changes = 1;
6906             $conf->{$opt} = print_drive($updated);
6907             print "$prefix ($opt): $msg\n";
6908         }
6909     });
6910
6911     # remove 'unusedX' entry if volume is used
6912     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6913         my ($opt, $drive) = @_;
6914
6915         my $volid = $drive->{file};
6916         return if !$volid;
6917
6918         my $path;
6919         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6920         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6921             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6922             $changes = 1;
6923             delete $conf->{$opt};
6924         }
6925
6926         $referenced->{$volid} = 1;
6927         $referencedpath->{$path} = 1 if $path;
6928     });
6929
6930     foreach my $volid (sort keys %$volid_hash) {
6931         next if $volid =~ m/vm-$vmid-state-/;
6932         next if $referenced->{$volid};
6933         my $path = $volid_hash->{$volid}->{path};
6934         next if !$path; # just to be sure
6935         next if $referencedpath->{$path};
6936         $changes = 1;
6937         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6938         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6939         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6940     }
6941
6942     return $changes;
6943 }
6944
6945 sub rescan {
6946     my ($vmid, $nolock, $dryrun) = @_;
6947
6948     my $cfg = PVE::Storage::config();
6949
6950     print "rescan volumes...\n";
6951     my $volid_hash = scan_volids($cfg, $vmid);
6952
6953     my $updatefn =  sub {
6954         my ($vmid) = @_;
6955
6956         my $conf = PVE::QemuConfig->load_config($vmid);
6957
6958         PVE::QemuConfig->check_lock($conf);
6959
6960         my $vm_volids = {};
6961         foreach my $volid (keys %$volid_hash) {
6962             my $info = $volid_hash->{$volid};
6963             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6964         }
6965
6966         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6967
6968         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6969     };
6970
6971     if (defined($vmid)) {
6972         if ($nolock) {
6973             &$updatefn($vmid);
6974         } else {
6975             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6976         }
6977     } else {
6978         my $vmlist = config_list();
6979         foreach my $vmid (keys %$vmlist) {
6980             if ($nolock) {
6981                 &$updatefn($vmid);
6982             } else {
6983                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6984             }
6985         }
6986     }
6987 }
6988
6989 sub restore_proxmox_backup_archive {
6990     my ($archive, $vmid, $user, $options) = @_;
6991
6992     my $storecfg = PVE::Storage::config();
6993
6994     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6995     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6996
6997     my $fingerprint = $scfg->{fingerprint};
6998     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6999
7000     my $repo = PVE::PBSClient::get_repository($scfg);
7001     my $namespace = $scfg->{namespace};
7002
7003     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7004     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7005     local $ENV{PBS_PASSWORD} = $password;
7006     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7007
7008     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7009         PVE::Storage::parse_volname($storecfg, $archive);
7010
7011     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7012
7013     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7014
7015     my $tmpdir = "/var/tmp/vzdumptmp$$";
7016     rmtree $tmpdir;
7017     mkpath $tmpdir;
7018
7019     my $conffile = PVE::QemuConfig->config_file($vmid);
7020      # disable interrupts (always do cleanups)
7021     local $SIG{INT} =
7022         local $SIG{TERM} =
7023         local $SIG{QUIT} =
7024         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7025
7026     # Note: $oldconf is undef if VM does not exists
7027     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7028     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7029     my $new_conf_raw = '';
7030
7031     my $rpcenv = PVE::RPCEnvironment::get();
7032     my $devinfo = {}; # info about drives included in backup
7033     my $virtdev_hash = {}; # info about allocated drives
7034
7035     eval {
7036         # enable interrupts
7037         local $SIG{INT} =
7038             local $SIG{TERM} =
7039             local $SIG{QUIT} =
7040             local $SIG{HUP} =
7041             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7042
7043         my $cfgfn = "$tmpdir/qemu-server.conf";
7044         my $firewall_config_fn = "$tmpdir/fw.conf";
7045         my $index_fn = "$tmpdir/index.json";
7046
7047         my $cmd = "restore";
7048
7049         my $param = [$pbs_backup_name, "index.json", $index_fn];
7050         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7051         my $index = PVE::Tools::file_get_contents($index_fn);
7052         $index = decode_json($index);
7053
7054         foreach my $info (@{$index->{files}}) {
7055             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7056                 my $devname = $1;
7057                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7058                     $devinfo->{$devname}->{size} = $1;
7059                 } else {
7060                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7061                 }
7062             }
7063         }
7064
7065         my $is_qemu_server_backup = scalar(
7066             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7067         );
7068         if (!$is_qemu_server_backup) {
7069             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7070         }
7071         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7072
7073         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7074         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7075
7076         if ($has_firewall_config) {
7077             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7078             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7079
7080             my $pve_firewall_dir = '/etc/pve/firewall';
7081             mkdir $pve_firewall_dir; # make sure the dir exists
7082             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7083         }
7084
7085         my $fh = IO::File->new($cfgfn, "r") ||
7086             die "unable to read qemu-server.conf - $!\n";
7087
7088         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7089
7090         # fixme: rate limit?
7091
7092         # create empty/temp config
7093         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7094
7095         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7096
7097         # allocate volumes
7098         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7099
7100         foreach my $virtdev (sort keys %$virtdev_hash) {
7101             my $d = $virtdev_hash->{$virtdev};
7102             next if $d->{is_cloudinit}; # no need to restore cloudinit
7103
7104             # this fails if storage is unavailable
7105             my $volid = $d->{volid};
7106             my $path = PVE::Storage::path($storecfg, $volid);
7107
7108             # for live-restore we only want to preload the efidisk and TPM state
7109             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7110
7111             my @ns_arg;
7112             if (defined(my $ns = $scfg->{namespace})) {
7113                 @ns_arg = ('--ns', $ns);
7114             }
7115
7116             my $pbs_restore_cmd = [
7117                 '/usr/bin/pbs-restore',
7118                 '--repository', $repo,
7119                 @ns_arg,
7120                 $pbs_backup_name,
7121                 "$d->{devname}.img.fidx",
7122                 $path,
7123                 '--verbose',
7124                 ];
7125
7126             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7127             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7128
7129             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7130                 push @$pbs_restore_cmd, '--skip-zero';
7131             }
7132
7133             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7134             print "restore proxmox backup image: $dbg_cmdstring\n";
7135             run_command($pbs_restore_cmd);
7136         }
7137
7138         $fh->seek(0, 0) || die "seek failed - $!\n";
7139
7140         my $cookie = { netcount => 0 };
7141         while (defined(my $line = <$fh>)) {
7142             $new_conf_raw .= restore_update_config_line(
7143                 $cookie,
7144                 $map,
7145                 $line,
7146                 $options->{unique},
7147             );
7148         }
7149
7150         $fh->close();
7151     };
7152     my $err = $@;
7153
7154     if ($err || !$options->{live}) {
7155         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7156     }
7157
7158     rmtree $tmpdir;
7159
7160     if ($err) {
7161         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7162         die $err;
7163     }
7164
7165     if ($options->{live}) {
7166         # keep lock during live-restore
7167         $new_conf_raw .= "\nlock: create";
7168     }
7169
7170     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7171     check_restore_permissions($rpcenv, $user, $new_conf);
7172     PVE::QemuConfig->write_config($vmid, $new_conf);
7173
7174     eval { rescan($vmid, 1); };
7175     warn $@ if $@;
7176
7177     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7178
7179     if ($options->{live}) {
7180         # enable interrupts
7181         local $SIG{INT} =
7182             local $SIG{TERM} =
7183             local $SIG{QUIT} =
7184             local $SIG{HUP} =
7185             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7186
7187         my $conf = PVE::QemuConfig->load_config($vmid);
7188         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7189
7190         # these special drives are already restored before start
7191         delete $devinfo->{'drive-efidisk0'};
7192         delete $devinfo->{'drive-tpmstate0-backup'};
7193
7194         my $pbs_opts = {
7195             repo => $repo,
7196             keyfile => $keyfile,
7197             snapshot => $pbs_backup_name,
7198             namespace => $namespace,
7199         };
7200         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7201
7202         PVE::QemuConfig->remove_lock($vmid, "create");
7203     }
7204 }
7205
7206 sub pbs_live_restore {
7207     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7208
7209     print "starting VM for live-restore\n";
7210     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7211
7212     my $pbs_backing = {};
7213     for my $ds (keys %$restored_disks) {
7214         $ds =~ m/^drive-(.*)$/;
7215         my $confname = $1;
7216         $pbs_backing->{$confname} = {
7217             repository => $opts->{repo},
7218             snapshot => $opts->{snapshot},
7219             archive => "$ds.img.fidx",
7220         };
7221         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7222         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7223
7224         my $drive = parse_drive($confname, $conf->{$confname});
7225         print "restoring '$ds' to '$drive->{file}'\n";
7226     }
7227
7228     my $drives_streamed = 0;
7229     eval {
7230         # make sure HA doesn't interrupt our restore by stopping the VM
7231         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7232             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7233         }
7234
7235         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7236         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7237         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7238
7239         my $qmeventd_fd = register_qmeventd_handle($vmid);
7240
7241         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7242         # this will effectively collapse the backing image chain consisting of
7243         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7244         # removes itself once all backing images vanish with 'auto-remove=on')
7245         my $jobs = {};
7246         for my $ds (sort keys %$restored_disks) {
7247             my $job_id = "restore-$ds";
7248             mon_cmd($vmid, 'block-stream',
7249                 'job-id' => $job_id,
7250                 device => "$ds",
7251             );
7252             $jobs->{$job_id} = {};
7253         }
7254
7255         mon_cmd($vmid, 'cont');
7256         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7257
7258         print "restore-drive jobs finished successfully, removing all tracking block devices"
7259             ." to disconnect from Proxmox Backup Server\n";
7260
7261         for my $ds (sort keys %$restored_disks) {
7262             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7263         }
7264
7265         close($qmeventd_fd);
7266     };
7267
7268     my $err = $@;
7269
7270     if ($err) {
7271         warn "An error occurred during live-restore: $err\n";
7272         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7273         die "live-restore failed\n";
7274     }
7275 }
7276
7277 sub restore_vma_archive {
7278     my ($archive, $vmid, $user, $opts, $comp) = @_;
7279
7280     my $readfrom = $archive;
7281
7282     my $cfg = PVE::Storage::config();
7283     my $commands = [];
7284     my $bwlimit = $opts->{bwlimit};
7285
7286     my $dbg_cmdstring = '';
7287     my $add_pipe = sub {
7288         my ($cmd) = @_;
7289         push @$commands, $cmd;
7290         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7291         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7292         $readfrom = '-';
7293     };
7294
7295     my $input = undef;
7296     if ($archive eq '-') {
7297         $input = '<&STDIN';
7298     } else {
7299         # If we use a backup from a PVE defined storage we also consider that
7300         # storage's rate limit:
7301         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7302         if (defined($volid)) {
7303             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7304             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7305             if ($readlimit) {
7306                 print STDERR "applying read rate limit: $readlimit\n";
7307                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7308                 $add_pipe->($cstream);
7309             }
7310         }
7311     }
7312
7313     if ($comp) {
7314         my $info = PVE::Storage::decompressor_info('vma', $comp);
7315         my $cmd = $info->{decompressor};
7316         push @$cmd, $readfrom;
7317         $add_pipe->($cmd);
7318     }
7319
7320     my $tmpdir = "/var/tmp/vzdumptmp$$";
7321     rmtree $tmpdir;
7322
7323     # disable interrupts (always do cleanups)
7324     local $SIG{INT} =
7325         local $SIG{TERM} =
7326         local $SIG{QUIT} =
7327         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7328
7329     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7330     POSIX::mkfifo($mapfifo, 0600);
7331     my $fifofh;
7332     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7333
7334     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7335
7336     my $devinfo = {}; # info about drives included in backup
7337     my $virtdev_hash = {}; # info about allocated drives
7338
7339     my $rpcenv = PVE::RPCEnvironment::get();
7340
7341     my $conffile = PVE::QemuConfig->config_file($vmid);
7342
7343     # Note: $oldconf is undef if VM does not exist
7344     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7345     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7346     my $new_conf_raw = '';
7347
7348     my %storage_limits;
7349
7350     my $print_devmap = sub {
7351         my $cfgfn = "$tmpdir/qemu-server.conf";
7352
7353         # we can read the config - that is already extracted
7354         my $fh = IO::File->new($cfgfn, "r") ||
7355             die "unable to read qemu-server.conf - $!\n";
7356
7357         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7358         if (-f $fwcfgfn) {
7359             my $pve_firewall_dir = '/etc/pve/firewall';
7360             mkdir $pve_firewall_dir; # make sure the dir exists
7361             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7362         }
7363
7364         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7365
7366         foreach my $info (values %{$virtdev_hash}) {
7367             my $storeid = $info->{storeid};
7368             next if defined($storage_limits{$storeid});
7369
7370             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7371             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7372             $storage_limits{$storeid} = $limit * 1024;
7373         }
7374
7375         foreach my $devname (keys %$devinfo) {
7376             die "found no device mapping information for device '$devname'\n"
7377                 if !$devinfo->{$devname}->{virtdev};
7378         }
7379
7380         # create empty/temp config
7381         if ($oldconf) {
7382             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7383             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7384         }
7385
7386         # allocate volumes
7387         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7388
7389         # print restore information to $fifofh
7390         foreach my $virtdev (sort keys %$virtdev_hash) {
7391             my $d = $virtdev_hash->{$virtdev};
7392             next if $d->{is_cloudinit}; # no need to restore cloudinit
7393
7394             my $storeid = $d->{storeid};
7395             my $volid = $d->{volid};
7396
7397             my $map_opts = '';
7398             if (my $limit = $storage_limits{$storeid}) {
7399                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7400             }
7401
7402             my $write_zeros = 1;
7403             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7404                 $write_zeros = 0;
7405             }
7406
7407             my $path = PVE::Storage::path($cfg, $volid);
7408
7409             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7410
7411             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7412         }
7413
7414         $fh->seek(0, 0) || die "seek failed - $!\n";
7415
7416         my $cookie = { netcount => 0 };
7417         while (defined(my $line = <$fh>)) {
7418             $new_conf_raw .= restore_update_config_line(
7419                 $cookie,
7420                 $map,
7421                 $line,
7422                 $opts->{unique},
7423             );
7424         }
7425
7426         $fh->close();
7427     };
7428
7429     my $oldtimeout;
7430
7431     eval {
7432         # enable interrupts
7433         local $SIG{INT} =
7434             local $SIG{TERM} =
7435             local $SIG{QUIT} =
7436             local $SIG{HUP} =
7437             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7438         local $SIG{ALRM} = sub { die "got timeout\n"; };
7439
7440         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7441
7442         my $parser = sub {
7443             my $line = shift;
7444
7445             print "$line\n";
7446
7447             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7448                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7449                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7450             } elsif ($line =~ m/^CTIME: /) {
7451                 # we correctly received the vma config, so we can disable
7452                 # the timeout now for disk allocation
7453                 alarm($oldtimeout || 0);
7454                 $oldtimeout = undef;
7455                 &$print_devmap();
7456                 print $fifofh "done\n";
7457                 close($fifofh);
7458                 $fifofh = undef;
7459             }
7460         };
7461
7462         print "restore vma archive: $dbg_cmdstring\n";
7463         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7464     };
7465     my $err = $@;
7466
7467     alarm($oldtimeout) if $oldtimeout;
7468
7469     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7470
7471     close($fifofh) if $fifofh;
7472     unlink $mapfifo;
7473     rmtree $tmpdir;
7474
7475     if ($err) {
7476         $restore_destroy_volumes->($cfg, $virtdev_hash);
7477         die $err;
7478     }
7479
7480     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7481     check_restore_permissions($rpcenv, $user, $new_conf);
7482     PVE::QemuConfig->write_config($vmid, $new_conf);
7483
7484     eval { rescan($vmid, 1); };
7485     warn $@ if $@;
7486
7487     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7488 }
7489
7490 sub restore_tar_archive {
7491     my ($archive, $vmid, $user, $opts) = @_;
7492
7493     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7494         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7495         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7496     }
7497
7498     if ($archive ne '-') {
7499         my $firstfile = tar_archive_read_firstfile($archive);
7500         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7501             if $firstfile ne 'qemu-server.conf';
7502     }
7503
7504     my $storecfg = PVE::Storage::config();
7505
7506     # avoid zombie disks when restoring over an existing VM -> cleanup first
7507     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7508     # skiplock=1 because qmrestore has set the 'create' lock itself already
7509     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7510     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7511
7512     my $tocmd = "/usr/lib/qemu-server/qmextract";
7513
7514     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7515     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7516     $tocmd .= ' --prealloc' if $opts->{prealloc};
7517     $tocmd .= ' --info' if $opts->{info};
7518
7519     # tar option "xf" does not autodetect compression when read from STDIN,
7520     # so we pipe to zcat
7521     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7522         PVE::Tools::shellquote("--to-command=$tocmd");
7523
7524     my $tmpdir = "/var/tmp/vzdumptmp$$";
7525     mkpath $tmpdir;
7526
7527     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7528     local $ENV{VZDUMP_VMID} = $vmid;
7529     local $ENV{VZDUMP_USER} = $user;
7530
7531     my $conffile = PVE::QemuConfig->config_file($vmid);
7532     my $new_conf_raw = '';
7533
7534     # disable interrupts (always do cleanups)
7535     local $SIG{INT} =
7536         local $SIG{TERM} =
7537         local $SIG{QUIT} =
7538         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7539
7540     eval {
7541         # enable interrupts
7542         local $SIG{INT} =
7543             local $SIG{TERM} =
7544             local $SIG{QUIT} =
7545             local $SIG{HUP} =
7546             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7547
7548         if ($archive eq '-') {
7549             print "extracting archive from STDIN\n";
7550             run_command($cmd, input => "<&STDIN");
7551         } else {
7552             print "extracting archive '$archive'\n";
7553             run_command($cmd);
7554         }
7555
7556         return if $opts->{info};
7557
7558         # read new mapping
7559         my $map = {};
7560         my $statfile = "$tmpdir/qmrestore.stat";
7561         if (my $fd = IO::File->new($statfile, "r")) {
7562             while (defined (my $line = <$fd>)) {
7563                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7564                     $map->{$1} = $2 if $1;
7565                 } else {
7566                     print STDERR "unable to parse line in statfile - $line\n";
7567                 }
7568             }
7569             $fd->close();
7570         }
7571
7572         my $confsrc = "$tmpdir/qemu-server.conf";
7573
7574         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7575
7576         my $cookie = { netcount => 0 };
7577         while (defined (my $line = <$srcfd>)) {
7578             $new_conf_raw .= restore_update_config_line(
7579                 $cookie,
7580                 $map,
7581                 $line,
7582                 $opts->{unique},
7583             );
7584         }
7585
7586         $srcfd->close();
7587     };
7588     if (my $err = $@) {
7589         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7590         die $err;
7591     }
7592
7593     rmtree $tmpdir;
7594
7595     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7596
7597     PVE::Cluster::cfs_update(); # make sure we read new file
7598
7599     eval { rescan($vmid, 1); };
7600     warn $@ if $@;
7601 };
7602
7603 sub foreach_storage_used_by_vm {
7604     my ($conf, $func) = @_;
7605
7606     my $sidhash = {};
7607
7608     PVE::QemuConfig->foreach_volume($conf, sub {
7609         my ($ds, $drive) = @_;
7610         return if drive_is_cdrom($drive);
7611
7612         my $volid = $drive->{file};
7613
7614         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7615         $sidhash->{$sid} = $sid if $sid;
7616     });
7617
7618     foreach my $sid (sort keys %$sidhash) {
7619         &$func($sid);
7620     }
7621 }
7622
7623 my $qemu_snap_storage = {
7624     rbd => 1,
7625 };
7626 sub do_snapshots_with_qemu {
7627     my ($storecfg, $volid, $deviceid) = @_;
7628
7629     return if $deviceid =~ m/tpmstate0/;
7630
7631     my $storage_name = PVE::Storage::parse_volume_id($volid);
7632     my $scfg = $storecfg->{ids}->{$storage_name};
7633     die "could not find storage '$storage_name'\n" if !defined($scfg);
7634
7635     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7636         return 1;
7637     }
7638
7639     if ($volid =~ m/\.(qcow2|qed)$/){
7640         return 1;
7641     }
7642
7643     return;
7644 }
7645
7646 sub qga_check_running {
7647     my ($vmid, $nowarn) = @_;
7648
7649     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7650     if ($@) {
7651         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7652         return 0;
7653     }
7654     return 1;
7655 }
7656
7657 sub template_create {
7658     my ($vmid, $conf, $disk) = @_;
7659
7660     my $storecfg = PVE::Storage::config();
7661
7662     PVE::QemuConfig->foreach_volume($conf, sub {
7663         my ($ds, $drive) = @_;
7664
7665         return if drive_is_cdrom($drive);
7666         return if $disk && $ds ne $disk;
7667
7668         my $volid = $drive->{file};
7669         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7670
7671         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7672         $drive->{file} = $voliddst;
7673         $conf->{$ds} = print_drive($drive);
7674         PVE::QemuConfig->write_config($vmid, $conf);
7675     });
7676 }
7677
7678 sub convert_iscsi_path {
7679     my ($path) = @_;
7680
7681     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7682         my $portal = $1;
7683         my $target = $2;
7684         my $lun = $3;
7685
7686         my $initiator_name = get_initiator_name();
7687
7688         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7689                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7690     }
7691
7692     die "cannot convert iscsi path '$path', unkown format\n";
7693 }
7694
7695 sub qemu_img_convert {
7696     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7697
7698     my $storecfg = PVE::Storage::config();
7699     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7700     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7701
7702     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7703
7704     my $cachemode;
7705     my $src_path;
7706     my $src_is_iscsi = 0;
7707     my $src_format;
7708
7709     if ($src_storeid) {
7710         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7711         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7712         $src_format = qemu_img_format($src_scfg, $src_volname);
7713         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7714         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7715         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7716     } elsif (-f $src_volid || -b $src_volid) {
7717         $src_path = $src_volid;
7718         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7719             $src_format = $1;
7720         }
7721     }
7722
7723     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7724
7725     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7726     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7727     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7728     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7729
7730     my $cmd = [];
7731     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7732     push @$cmd, '-l', "snapshot.name=$snapname"
7733         if $snapname && $src_format && $src_format eq "qcow2";
7734     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7735     push @$cmd, '-T', $cachemode if defined($cachemode);
7736     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7737
7738     if ($src_is_iscsi) {
7739         push @$cmd, '--image-opts';
7740         $src_path = convert_iscsi_path($src_path);
7741     } elsif ($src_format) {
7742         push @$cmd, '-f', $src_format;
7743     }
7744
7745     if ($dst_is_iscsi) {
7746         push @$cmd, '--target-image-opts';
7747         $dst_path = convert_iscsi_path($dst_path);
7748     } else {
7749         push @$cmd, '-O', $dst_format;
7750     }
7751
7752     push @$cmd, $src_path;
7753
7754     if (!$dst_is_iscsi && $is_zero_initialized) {
7755         push @$cmd, "zeroinit:$dst_path";
7756     } else {
7757         push @$cmd, $dst_path;
7758     }
7759
7760     my $parser = sub {
7761         my $line = shift;
7762         if($line =~ m/\((\S+)\/100\%\)/){
7763             my $percent = $1;
7764             my $transferred = int($size * $percent / 100);
7765             my $total_h = render_bytes($size, 1);
7766             my $transferred_h = render_bytes($transferred, 1);
7767
7768             print "transferred $transferred_h of $total_h ($percent%)\n";
7769         }
7770
7771     };
7772
7773     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7774     my $err = $@;
7775     die "copy failed: $err" if $err;
7776 }
7777
7778 sub qemu_img_format {
7779     my ($scfg, $volname) = @_;
7780
7781     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7782         return $1;
7783     } else {
7784         return "raw";
7785     }
7786 }
7787
7788 sub qemu_drive_mirror {
7789     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7790
7791     $jobs = {} if !$jobs;
7792
7793     my $qemu_target;
7794     my $format;
7795     $jobs->{"drive-$drive"} = {};
7796
7797     if ($dst_volid =~ /^nbd:/) {
7798         $qemu_target = $dst_volid;
7799         $format = "nbd";
7800     } else {
7801         my $storecfg = PVE::Storage::config();
7802         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7803
7804         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7805
7806         $format = qemu_img_format($dst_scfg, $dst_volname);
7807
7808         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7809
7810         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7811     }
7812
7813     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7814     $opts->{format} = $format if $format;
7815
7816     if (defined($src_bitmap)) {
7817         $opts->{sync} = 'incremental';
7818         $opts->{bitmap} = $src_bitmap;
7819         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7820     }
7821
7822     if (defined($bwlimit)) {
7823         $opts->{speed} = $bwlimit * 1024;
7824         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7825     } else {
7826         print "drive mirror is starting for drive-$drive\n";
7827     }
7828
7829     # if a job already runs for this device we get an error, catch it for cleanup
7830     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7831     if (my $err = $@) {
7832         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7833         warn "$@\n" if $@;
7834         die "mirroring error: $err\n";
7835     }
7836
7837     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7838 }
7839
7840 # $completion can be either
7841 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7842 # 'cancel': wait until all jobs are ready, block-job-cancel them
7843 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7844 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7845 sub qemu_drive_mirror_monitor {
7846     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7847
7848     $completion //= 'complete';
7849     $op //= "mirror";
7850
7851     eval {
7852         my $err_complete = 0;
7853
7854         my $starttime = time ();
7855         while (1) {
7856             die "block job ('$op') timed out\n" if $err_complete > 300;
7857
7858             my $stats = mon_cmd($vmid, "query-block-jobs");
7859             my $ctime = time();
7860
7861             my $running_jobs = {};
7862             for my $stat (@$stats) {
7863                 next if $stat->{type} ne $op;
7864                 $running_jobs->{$stat->{device}} = $stat;
7865             }
7866
7867             my $readycounter = 0;
7868
7869             for my $job_id (sort keys %$jobs) {
7870                 my $job = $running_jobs->{$job_id};
7871
7872                 my $vanished = !defined($job);
7873                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7874                 if($complete || ($vanished && $completion eq 'auto')) {
7875                     print "$job_id: $op-job finished\n";
7876                     delete $jobs->{$job_id};
7877                     next;
7878                 }
7879
7880                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7881
7882                 my $busy = $job->{busy};
7883                 my $ready = $job->{ready};
7884                 if (my $total = $job->{len}) {
7885                     my $transferred = $job->{offset} || 0;
7886                     my $remaining = $total - $transferred;
7887                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7888
7889                     my $duration = $ctime - $starttime;
7890                     my $total_h = render_bytes($total, 1);
7891                     my $transferred_h = render_bytes($transferred, 1);
7892
7893                     my $status = sprintf(
7894                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7895                         render_duration($duration),
7896                     );
7897
7898                     if ($ready) {
7899                         if ($busy) {
7900                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7901                         } else {
7902                             $status .= ", ready";
7903                         }
7904                     }
7905                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7906                     $jobs->{$job_id}->{ready} = $ready;
7907                 }
7908
7909                 $readycounter++ if $job->{ready};
7910             }
7911
7912             last if scalar(keys %$jobs) == 0;
7913
7914             if ($readycounter == scalar(keys %$jobs)) {
7915                 print "all '$op' jobs are ready\n";
7916
7917                 # do the complete later (or has already been done)
7918                 last if $completion eq 'skip' || $completion eq 'auto';
7919
7920                 if ($vmiddst && $vmiddst != $vmid) {
7921                     my $agent_running = $qga && qga_check_running($vmid);
7922                     if ($agent_running) {
7923                         print "freeze filesystem\n";
7924                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7925                         warn $@ if $@;
7926                     } else {
7927                         print "suspend vm\n";
7928                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7929                         warn $@ if $@;
7930                     }
7931
7932                     # if we clone a disk for a new target vm, we don't switch the disk
7933                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7934
7935                     if ($agent_running) {
7936                         print "unfreeze filesystem\n";
7937                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7938                         warn $@ if $@;
7939                     } else {
7940                         print "resume vm\n";
7941                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7942                         warn $@ if $@;
7943                     }
7944
7945                     last;
7946                 } else {
7947
7948                     for my $job_id (sort keys %$jobs) {
7949                         # try to switch the disk if source and destination are on the same guest
7950                         print "$job_id: Completing block job_id...\n";
7951
7952                         my $op;
7953                         if ($completion eq 'complete') {
7954                             $op = 'block-job-complete';
7955                         } elsif ($completion eq 'cancel') {
7956                             $op = 'block-job-cancel';
7957                         } else {
7958                             die "invalid completion value: $completion\n";
7959                         }
7960                         eval { mon_cmd($vmid, $op, device => $job_id) };
7961                         if ($@ =~ m/cannot be completed/) {
7962                             print "$job_id: block job cannot be completed, trying again.\n";
7963                             $err_complete++;
7964                         }else {
7965                             print "$job_id: Completed successfully.\n";
7966                             $jobs->{$job_id}->{complete} = 1;
7967                         }
7968                     }
7969                 }
7970             }
7971             sleep 1;
7972         }
7973     };
7974     my $err = $@;
7975
7976     if ($err) {
7977         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7978         die "block job ($op) error: $err";
7979     }
7980 }
7981
7982 sub qemu_blockjobs_cancel {
7983     my ($vmid, $jobs) = @_;
7984
7985     foreach my $job (keys %$jobs) {
7986         print "$job: Cancelling block job\n";
7987         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7988         $jobs->{$job}->{cancel} = 1;
7989     }
7990
7991     while (1) {
7992         my $stats = mon_cmd($vmid, "query-block-jobs");
7993
7994         my $running_jobs = {};
7995         foreach my $stat (@$stats) {
7996             $running_jobs->{$stat->{device}} = $stat;
7997         }
7998
7999         foreach my $job (keys %$jobs) {
8000
8001             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8002                 print "$job: Done.\n";
8003                 delete $jobs->{$job};
8004             }
8005         }
8006
8007         last if scalar(keys %$jobs) == 0;
8008
8009         sleep 1;
8010     }
8011 }
8012
8013 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8014 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8015 my sub clone_disk_check_io_uring {
8016     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8017
8018     return if !$use_drive_mirror;
8019
8020     # Don't complain when not changing storage.
8021     # Assume if it works for the source, it'll work for the target too.
8022     return if $src_storeid eq $dst_storeid;
8023
8024     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8025     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8026
8027     my $cache_direct = drive_uses_cache_direct($src_drive);
8028
8029     my $src_uses_io_uring;
8030     if ($src_drive->{aio}) {
8031         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8032     } else {
8033         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8034     }
8035
8036     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8037         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8038 }
8039
8040 sub clone_disk {
8041     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8042
8043     my ($vmid, $running) = $source->@{qw(vmid running)};
8044     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8045
8046     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8047     my ($storage, $format) = $dest->@{qw(storage format)};
8048
8049     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8050
8051     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8052         die "cloning from/to EFI disk requires EFI disk\n"
8053             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8054         die "cloning from/to TPM state requires TPM state\n"
8055             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8056
8057         # This would lead to two device nodes in QEMU pointing to the same backing image!
8058         die "cannot change drive name when cloning disk from/to the same VM\n"
8059             if $use_drive_mirror && $vmid == $newvmid;
8060     }
8061
8062     die "cannot move TPM state while VM is running\n"
8063         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8064
8065     my $newvolid;
8066
8067     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8068     print "$src_drivename " if $src_drivename;
8069     print "($drive->{file})\n";
8070
8071     if (!$full) {
8072         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8073         push @$newvollist, $newvolid;
8074     } else {
8075         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8076         my $storeid = $storage || $src_storeid;
8077
8078         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8079
8080         my $name = undef;
8081         my $size = undef;
8082         if (drive_is_cloudinit($drive)) {
8083             $name = "vm-$newvmid-cloudinit";
8084             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8085             if ($scfg->{path}) {
8086                 $name .= ".$dst_format";
8087             }
8088             $snapname = undef;
8089             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8090         } elsif ($dst_drivename eq 'efidisk0') {
8091             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8092         } elsif ($dst_drivename eq 'tpmstate0') {
8093             $dst_format = 'raw';
8094             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8095         } else {
8096             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8097
8098             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8099         }
8100         $newvolid = PVE::Storage::vdisk_alloc(
8101             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8102         );
8103         push @$newvollist, $newvolid;
8104
8105         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8106
8107         if (drive_is_cloudinit($drive)) {
8108             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8109             # if this is the case, we have to complete any block-jobs still there from
8110             # previous drive-mirrors
8111             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8112                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8113             }
8114             goto no_data_clone;
8115         }
8116
8117         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8118         if ($use_drive_mirror) {
8119             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8120                 $completion, $qga, $bwlimit);
8121         } else {
8122             if ($dst_drivename eq 'efidisk0') {
8123                 # the relevant data on the efidisk may be smaller than the source
8124                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8125                 # that is given by the OVMF_VARS.fd
8126                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8127                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8128
8129                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8130
8131                 # better for Ceph if block size is not too small, see bug #3324
8132                 my $bs = 1024*1024;
8133
8134                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8135
8136                 if ($src_format eq 'qcow2' && $snapname) {
8137                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8138                         if !min_version(kvm_user_version(), 6, 2);
8139                     push $cmd->@*, '-l', $snapname;
8140                 }
8141                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8142                 run_command($cmd);
8143             } else {
8144                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8145             }
8146         }
8147     }
8148
8149 no_data_clone:
8150     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8151
8152     my $disk = dclone($drive);
8153     delete $disk->{format};
8154     $disk->{file} = $newvolid;
8155     $disk->{size} = $size if defined($size);
8156
8157     return $disk;
8158 }
8159
8160 sub get_running_qemu_version {
8161     my ($vmid) = @_;
8162     my $res = mon_cmd($vmid, "query-version");
8163     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8164 }
8165
8166 sub qemu_use_old_bios_files {
8167     my ($machine_type) = @_;
8168
8169     return if !$machine_type;
8170
8171     my $use_old_bios_files = undef;
8172
8173     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8174         $machine_type = $1;
8175         $use_old_bios_files = 1;
8176     } else {
8177         my $version = extract_version($machine_type, kvm_user_version());
8178         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8179         # load new efi bios files on migration. So this hack is required to allow
8180         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8181         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8182         $use_old_bios_files = !min_version($version, 2, 4);
8183     }
8184
8185     return ($use_old_bios_files, $machine_type);
8186 }
8187
8188 sub get_efivars_size {
8189     my ($conf, $efidisk) = @_;
8190
8191     my $arch = get_vm_arch($conf);
8192     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8193     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8194     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8195     return -s $ovmf_vars;
8196 }
8197
8198 sub update_efidisk_size {
8199     my ($conf) = @_;
8200
8201     return if !defined($conf->{efidisk0});
8202
8203     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8204     $disk->{size} = get_efivars_size($conf);
8205     $conf->{efidisk0} = print_drive($disk);
8206
8207     return;
8208 }
8209
8210 sub update_tpmstate_size {
8211     my ($conf) = @_;
8212
8213     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8214     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8215     $conf->{tpmstate0} = print_drive($disk);
8216 }
8217
8218 sub create_efidisk($$$$$$$) {
8219     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8220
8221     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8222
8223     my $vars_size_b = -s $ovmf_vars;
8224     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8225     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8226     PVE::Storage::activate_volumes($storecfg, [$volid]);
8227
8228     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8229     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8230
8231     return ($volid, $size/1024);
8232 }
8233
8234 sub vm_iothreads_list {
8235     my ($vmid) = @_;
8236
8237     my $res = mon_cmd($vmid, 'query-iothreads');
8238
8239     my $iothreads = {};
8240     foreach my $iothread (@$res) {
8241         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8242     }
8243
8244     return $iothreads;
8245 }
8246
8247 sub scsihw_infos {
8248     my ($conf, $drive) = @_;
8249
8250     my $maxdev = 0;
8251
8252     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8253         $maxdev = 7;
8254     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8255         $maxdev = 1;
8256     } else {
8257         $maxdev = 256;
8258     }
8259
8260     my $controller = int($drive->{index} / $maxdev);
8261     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8262         ? "virtioscsi"
8263         : "scsihw";
8264
8265     return ($maxdev, $controller, $controller_prefix);
8266 }
8267
8268 sub resolve_dst_disk_format {
8269         my ($storecfg, $storeid, $src_volname, $format) = @_;
8270         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8271
8272         if (!$format) {
8273             # if no target format is specified, use the source disk format as hint
8274             if ($src_volname) {
8275                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8276                 $format = qemu_img_format($scfg, $src_volname);
8277             } else {
8278                 return $defFormat;
8279             }
8280         }
8281
8282         # test if requested format is supported - else use default
8283         my $supported = grep { $_ eq $format } @$validFormats;
8284         $format = $defFormat if !$supported;
8285         return $format;
8286 }
8287
8288 # NOTE: if this logic changes, please update docs & possibly gui logic
8289 sub find_vmstate_storage {
8290     my ($conf, $storecfg) = @_;
8291
8292     # first, return storage from conf if set
8293     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8294
8295     my ($target, $shared, $local);
8296
8297     foreach_storage_used_by_vm($conf, sub {
8298         my ($sid) = @_;
8299         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8300         my $dst = $scfg->{shared} ? \$shared : \$local;
8301         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8302     });
8303
8304     # second, use shared storage where VM has at least one disk
8305     # third, use local storage where VM has at least one disk
8306     # fall back to local storage
8307     $target = $shared // $local // 'local';
8308
8309     return $target;
8310 }
8311
8312 sub generate_uuid {
8313     my ($uuid, $uuid_str);
8314     UUID::generate($uuid);
8315     UUID::unparse($uuid, $uuid_str);
8316     return $uuid_str;
8317 }
8318
8319 sub generate_smbios1_uuid {
8320     return "uuid=".generate_uuid();
8321 }
8322
8323 sub nbd_stop {
8324     my ($vmid) = @_;
8325
8326     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8327 }
8328
8329 sub create_reboot_request {
8330     my ($vmid) = @_;
8331     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8332         or die "failed to create reboot trigger file: $!\n";
8333     close($fh);
8334 }
8335
8336 sub clear_reboot_request {
8337     my ($vmid) = @_;
8338     my $path = "/run/qemu-server/$vmid.reboot";
8339     my $res = 0;
8340
8341     $res = unlink($path);
8342     die "could not remove reboot request for $vmid: $!"
8343         if !$res && $! != POSIX::ENOENT;
8344
8345     return $res;
8346 }
8347
8348 sub bootorder_from_legacy {
8349     my ($conf, $bootcfg) = @_;
8350
8351     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8352     my $bootindex_hash = {};
8353     my $i = 1;
8354     foreach my $o (split(//, $boot)) {
8355         $bootindex_hash->{$o} = $i*100;
8356         $i++;
8357     }
8358
8359     my $bootorder = {};
8360
8361     PVE::QemuConfig->foreach_volume($conf, sub {
8362         my ($ds, $drive) = @_;
8363
8364         if (drive_is_cdrom ($drive, 1)) {
8365             if ($bootindex_hash->{d}) {
8366                 $bootorder->{$ds} = $bootindex_hash->{d};
8367                 $bootindex_hash->{d} += 1;
8368             }
8369         } elsif ($bootindex_hash->{c}) {
8370             $bootorder->{$ds} = $bootindex_hash->{c}
8371                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8372             $bootindex_hash->{c} += 1;
8373         }
8374     });
8375
8376     if ($bootindex_hash->{n}) {
8377         for (my $i = 0; $i < $MAX_NETS; $i++) {
8378             my $netname = "net$i";
8379             next if !$conf->{$netname};
8380             $bootorder->{$netname} = $bootindex_hash->{n};
8381             $bootindex_hash->{n} += 1;
8382         }
8383     }
8384
8385     return $bootorder;
8386 }
8387
8388 # Generate default device list for 'boot: order=' property. Matches legacy
8389 # default boot order, but with explicit device names. This is important, since
8390 # the fallback for when neither 'order' nor the old format is specified relies
8391 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8392 sub get_default_bootdevices {
8393     my ($conf) = @_;
8394
8395     my @ret = ();
8396
8397     # harddisk
8398     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8399     push @ret, $first if $first;
8400
8401     # cdrom
8402     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8403     push @ret, $first if $first;
8404
8405     # network
8406     for (my $i = 0; $i < $MAX_NETS; $i++) {
8407         my $netname = "net$i";
8408         next if !$conf->{$netname};
8409         push @ret, $netname;
8410         last;
8411     }
8412
8413     return \@ret;
8414 }
8415
8416 sub device_bootorder {
8417     my ($conf) = @_;
8418
8419     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8420
8421     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8422
8423     my $bootorder = {};
8424     if (!defined($boot) || $boot->{legacy}) {
8425         $bootorder = bootorder_from_legacy($conf, $boot);
8426     } elsif ($boot->{order}) {
8427         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8428         for my $dev (PVE::Tools::split_list($boot->{order})) {
8429             $bootorder->{$dev} = $i++;
8430         }
8431     }
8432
8433     return $bootorder;
8434 }
8435
8436 sub register_qmeventd_handle {
8437     my ($vmid) = @_;
8438
8439     my $fh;
8440     my $peer = "/var/run/qmeventd.sock";
8441     my $count = 0;
8442
8443     for (;;) {
8444         $count++;
8445         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8446         last if $fh;
8447         if ($! != EINTR && $! != EAGAIN) {
8448             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8449         }
8450         if ($count > 4) {
8451             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8452               . "after $count retries\n";
8453         }
8454         usleep(25000);
8455     }
8456
8457     # send handshake to mark VM as backing up
8458     print $fh to_json({vzdump => {vmid => "$vmid"}});
8459
8460     # return handle to be closed later when inhibit is no longer required
8461     return $fh;
8462 }
8463
8464 # bash completion helper
8465
8466 sub complete_backup_archives {
8467     my ($cmdname, $pname, $cvalue) = @_;
8468
8469     my $cfg = PVE::Storage::config();
8470
8471     my $storeid;
8472
8473     if ($cvalue =~ m/^([^:]+):/) {
8474         $storeid = $1;
8475     }
8476
8477     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8478
8479     my $res = [];
8480     foreach my $id (keys %$data) {
8481         foreach my $item (@{$data->{$id}}) {
8482             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8483             push @$res, $item->{volid} if defined($item->{volid});
8484         }
8485     }
8486
8487     return $res;
8488 }
8489
8490 my $complete_vmid_full = sub {
8491     my ($running) = @_;
8492
8493     my $idlist = vmstatus();
8494
8495     my $res = [];
8496
8497     foreach my $id (keys %$idlist) {
8498         my $d = $idlist->{$id};
8499         if (defined($running)) {
8500             next if $d->{template};
8501             next if $running && $d->{status} ne 'running';
8502             next if !$running && $d->{status} eq 'running';
8503         }
8504         push @$res, $id;
8505
8506     }
8507     return $res;
8508 };
8509
8510 sub complete_vmid {
8511     return &$complete_vmid_full();
8512 }
8513
8514 sub complete_vmid_stopped {
8515     return &$complete_vmid_full(0);
8516 }
8517
8518 sub complete_vmid_running {
8519     return &$complete_vmid_full(1);
8520 }
8521
8522 sub complete_storage {
8523
8524     my $cfg = PVE::Storage::config();
8525     my $ids = $cfg->{ids};
8526
8527     my $res = [];
8528     foreach my $sid (keys %$ids) {
8529         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8530         next if !$ids->{$sid}->{content}->{images};
8531         push @$res, $sid;
8532     }
8533
8534     return $res;
8535 }
8536
8537 sub complete_migration_storage {
8538     my ($cmd, $param, $current_value, $all_args) = @_;
8539
8540     my $targetnode = @$all_args[1];
8541
8542     my $cfg = PVE::Storage::config();
8543     my $ids = $cfg->{ids};
8544
8545     my $res = [];
8546     foreach my $sid (keys %$ids) {
8547         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8548         next if !$ids->{$sid}->{content}->{images};
8549         push @$res, $sid;
8550     }
8551
8552     return $res;
8553 }
8554
8555 sub vm_is_paused {
8556     my ($vmid, $include_suspended) = @_;
8557     my $qmpstatus = eval {
8558         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8559         mon_cmd($vmid, "query-status");
8560     };
8561     warn "$@\n" if $@;
8562     return $qmpstatus && (
8563         $qmpstatus->{status} eq "paused" ||
8564         $qmpstatus->{status} eq "prelaunch" ||
8565         ($include_suspended && $qmpstatus->{status} eq "suspended")
8566     );
8567 }
8568
8569 sub check_volume_storage_type {
8570     my ($storecfg, $vol) = @_;
8571
8572     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8573     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8574     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8575
8576     die "storage '$storeid' does not support content-type '$vtype'\n"
8577         if !$scfg->{content}->{$vtype};
8578
8579     return 1;
8580 }
8581
8582 sub add_nets_bridge_fdb {
8583     my ($conf, $vmid) = @_;
8584
8585     for my $opt (keys %$conf) {
8586         next if $opt !~ m/^net(\d+)$/;
8587         my $iface = "tap${vmid}i$1";
8588         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8589         my $net = parse_net($conf->{$opt}, 1) or next;
8590
8591         my $mac = $net->{macaddr};
8592         if (!$mac) {
8593             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8594                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8595             next;
8596         }
8597
8598         my $bridge = $net->{bridge};
8599         if (!$bridge) {
8600             log_warn("Interface '$iface' not attached to any bridge.");
8601             next;
8602         }
8603         if ($have_sdn) {
8604             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8605         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8606             PVE::Network::add_bridge_fdb($iface, $mac);
8607         }
8608     }
8609 }
8610
8611 sub del_nets_bridge_fdb {
8612     my ($conf, $vmid) = @_;
8613
8614     for my $opt (keys %$conf) {
8615         next if $opt !~ m/^net(\d+)$/;
8616         my $iface = "tap${vmid}i$1";
8617
8618         my $net = parse_net($conf->{$opt}) or next;
8619         my $mac = $net->{macaddr} or next;
8620
8621         my $bridge = $net->{bridge};
8622         if ($have_sdn) {
8623             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8624         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8625             PVE::Network::del_bridge_fdb($iface, $mac);
8626         }
8627     }
8628 }
8629
8630 sub create_ifaces_ipams_ips {
8631     my ($conf, $vmid) = @_;
8632
8633     return if !$have_sdn;
8634
8635     foreach my $opt (keys %$conf) {
8636         if ($opt =~ m/^net(\d+)$/) {
8637             my $value = $conf->{$opt};
8638             my $net = PVE::QemuServer::parse_net($value);
8639             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8640             warn $@ if $@;
8641         }
8642     }
8643 }
8644
8645 sub delete_ifaces_ipams_ips {
8646     my ($conf, $vmid) = @_;
8647
8648     return if !$have_sdn;
8649
8650     foreach my $opt (keys %$conf) {
8651         if ($opt =~ m/^net(\d+)$/) {
8652             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8653             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8654             warn $@ if $@;
8655         }
8656     }
8657 }
8658
8659 1;