PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::HA::Config;
  38 use PVE::Mapping::PCI;
  39 use PVE::Mapping::USB;
  40 use PVE::INotify;
  41 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  42 use PVE::ProcFSTools;
  43 use PVE::PBSClient;
  44 use PVE::RESTEnvironment qw(log_warn);
  45 use PVE::RPCEnvironment;
  46 use PVE::Storage;
  47 use PVE::SysFSTools;
  48 use PVE::Systemd;
  49 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  50
  51 use PVE::QMPClient;
  52 use PVE::QemuConfig;
  53 use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
  54 use PVE::QemuServer::Cloudinit;
  55 use PVE::QemuServer::CGroup;
  56 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
  57 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  58 use PVE::QemuServer::Machine;
  59 use PVE::QemuServer::Memory qw(get_current_memory);
  60 use PVE::QemuServer::Monitor qw(mon_cmd);
  61 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  62 use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
  63 use PVE::QemuServer::USB;
  64
  65 my $have_sdn;
  66 eval {
  67     require PVE::Network::SDN::Zones;
  68     require PVE::Network::SDN::Vnets;
  69     $have_sdn = 1;
  70 };
  71
  72 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  73 my $OVMF = {
  74     x86_64 => {
  75         '4m-no-smm' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-no-smm-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         '4m' => [
  84             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  86         ],
  87         '4m-ms' => [
  88             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  89             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  90         ],
  91         # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
  92         # anymore. how can we deperacate this sanely without breaking existing instances, or using
  93         # older backups and snapshot?
  94         default => [
  95             "$EDK2_FW_BASE/OVMF_CODE.fd",
  96             "$EDK2_FW_BASE/OVMF_VARS.fd",
  97         ],
  98     },
  99     aarch64 => {
 100         default => [
 101             "$EDK2_FW_BASE/AAVMF_CODE.fd",
 102             "$EDK2_FW_BASE/AAVMF_VARS.fd",
 103         ],
 104     },
 105 };
 106
 107 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 108
 109 # Note about locking: we use flock on the config file protect against concurent actions.
 110 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 111 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 112 # But you can ignore this kind of lock with the --skiplock flag.
 113
 114 cfs_register_file(
 115     '/qemu-server/',
 116     \&parse_vm_config,
 117     \&write_vm_config
 118 );
 119
 120 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 121     description => "Some command save/restore state from this location.",
 122     type => 'string',
 123     maxLength => 128,
 124     optional => 1,
 125 });
 126
 127 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 128         description => "Specifies the QEMU machine type.",
 129         type => 'string',
 130         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 131         maxLength => 40,
 132         optional => 1,
 133 });
 134
 135 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 136 my $nodename_cache;
 137 sub nodename {
 138     $nodename_cache //= PVE::INotify::nodename();
 139     return $nodename_cache;
 140 }
 141
 142 my $watchdog_fmt = {
 143     model => {
 144         default_key => 1,
 145         type => 'string',
 146         enum => [qw(i6300esb ib700)],
 147         description => "Watchdog type to emulate.",
 148         default => 'i6300esb',
 149         optional => 1,
 150     },
 151     action => {
 152         type => 'string',
 153         enum => [qw(reset shutdown poweroff pause debug none)],
 154         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 155         optional => 1,
 156     },
 157 };
 158 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 159
 160 my $agent_fmt = {
 161     enabled => {
 162         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 163         type => 'boolean',
 164         default => 0,
 165         default_key => 1,
 166     },
 167     fstrim_cloned_disks => {
 168         description => "Run fstrim after moving a disk or migrating the VM.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 0,
 172     },
 173     'freeze-fs-on-backup' => {
 174         description => "Freeze/thaw guest filesystems on backup for consistency.",
 175         type => 'boolean',
 176         optional => 1,
 177         default => 1,
 178     },
 179     type => {
 180         description => "Select the agent type",
 181         type => 'string',
 182         default => 'virtio',
 183         optional => 1,
 184         enum => [qw(virtio isa)],
 185     },
 186 };
 187
 188 my $vga_fmt = {
 189     type => {
 190         description => "Select the VGA type.",
 191         type => 'string',
 192         default => 'std',
 193         optional => 1,
 194         default_key => 1,
 195         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 196     },
 197     memory => {
 198         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 199         type => 'integer',
 200         optional => 1,
 201         minimum => 4,
 202         maximum => 512,
 203     },
 204     clipboard => {
 205         description => 'Enable a specific clipboard. If not set, depending on the display type the'
 206             .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
 207         type => 'string',
 208         enum => ['vnc'],
 209         optional => 1,
 210     },
 211 };
 212
 213 my $ivshmem_fmt = {
 214     size => {
 215         type => 'integer',
 216         minimum => 1,
 217         description => "The size of the file in MB.",
 218     },
 219     name => {
 220         type => 'string',
 221         pattern => '[a-zA-Z0-9\-]+',
 222         optional => 1,
 223         format_description => 'string',
 224         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 225     },
 226 };
 227
 228 my $audio_fmt = {
 229     device => {
 230         type => 'string',
 231         enum => [qw(ich9-intel-hda intel-hda AC97)],
 232         description =>  "Configure an audio device."
 233     },
 234     driver =>  {
 235         type => 'string',
 236         enum => ['spice', 'none'],
 237         default => 'spice',
 238         optional => 1,
 239         description => "Driver backend for the audio device."
 240     },
 241 };
 242
 243 my $spice_enhancements_fmt = {
 244     foldersharing => {
 245         type => 'boolean',
 246         optional => 1,
 247         default => '0',
 248         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 249     },
 250     videostreaming =>  {
 251         type => 'string',
 252         enum => ['off', 'all', 'filter'],
 253         default => 'off',
 254         optional => 1,
 255         description => "Enable video streaming. Uses compression for detected video streams."
 256     },
 257 };
 258
 259 my $rng_fmt = {
 260     source => {
 261         type => 'string',
 262         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 263         default_key => 1,
 264         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 265             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 266             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 267             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 268             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 269             ." a hardware RNG from the host.",
 270     },
 271     max_bytes => {
 272         type => 'integer',
 273         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 274             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 275             ." `0` to disable limiting (potentially dangerous!).",
 276         optional => 1,
 277
 278         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 279         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 280         # reading from /dev/urandom
 281         default => 1024,
 282     },
 283     period => {
 284         type => 'integer',
 285         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 286             ." the guest to retrieve another 'max_bytes' of entropy.",
 287         optional => 1,
 288         default => 1000,
 289     },
 290 };
 291
 292 my $meta_info_fmt = {
 293     'ctime' => {
 294         type => 'integer',
 295         description => "The guest creation timestamp as UNIX epoch time",
 296         minimum => 0,
 297         optional => 1,
 298     },
 299     'creation-qemu' => {
 300         type => 'string',
 301         description => "The QEMU (machine) version from the time this VM was created.",
 302         pattern => '\d+(\.\d+)+',
 303         optional => 1,
 304     },
 305 };
 306
 307 my $confdesc = {
 308     onboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Specifies whether a VM will be started during system bootup.",
 312         default => 0,
 313     },
 314     autostart => {
 315         optional => 1,
 316         type => 'boolean',
 317         description => "Automatic restart after crash (currently ignored).",
 318         default => 0,
 319     },
 320     hotplug => {
 321         optional => 1,
 322         type => 'string', format => 'pve-hotplug-features',
 323         description => "Selectively enable hotplug features. This is a comma separated list of"
 324             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 325             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 326             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 327             ." windows > 7.",
 328         default => 'network,disk,usb',
 329     },
 330     reboot => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 334         default => 1,
 335     },
 336     lock => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Lock/unlock the VM.",
 340         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 341     },
 342     cpulimit => {
 343         optional => 1,
 344         type => 'number',
 345         description => "Limit of CPU usage.",
 346         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 347             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 348         minimum => 0,
 349         maximum => 128,
 350         default => 0,
 351     },
 352     cpuunits => {
 353         optional => 1,
 354         type => 'integer',
 355         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 356         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 357             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 358             ." weights of all the other running VMs.",
 359         minimum => 1,
 360         maximum => 262144,
 361         default => 'cgroup v1: 1024, cgroup v2: 100',
 362     },
 363     memory => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Memory properties.",
 367         format => $PVE::QemuServer::Memory::memory_fmt
 368     },
 369     balloon => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 373         minimum => 0,
 374     },
 375     shares => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 379             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 380             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 381         minimum => 0,
 382         maximum => 50000,
 383         default => 1000,
 384     },
 385     keyboard => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Keyboard layout for VNC server. This option is generally not required and"
 389         ." is often better handled from within the guest OS.",
 390         enum => PVE::Tools::kvmkeymaplist(),
 391         default => undef,
 392     },
 393     name => {
 394         optional => 1,
 395         type => 'string', format => 'dns-name',
 396         description => "Set a name for the VM. Only used on the configuration web interface.",
 397     },
 398     scsihw => {
 399         optional => 1,
 400         type => 'string',
 401         description => "SCSI controller model",
 402         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 403         default => 'lsi',
 404     },
 405     description => {
 406         optional => 1,
 407         type => 'string',
 408         description => "Description for the VM. Shown in the web-interface VM's summary."
 409             ." This is saved as comment inside the configuration file.",
 410         maxLength => 1024 * 8,
 411     },
 412     ostype => {
 413         optional => 1,
 414         type => 'string',
 415         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 416         description => "Specify guest operating system.",
 417         verbose_description => <<EODESC,
 418 Specify guest operating system. This is used to enable special
 419 optimization/features for specific operating systems:
 420
 421 [horizontal]
 422 other;; unspecified OS
 423 wxp;; Microsoft Windows XP
 424 w2k;; Microsoft Windows 2000
 425 w2k3;; Microsoft Windows 2003
 426 w2k8;; Microsoft Windows 2008
 427 wvista;; Microsoft Windows Vista
 428 win7;; Microsoft Windows 7
 429 win8;; Microsoft Windows 8/2012/2012r2
 430 win10;; Microsoft Windows 10/2016/2019
 431 win11;; Microsoft Windows 11/2022
 432 l24;; Linux 2.4 Kernel
 433 l26;; Linux 2.6 - 6.X Kernel
 434 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 435 EODESC
 436     },
 437     boot => {
 438         optional => 1,
 439         type => 'string', format => 'pve-qm-boot',
 440         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 441             ." key or 'legacy=' is deprecated.",
 442     },
 443     bootdisk => {
 444         optional => 1,
 445         type => 'string', format => 'pve-qm-bootdisk',
 446         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 447         pattern => '(ide|sata|scsi|virtio)\d+',
 448     },
 449     smp => {
 450         optional => 1,
 451         type => 'integer',
 452         description => "The number of CPUs. Please use option -sockets instead.",
 453         minimum => 1,
 454         default => 1,
 455     },
 456     sockets => {
 457         optional => 1,
 458         type => 'integer',
 459         description => "The number of CPU sockets.",
 460         minimum => 1,
 461         default => 1,
 462     },
 463     cores => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "The number of cores per socket.",
 467         minimum => 1,
 468         default => 1,
 469     },
 470     numa => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable NUMA.",
 474         default => 0,
 475     },
 476     hugepages => {
 477         optional => 1,
 478         type => 'string',
 479         description => "Enable/disable hugepages memory.",
 480         enum => [qw(any 2 1024)],
 481     },
 482     keephugepages => {
 483         optional => 1,
 484         type => 'boolean',
 485         default => 0,
 486         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 487             ." after VM shutdown and can be used for subsequent starts.",
 488     },
 489     vcpus => {
 490         optional => 1,
 491         type => 'integer',
 492         description => "Number of hotplugged vcpus.",
 493         minimum => 1,
 494         default => 0,
 495     },
 496     acpi => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable ACPI.",
 500         default => 1,
 501     },
 502     agent => {
 503         optional => 1,
 504         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 505         type => 'string',
 506         format => $agent_fmt,
 507     },
 508     kvm => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Enable/disable KVM hardware virtualization.",
 512         default => 1,
 513     },
 514     tdf => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Enable/disable time drift fix.",
 518         default => 0,
 519     },
 520     localtime => {
 521         optional => 1,
 522         type => 'boolean',
 523         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 524             ." the `ostype` indicates a Microsoft Windows OS.",
 525     },
 526     freeze => {
 527         optional => 1,
 528         type => 'boolean',
 529         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 530     },
 531     vga => {
 532         optional => 1,
 533         type => 'string', format => $vga_fmt,
 534         description => "Configure the VGA hardware.",
 535         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 536             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 537             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 538             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 539             ." display server. For win* OS you can select how many independent displays you want,"
 540             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 541             ." using a serial device as terminal.",
 542     },
 543     watchdog => {
 544         optional => 1,
 545         type => 'string', format => 'pve-qm-watchdog',
 546         description => "Create a virtual hardware watchdog device.",
 547         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 548             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 549             ." else the watchdog will reset the guest (or execute the respective action specified)",
 550     },
 551     startdate => {
 552         optional => 1,
 553         type => 'string',
 554         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 555         description => "Set the initial date of the real time clock. Valid format for date are:"
 556             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 557         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 558         default => 'now',
 559     },
 560     startup =>  get_standard_option('pve-startup-order'),
 561     template => {
 562         optional => 1,
 563         type => 'boolean',
 564         description => "Enable/disable Template.",
 565         default => 0,
 566     },
 567     args => {
 568         optional => 1,
 569         type => 'string',
 570         description => "Arbitrary arguments passed to kvm.",
 571         verbose_description => <<EODESCR,
 572 Arbitrary arguments passed to kvm, for example:
 573
 574 args: -no-reboot -smbios 'type=0,vendor=FOO'
 575
 576 NOTE: this option is for experts only.
 577 EODESCR
 578     },
 579     tablet => {
 580         optional => 1,
 581         type => 'boolean',
 582         default => 1,
 583         description => "Enable/disable the USB tablet device.",
 584         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 585             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 586             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 587             ." may consider disabling this to save some context switches. This is turned off by"
 588             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 589     },
 590     migrate_speed => {
 591         optional => 1,
 592         type => 'integer',
 593         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 594         minimum => 0,
 595         default => 0,
 596     },
 597     migrate_downtime => {
 598         optional => 1,
 599         type => 'number',
 600         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 601         minimum => 0,
 602         default => 0.1,
 603     },
 604     cdrom => {
 605         optional => 1,
 606         type => 'string', format => 'pve-qm-ide',
 607         typetext => '<volume>',
 608         description => "This is an alias for option -ide2",
 609     },
 610     cpu => {
 611         optional => 1,
 612         description => "Emulated CPU type.",
 613         type => 'string',
 614         format => 'pve-vm-cpu-conf',
 615     },
 616     parent => get_standard_option('pve-snapshot-name', {
 617         optional => 1,
 618         description => "Parent snapshot name. This is used internally, and should not be modified.",
 619     }),
 620     snaptime => {
 621         optional => 1,
 622         description => "Timestamp for snapshots.",
 623         type => 'integer',
 624         minimum => 0,
 625     },
 626     vmstate => {
 627         optional => 1,
 628         type => 'string', format => 'pve-volume-id',
 629         description => "Reference to a volume which stores the VM state. This is used internally"
 630             ." for snapshots.",
 631     },
 632     vmstatestorage => get_standard_option('pve-storage-id', {
 633         description => "Default storage for VM state volumes/files.",
 634         optional => 1,
 635     }),
 636     runningmachine => get_standard_option('pve-qemu-machine', {
 637         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 638             ." for snapshots.",
 639     }),
 640     runningcpu => {
 641         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 642             ." internally for snapshots.",
 643         optional => 1,
 644         type => 'string',
 645         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 646         format_description => 'QEMU -cpu parameter'
 647     },
 648     machine => get_standard_option('pve-qemu-machine'),
 649     arch => {
 650         description => "Virtual processor architecture. Defaults to the host.",
 651         optional => 1,
 652         type => 'string',
 653         enum => [qw(x86_64 aarch64)],
 654     },
 655     smbios1 => {
 656         description => "Specify SMBIOS type 1 fields.",
 657         type => 'string', format => 'pve-qm-smbios1',
 658         maxLength => 512,
 659         optional => 1,
 660     },
 661     protection => {
 662         optional => 1,
 663         type => 'boolean',
 664         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 665             ." remove disk operations.",
 666         default => 0,
 667     },
 668     bios => {
 669         optional => 1,
 670         type => 'string',
 671         enum => [ qw(seabios ovmf) ],
 672         description => "Select BIOS implementation.",
 673         default => 'seabios',
 674     },
 675     vmgenid => {
 676         type => 'string',
 677         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 678         format_description => 'UUID',
 679         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 680             ." to disable explicitly.",
 681         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 682             ." value identifier to the guest OS. This allows to notify the guest operating system"
 683             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 684             ." execution or creation from a template). The guest operating system notices the"
 685             ." change, and is then able to react as appropriate by marking its copies of"
 686             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 687             ."Note that auto-creation only works when done through API/CLI create or update methods"
 688             .", but not when manually editing the config file.",
 689         default => "1 (autogenerated)",
 690         optional => 1,
 691     },
 692     hookscript => {
 693         type => 'string',
 694         format => 'pve-volume-id',
 695         optional => 1,
 696         description => "Script that will be executed during various steps in the vms lifetime.",
 697     },
 698     ivshmem => {
 699         type => 'string',
 700         format => $ivshmem_fmt,
 701         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 702             ." the host.",
 703         optional => 1,
 704     },
 705     audio0 => {
 706         type => 'string',
 707         format => $audio_fmt,
 708         description => "Configure a audio device, useful in combination with QXL/Spice.",
 709         optional => 1
 710     },
 711     spice_enhancements => {
 712         type => 'string',
 713         format => $spice_enhancements_fmt,
 714         description => "Configure additional enhancements for SPICE.",
 715         optional => 1
 716     },
 717     tags => {
 718         type => 'string', format => 'pve-tag-list',
 719         description => 'Tags of the VM. This is only meta information.',
 720         optional => 1,
 721     },
 722     rng0 => {
 723         type => 'string',
 724         format => $rng_fmt,
 725         description => "Configure a VirtIO-based Random Number Generator.",
 726         optional => 1,
 727     },
 728     meta => {
 729         type => 'string',
 730         format => $meta_info_fmt,
 731         description => "Some (read-only) meta-information about this guest.",
 732         optional => 1,
 733     },
 734     affinity => {
 735         type => 'string', format => 'pve-cpuset',
 736         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 737         optional => 1,
 738     },
 739 };
 740
 741 my $cicustom_fmt = {
 742     meta => {
 743         type => 'string',
 744         optional => 1,
 745         description => 'Specify a custom file containing all meta data passed to the VM via"
 746             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 747         format => 'pve-volume-id',
 748         format_description => 'volume',
 749     },
 750     network => {
 751         type => 'string',
 752         optional => 1,
 753         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 754         format => 'pve-volume-id',
 755         format_description => 'volume',
 756     },
 757     user => {
 758         type => 'string',
 759         optional => 1,
 760         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 761         format => 'pve-volume-id',
 762         format_description => 'volume',
 763     },
 764     vendor => {
 765         type => 'string',
 766         optional => 1,
 767         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 768         format => 'pve-volume-id',
 769         format_description => 'volume',
 770     },
 771 };
 772 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 773
 774 # any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
 775 my $confdesc_cloudinit = {
 776     citype => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'Specifies the cloud-init configuration format. The default depends on the'
 780             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 781             .' and `configdrive2` for windows.',
 782         enum => ['configdrive2', 'nocloud', 'opennebula'],
 783     },
 784     ciuser => {
 785         optional => 1,
 786         type => 'string',
 787         description => "cloud-init: User name to change ssh keys and password for instead of the"
 788             ." image's configured default user.",
 789     },
 790     cipassword => {
 791         optional => 1,
 792         type => 'string',
 793         description => 'cloud-init: Password to assign the user. Using this is generally not'
 794             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 795             .' support hashed passwords.',
 796     },
 797     ciupgrade => {
 798         optional => 1,
 799         type => 'boolean',
 800         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 801         default => 1,
 802     },
 803     cicustom => {
 804         optional => 1,
 805         type => 'string',
 806         description => 'cloud-init: Specify custom files to replace the automatically generated'
 807             .' ones at start.',
 808         format => 'pve-qm-cicustom',
 809     },
 810     searchdomain => {
 811         optional => 1,
 812         type => 'string',
 813         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 814             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 815             .' are set.',
 816     },
 817     nameserver => {
 818         optional => 1,
 819         type => 'string', format => 'address-list',
 820         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 821             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 822             .' are set.',
 823     },
 824     sshkeys => {
 825         optional => 1,
 826         type => 'string',
 827         format => 'urlencoded',
 828         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 829     },
 830 };
 831
 832 # what about other qemu settings ?
 833 #cpu => 'string',
 834 #machine => 'string',
 835 #fda => 'file',
 836 #fdb => 'file',
 837 #mtdblock => 'file',
 838 #sd => 'file',
 839 #pflash => 'file',
 840 #snapshot => 'bool',
 841 #bootp => 'file',
 842 ##tftp => 'dir',
 843 ##smb => 'dir',
 844 #kernel => 'file',
 845 #append => 'string',
 846 #initrd => 'file',
 847 ##soundhw => 'string',
 848
 849 while (my ($k, $v) = each %$confdesc) {
 850     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 851 }
 852
 853 my $MAX_NETS = 32;
 854 my $MAX_SERIAL_PORTS = 4;
 855 my $MAX_PARALLEL_PORTS = 3;
 856
 857 for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++)  {
 858     $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
 859 }
 860
 861 my $nic_model_list = [
 862     'e1000',
 863     'e1000-82540em',
 864     'e1000-82544gc',
 865     'e1000-82545em',
 866     'e1000e',
 867     'i82551',
 868     'i82557b',
 869     'i82559er',
 870     'ne2k_isa',
 871     'ne2k_pci',
 872     'pcnet',
 873     'rtl8139',
 874     'virtio',
 875     'vmxnet3',
 876 ];
 877 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 878
 879 my $net_fmt_bridge_descr = <<__EOD__;
 880 Bridge to attach the network device to. The Proxmox VE standard bridge
 881 is called 'vmbr0'.
 882
 883 If you do not specify a bridge, we create a kvm user (NATed) network
 884 device, which provides DHCP and DNS services. The following addresses
 885 are used:
 886
 887  10.0.2.2   Gateway
 888  10.0.2.3   DNS Server
 889  10.0.2.4   SMB Server
 890
 891 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 892 __EOD__
 893
 894 my $net_fmt = {
 895     macaddr  => get_standard_option('mac-addr', {
 896         description => "MAC address. That address must be unique withing your network. This is"
 897             ." automatically generated if not specified.",
 898     }),
 899     model => {
 900         type => 'string',
 901         description => "Network Card Model. The 'virtio' model provides the best performance with"
 902             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 903             ." best to use 'e1000'.",
 904         enum => $nic_model_list,
 905         default_key => 1,
 906     },
 907     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 908     bridge => get_standard_option('pve-bridge-id', {
 909         description => $net_fmt_bridge_descr,
 910         optional => 1,
 911     }),
 912     queues => {
 913         type => 'integer',
 914         minimum => 0, maximum => 64,
 915         description => 'Number of packet queues to be used on the device.',
 916         optional => 1,
 917     },
 918     rate => {
 919         type => 'number',
 920         minimum => 0,
 921         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 922         optional => 1,
 923     },
 924     tag => {
 925         type => 'integer',
 926         minimum => 1, maximum => 4094,
 927         description => 'VLAN tag to apply to packets on this interface.',
 928         optional => 1,
 929     },
 930     trunks => {
 931         type => 'string',
 932         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 933         description => 'VLAN trunks to pass through this interface.',
 934         format_description => 'vlanid[;vlanid...]',
 935         optional => 1,
 936     },
 937     firewall => {
 938         type => 'boolean',
 939         description => 'Whether this interface should be protected by the firewall.',
 940         optional => 1,
 941     },
 942     link_down => {
 943         type => 'boolean',
 944         description => 'Whether this interface should be disconnected (like pulling the plug).',
 945         optional => 1,
 946     },
 947     mtu => {
 948         type => 'integer',
 949         minimum => 1, maximum => 65520,
 950         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 951         optional => 1,
 952     },
 953 };
 954
 955 my $netdesc = {
 956     optional => 1,
 957     type => 'string', format => $net_fmt,
 958     description => "Specify network devices.",
 959 };
 960
 961 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 962
 963 my $ipconfig_fmt = {
 964     ip => {
 965         type => 'string',
 966         format => 'pve-ipv4-config',
 967         format_description => 'IPv4Format/CIDR',
 968         description => 'IPv4 address in CIDR format.',
 969         optional => 1,
 970         default => 'dhcp',
 971     },
 972     gw => {
 973         type => 'string',
 974         format => 'ipv4',
 975         format_description => 'GatewayIPv4',
 976         description => 'Default gateway for IPv4 traffic.',
 977         optional => 1,
 978         requires => 'ip',
 979     },
 980     ip6 => {
 981         type => 'string',
 982         format => 'pve-ipv6-config',
 983         format_description => 'IPv6Format/CIDR',
 984         description => 'IPv6 address in CIDR format.',
 985         optional => 1,
 986         default => 'dhcp',
 987     },
 988     gw6 => {
 989         type => 'string',
 990         format => 'ipv6',
 991         format_description => 'GatewayIPv6',
 992         description => 'Default gateway for IPv6 traffic.',
 993         optional => 1,
 994         requires => 'ip6',
 995     },
 996 };
 997 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 998 my $ipconfigdesc = {
 999     optional => 1,
1000     type => 'string', format => 'pve-qm-ipconfig',
1001     description => <<'EODESCR',
1002 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1003
1004 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1005
1006 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1007 gateway should be provided.
1008 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1009 cloud-init 19.4 or newer.
1010
1011 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1012 dhcp on IPv4.
1013 EODESCR
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1016
1017 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1018     $confdesc->{"net$i"} = $netdesc;
1019     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1020 }
1021
1022 foreach my $key (keys %$confdesc_cloudinit) {
1023     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1024 }
1025
1026 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1027 sub pve_verify_cpuset {
1028     my ($set_text, $noerr) = @_;
1029
1030     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1031
1032     if ($@) {
1033         return if $noerr;
1034         die "unable to parse cpuset option\n";
1035     }
1036
1037     return PVE::CpuSet->new($members)->short_string();
1038 }
1039
1040 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1041 sub verify_volume_id_or_qm_path {
1042     my ($volid, $noerr) = @_;
1043
1044     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1045
1046     return verify_volume_id_or_absolute_path($volid, $noerr);
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1050 sub verify_volume_id_or_absolute_path {
1051     my ($volid, $noerr) = @_;
1052
1053     return $volid if $volid =~ m|^/|;
1054
1055     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1056     if ($@) {
1057         return if $noerr;
1058         die $@;
1059     }
1060     return $volid;
1061 }
1062
1063 my $serialdesc = {
1064         optional => 1,
1065         type => 'string',
1066         pattern => '(/dev/.+|socket)',
1067         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1068         verbose_description =>  <<EODESCR,
1069 Create a serial device inside the VM (n is 0 to 3), and pass through a
1070 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1071 host side (use 'qm terminal' to open a terminal connection).
1072
1073 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1074 use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 my $paralleldesc= {
1081         optional => 1,
1082         type => 'string',
1083         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1084         description =>  "Map host parallel devices (n is 0 to 2).",
1085         verbose_description =>  <<EODESCR,
1086 Map host parallel devices (n is 0 to 2).
1087
1088 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1089 machines - use with special care.
1090
1091 CAUTION: Experimental! User reported problems with this option.
1092 EODESCR
1093 };
1094
1095 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1096     $confdesc->{"parallel$i"} = $paralleldesc;
1097 }
1098
1099 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1100     $confdesc->{"serial$i"} = $serialdesc;
1101 }
1102
1103 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1104     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1105 }
1106
1107 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1108     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1109 }
1110
1111 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1112     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1113 }
1114
1115 my $boot_fmt = {
1116     legacy => {
1117         optional => 1,
1118         default_key => 1,
1119         type => 'string',
1120         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1121                      . " Deprecated, use 'order=' instead.",
1122         pattern => '[acdn]{1,4}',
1123         format_description => "[acdn]{1,4}",
1124
1125         # note: this is also the fallback if boot: is not given at all
1126         default => 'cdn',
1127     },
1128     order => {
1129         optional => 1,
1130         type => 'string',
1131         format => 'pve-qm-bootdev-list',
1132         format_description => "device[;device...]",
1133         description => <<EODESC,
1134 The guest will attempt to boot from devices in the order they appear here.
1135
1136 Disks, optical drives and passed-through storage USB devices will be directly
1137 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1138 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1139
1140 Note that only devices in this list will be marked as bootable and thus loaded
1141 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1142 (e.g. software-raid), you need to specify all of them here.
1143
1144 Overrides the deprecated 'legacy=[acdn]*' value when given.
1145 EODESC
1146     },
1147 };
1148 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1149
1150 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1151 sub verify_bootdev {
1152     my ($dev, $noerr) = @_;
1153
1154     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1155     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1156
1157     my $check = sub {
1158         my ($base) = @_;
1159         return 0 if $dev !~ m/^$base\d+$/;
1160         return 0 if !$confdesc->{$dev};
1161         return 1;
1162     };
1163
1164     return $dev if $check->("net");
1165     return $dev if $check->("usb");
1166     return $dev if $check->("hostpci");
1167
1168     return if $noerr;
1169     die "invalid boot device '$dev'\n";
1170 }
1171
1172 sub print_bootorder {
1173     my ($devs) = @_;
1174     return "" if !@$devs;
1175     my $data = { order => join(';', @$devs) };
1176     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1177 }
1178
1179 my $kvm_api_version = 0;
1180
1181 sub kvm_version {
1182     return $kvm_api_version if $kvm_api_version;
1183
1184     open my $fh, '<', '/dev/kvm' or return;
1185
1186     # 0xae00 => KVM_GET_API_VERSION
1187     $kvm_api_version = ioctl($fh, 0xae00, 0);
1188     close($fh);
1189
1190     return $kvm_api_version;
1191 }
1192
1193 my $kvm_user_version = {};
1194 my $kvm_mtime = {};
1195
1196 sub kvm_user_version {
1197     my ($binary) = @_;
1198
1199     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1200     my $st = stat($binary);
1201
1202     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1203     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1204         $cachedmtime == $st->mtime;
1205
1206     $kvm_user_version->{$binary} = 'unknown';
1207     $kvm_mtime->{$binary} = $st->mtime;
1208
1209     my $code = sub {
1210         my $line = shift;
1211         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1212             $kvm_user_version->{$binary} = $2;
1213         }
1214     };
1215
1216     eval { run_command([$binary, '--version'], outfunc => $code); };
1217     warn $@ if $@;
1218
1219     return $kvm_user_version->{$binary};
1220
1221 }
1222 my sub extract_version {
1223     my ($machine_type, $version) = @_;
1224     $version = kvm_user_version() if !defined($version);
1225     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1226 }
1227
1228 sub kernel_has_vhost_net {
1229     return -c '/dev/vhost-net';
1230 }
1231
1232 sub option_exists {
1233     my $key = shift;
1234     return defined($confdesc->{$key});
1235 }
1236
1237 my $cdrom_path;
1238 sub get_cdrom_path {
1239
1240     return $cdrom_path if defined($cdrom_path);
1241
1242     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1243
1244     if (!defined($cdrom_path)) {
1245         log_warn("no physical CD-ROM available, ignoring");
1246         $cdrom_path = '';
1247     }
1248
1249     return $cdrom_path;
1250 }
1251
1252 sub get_iso_path {
1253     my ($storecfg, $vmid, $cdrom) = @_;
1254
1255     if ($cdrom eq 'cdrom') {
1256         return get_cdrom_path();
1257     } elsif ($cdrom eq 'none') {
1258         return '';
1259     } elsif ($cdrom =~ m|^/|) {
1260         return $cdrom;
1261     } else {
1262         return PVE::Storage::path($storecfg, $cdrom);
1263     }
1264 }
1265
1266 # try to convert old style file names to volume IDs
1267 sub filename_to_volume_id {
1268     my ($vmid, $file, $media) = @_;
1269
1270      if (!($file eq 'none' || $file eq 'cdrom' ||
1271           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1272
1273         return if $file =~ m|/|;
1274
1275         if ($media && $media eq 'cdrom') {
1276             $file = "local:iso/$file";
1277         } else {
1278             $file = "local:$vmid/$file";
1279         }
1280     }
1281
1282     return $file;
1283 }
1284
1285 sub verify_media_type {
1286     my ($opt, $vtype, $media) = @_;
1287
1288     return if !$media;
1289
1290     my $etype;
1291     if ($media eq 'disk') {
1292         $etype = 'images';
1293     } elsif ($media eq 'cdrom') {
1294         $etype = 'iso';
1295     } else {
1296         die "internal error";
1297     }
1298
1299     return if ($vtype eq $etype);
1300
1301     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1302 }
1303
1304 sub cleanup_drive_path {
1305     my ($opt, $storecfg, $drive) = @_;
1306
1307     # try to convert filesystem paths to volume IDs
1308
1309     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1310         ($drive->{file} !~ m|^/dev/.+|) &&
1311         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1312         ($drive->{file} !~ m/^\d+$/)) {
1313         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1314         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1315             if !$vtype;
1316         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1317         verify_media_type($opt, $vtype, $drive->{media});
1318         $drive->{file} = $volid;
1319     }
1320
1321     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1322 }
1323
1324 sub parse_hotplug_features {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     return $res if $data eq '0';
1330
1331     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1332
1333     foreach my $feature (PVE::Tools::split_list($data)) {
1334         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1335             $res->{$1} = 1;
1336         } else {
1337             die "invalid hotplug feature '$feature'\n";
1338         }
1339     }
1340     return $res;
1341 }
1342
1343 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1344 sub pve_verify_hotplug_features {
1345     my ($value, $noerr) = @_;
1346
1347     return $value if parse_hotplug_features($value);
1348
1349     return if $noerr;
1350
1351     die "unable to parse hotplug option\n";
1352 }
1353
1354 sub assert_clipboard_config {
1355     my ($vga) = @_;
1356
1357     my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1358
1359     if (
1360         $vga->{'clipboard'}
1361         && $vga->{'clipboard'} eq 'vnc'
1362         && $vga->{type}
1363         && $vga->{type} !~ $clipboard_regex
1364     ) {
1365         die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1366     }
1367 }
1368
1369 sub print_tabletdevice_full {
1370     my ($conf, $arch) = @_;
1371
1372     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1373
1374     # we use uhci for old VMs because tablet driver was buggy in older qemu
1375     my $usbbus;
1376     if ($q35 || $arch eq 'aarch64') {
1377         $usbbus = 'ehci';
1378     } else {
1379         $usbbus = 'uhci';
1380     }
1381
1382     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1383 }
1384
1385 sub print_keyboarddevice_full {
1386     my ($conf, $arch) = @_;
1387
1388     return if $arch ne 'aarch64';
1389
1390     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1391 }
1392
1393 my sub get_drive_id {
1394     my ($drive) = @_;
1395     return "$drive->{interface}$drive->{index}";
1396 }
1397
1398 sub print_drivedevice_full {
1399     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1400
1401     my $device = '';
1402     my $maxdev = 0;
1403
1404     my $drive_id = get_drive_id($drive);
1405     if ($drive->{interface} eq 'virtio') {
1406         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1407         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1408         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1409     } elsif ($drive->{interface} eq 'scsi') {
1410
1411         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1412         my $unit = $drive->{index} % $maxdev;
1413
1414         my $machine_version = extract_version($machine_type, kvm_user_version());
1415         my $devicetype  = PVE::QemuServer::Drive::get_scsi_devicetype(
1416             $drive, $storecfg, $machine_version);
1417
1418         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1419             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1420         } else {
1421             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1422                 .",lun=$drive->{index}";
1423         }
1424         $device .= ",drive=drive-$drive_id,id=$drive_id";
1425
1426         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1427             $device .= ",rotation_rate=1";
1428         }
1429         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1430
1431         # only scsi-hd and scsi-cd support passing vendor and product information
1432         if ($devicetype eq 'hd' || $devicetype eq 'cd') {
1433             if (my $vendor = $drive->{vendor}) {
1434                 $device .= ",vendor=$vendor";
1435             }
1436             if (my $product = $drive->{product}) {
1437                 $device .= ",product=$product";
1438             }
1439         }
1440
1441     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1442         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1443         my $controller = int($drive->{index} / $maxdev);
1444         my $unit = $drive->{index} % $maxdev;
1445
1446         # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1447         # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1448         # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1449         # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1450         # were before. Move odd ones up by 2 where they don't clash.
1451         if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1452             $controller += 2 * ($unit % 2);
1453             $unit = 0;
1454         }
1455
1456         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1457
1458         $device = "ide-$devicetype";
1459         if ($drive->{interface} eq 'ide') {
1460             $device .= ",bus=ide.$controller,unit=$unit";
1461         } else {
1462             $device .= ",bus=ahci$controller.$unit";
1463         }
1464         $device .= ",drive=drive-$drive_id,id=$drive_id";
1465
1466         if ($devicetype eq 'hd') {
1467             if (my $model = $drive->{model}) {
1468                 $model = URI::Escape::uri_unescape($model);
1469                 $device .= ",model=$model";
1470             }
1471             if ($drive->{ssd}) {
1472                 $device .= ",rotation_rate=1";
1473             }
1474         }
1475         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1476     } elsif ($drive->{interface} eq 'usb') {
1477         die "implement me";
1478         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1479     } else {
1480         die "unsupported interface type";
1481     }
1482
1483     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1484
1485     if (my $serial = $drive->{serial}) {
1486         $serial = URI::Escape::uri_unescape($serial);
1487         $device .= ",serial=$serial";
1488     }
1489
1490
1491     return $device;
1492 }
1493
1494 sub get_initiator_name {
1495     my $initiator;
1496
1497     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1498     while (defined(my $line = <$fh>)) {
1499         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1500         $initiator = $1;
1501         last;
1502     }
1503     $fh->close();
1504
1505     return $initiator;
1506 }
1507
1508 my sub storage_allows_io_uring_default {
1509     my ($scfg, $cache_direct) = @_;
1510
1511     # io_uring with cache mode writeback or writethrough on krbd will hang...
1512     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1513
1514     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1515     # sometimes, just plain disable...
1516     return if $scfg && $scfg->{type} eq 'lvm';
1517
1518     # io_uring causes problems when used with CIFS since kernel 5.15
1519     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1520     return if $scfg && $scfg->{type} eq 'cifs';
1521
1522     return 1;
1523 }
1524
1525 my sub drive_uses_cache_direct {
1526     my ($drive, $scfg) = @_;
1527
1528     my $cache_direct = 0;
1529
1530     if (my $cache = $drive->{cache}) {
1531         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1532     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1533         $cache_direct = 1;
1534     }
1535
1536     return $cache_direct;
1537 }
1538
1539 sub print_drive_commandline_full {
1540     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1541
1542     my $path;
1543     my $volid = $drive->{file};
1544     my $format = $drive->{format};
1545     my $drive_id = get_drive_id($drive);
1546
1547     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1548     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1549
1550     if (drive_is_cdrom($drive)) {
1551         $path = get_iso_path($storecfg, $vmid, $volid);
1552         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1553     } else {
1554         if ($storeid) {
1555             $path = PVE::Storage::path($storecfg, $volid);
1556             $format //= qemu_img_format($scfg, $volname);
1557         } else {
1558             $path = $volid;
1559             $format //= "raw";
1560         }
1561    }
1562
1563    my $is_rbd = $path =~ m/^rbd:/;
1564
1565     my $opts = '';
1566     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1567     foreach my $o (@qemu_drive_options) {
1568         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1569     }
1570
1571     # snapshot only accepts on|off
1572     if (defined($drive->{snapshot})) {
1573         my $v = $drive->{snapshot} ? 'on' : 'off';
1574         $opts .= ",snapshot=$v";
1575     }
1576
1577     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1578         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1579     }
1580
1581     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1582         my ($dir, $qmpname) = @$type;
1583         if (my $v = $drive->{"mbps$dir"}) {
1584             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1585         }
1586         if (my $v = $drive->{"mbps${dir}_max"}) {
1587             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1588         }
1589         if (my $v = $drive->{"bps${dir}_max_length"}) {
1590             $opts .= ",throttling.bps$qmpname-max-length=$v";
1591         }
1592         if (my $v = $drive->{"iops${dir}"}) {
1593             $opts .= ",throttling.iops$qmpname=$v";
1594         }
1595         if (my $v = $drive->{"iops${dir}_max"}) {
1596             $opts .= ",throttling.iops$qmpname-max=$v";
1597         }
1598         if (my $v = $drive->{"iops${dir}_max_length"}) {
1599             $opts .= ",throttling.iops$qmpname-max-length=$v";
1600         }
1601     }
1602
1603     if ($pbs_name) {
1604         $format = "rbd" if $is_rbd;
1605         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1606             if !$format;
1607         $opts .= ",format=alloc-track,file.driver=$format";
1608     } elsif ($format) {
1609         $opts .= ",format=$format";
1610     }
1611
1612     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1613
1614     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1615
1616     if (!$drive->{aio}) {
1617         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1618             # io_uring supports all cache modes
1619             $opts .= ",aio=io_uring";
1620         } else {
1621             # aio native works only with O_DIRECT
1622             if($cache_direct) {
1623                 $opts .= ",aio=native";
1624             } else {
1625                 $opts .= ",aio=threads";
1626             }
1627         }
1628     }
1629
1630     if (!drive_is_cdrom($drive)) {
1631         my $detectzeroes;
1632         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1633             $detectzeroes = 'off';
1634         } elsif ($drive->{discard}) {
1635             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1636         } else {
1637             # This used to be our default with discard not being specified:
1638             $detectzeroes = 'on';
1639         }
1640
1641         # note: 'detect-zeroes' works per blockdev and we want it to persist
1642         # after the alloc-track is removed, so put it on 'file' directly
1643         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1644         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1645     }
1646
1647     if ($pbs_name) {
1648         $opts .= ",backing=$pbs_name";
1649         $opts .= ",auto-remove=on";
1650     }
1651
1652     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1653     my $file_param = "file";
1654     if ($pbs_name) {
1655         # non-rbd drivers require the underlying file to be a seperate block
1656         # node, so add a second .file indirection
1657         $file_param .= ".file" if !$is_rbd;
1658         $file_param .= ".filename";
1659     }
1660     my $pathinfo = $path ? "$file_param=$path," : '';
1661
1662     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1663 }
1664
1665 sub print_pbs_blockdev {
1666     my ($pbs_conf, $pbs_name) = @_;
1667     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1668     $blockdev .= ",repository=$pbs_conf->{repository}";
1669     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1670     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1671     $blockdev .= ",archive=$pbs_conf->{archive}";
1672     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1673     return $blockdev;
1674 }
1675
1676 sub print_netdevice_full {
1677     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1678
1679     my $device = $net->{model};
1680     if ($net->{model} eq 'virtio') {
1681          $device = 'virtio-net-pci';
1682      };
1683
1684     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1685     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1686     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1687         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1688         # and out of each queue plus one config interrupt and control vector queue
1689         my $vectors = $net->{queues} * 2 + 2;
1690         $tmpstr .= ",vectors=$vectors,mq=on";
1691         if (min_version($machine_version, 7, 1)) {
1692             $tmpstr .= ",packed=on";
1693         }
1694     }
1695
1696     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1697         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1698     }
1699
1700     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1701
1702     if (my $mtu = $net->{mtu}) {
1703         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1704             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1705             if ($mtu == 1) {
1706                  $mtu = $bridge_mtu;
1707             } elsif ($mtu < 576) {
1708                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709             } elsif ($mtu > $bridge_mtu) {
1710                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1711             }
1712             $tmpstr .= ",host_mtu=$mtu";
1713         } else {
1714             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1715         }
1716     }
1717
1718     if ($use_old_bios_files) {
1719         my $romfile;
1720         if ($device eq 'virtio-net-pci') {
1721             $romfile = 'pxe-virtio.rom';
1722         } elsif ($device eq 'e1000') {
1723             $romfile = 'pxe-e1000.rom';
1724         } elsif ($device eq 'e1000e') {
1725             $romfile = 'pxe-e1000e.rom';
1726         } elsif ($device eq 'ne2k') {
1727             $romfile = 'pxe-ne2k_pci.rom';
1728         } elsif ($device eq 'pcnet') {
1729             $romfile = 'pxe-pcnet.rom';
1730         } elsif ($device eq 'rtl8139') {
1731             $romfile = 'pxe-rtl8139.rom';
1732         }
1733         $tmpstr .= ",romfile=$romfile" if $romfile;
1734     }
1735
1736     return $tmpstr;
1737 }
1738
1739 sub print_netdev_full {
1740     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1741
1742     my $i = '';
1743     if ($netid =~ m/^net(\d+)$/) {
1744         $i = int($1);
1745     }
1746
1747     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749     my $ifname = "tap${vmid}i$i";
1750
1751     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752     die "interface name '$ifname' is too long (max 15 character)\n"
1753         if length($ifname) >= 16;
1754
1755     my $vhostparam = '';
1756     if (is_native_arch($arch)) {
1757         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1758     }
1759
1760     my $vmname = $conf->{name} || "vm$vmid";
1761
1762     my $netdev = "";
1763     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765     if ($net->{bridge}) {
1766         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1768     } else {
1769         $netdev = "type=user,id=$netid,hostname=$vmname";
1770     }
1771
1772     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1773
1774     return $netdev;
1775 }
1776
1777 my $vga_map = {
1778     'cirrus' => 'cirrus-vga',
1779     'std' => 'VGA',
1780     'vmware' => 'vmware-svga',
1781     'virtio' => 'virtio-vga',
1782     'virtio-gl' => 'virtio-vga-gl',
1783 };
1784
1785 sub print_vga_device {
1786     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1787
1788     my $type = $vga_map->{$vga->{type}};
1789     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1790         $type = 'virtio-gpu';
1791     }
1792     my $vgamem_mb = $vga->{memory};
1793
1794     my $max_outputs = '';
1795     if ($qxlnum) {
1796         $type = $id ? 'qxl' : 'qxl-vga';
1797
1798         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1799             # set max outputs so linux can have up to 4 qxl displays with one device
1800             if (min_version($machine_version, 4, 1)) {
1801                 $max_outputs = ",max_outputs=4";
1802             }
1803         }
1804     }
1805
1806     die "no devicetype for $vga->{type}\n" if !$type;
1807
1808     my $memory = "";
1809     if ($vgamem_mb) {
1810         if ($vga->{type} =~ /^virtio/) {
1811             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1812             $memory = ",max_hostmem=$bytes";
1813         } elsif ($qxlnum) {
1814             # from https://www.spice-space.org/multiple-monitors.html
1815             $memory = ",vgamem_mb=$vga->{memory}";
1816             my $ram = $vgamem_mb * 4;
1817             my $vram = $vgamem_mb * 2;
1818             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1819         } else {
1820             $memory = ",vgamem_mb=$vga->{memory}";
1821         }
1822     } elsif ($qxlnum && $id) {
1823         $memory = ",ram_size=67108864,vram_size=33554432";
1824     }
1825
1826     my $edidoff = "";
1827     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1828         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1829     }
1830
1831     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1832     my $vgaid = "vga" . ($id // '');
1833     my $pciaddr;
1834     if ($q35 && $vgaid eq 'vga') {
1835         # the first display uses pcie.0 bus on q35 machines
1836         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1837     } else {
1838         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1839     }
1840
1841     if ($vga->{type} eq 'virtio-gl') {
1842         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1843         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1844             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1845
1846         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1847             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1848     }
1849
1850     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1851 }
1852
1853 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1854 sub parse_net {
1855     my ($data, $disable_mac_autogen) = @_;
1856
1857     my $res = eval { parse_property_string($net_fmt, $data) };
1858     if ($@) {
1859         warn $@;
1860         return;
1861     }
1862     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1863         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1864         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1865     }
1866     return $res;
1867 }
1868
1869 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1870 sub parse_ipconfig {
1871     my ($data) = @_;
1872
1873     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1874     if ($@) {
1875         warn $@;
1876         return;
1877     }
1878
1879     if ($res->{gw} && !$res->{ip}) {
1880         warn 'gateway specified without specifying an IP address';
1881         return;
1882     }
1883     if ($res->{gw6} && !$res->{ip6}) {
1884         warn 'IPv6 gateway specified without specifying an IPv6 address';
1885         return;
1886     }
1887     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1888         warn 'gateway specified together with DHCP';
1889         return;
1890     }
1891     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1892         # gw6 + auto/dhcp
1893         warn "IPv6 gateway specified together with $res->{ip6} address";
1894         return;
1895     }
1896
1897     if (!$res->{ip} && !$res->{ip6}) {
1898         return { ip => 'dhcp', ip6 => 'dhcp' };
1899     }
1900
1901     return $res;
1902 }
1903
1904 sub print_net {
1905     my $net = shift;
1906
1907     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1908 }
1909
1910 sub add_random_macs {
1911     my ($settings) = @_;
1912
1913     foreach my $opt (keys %$settings) {
1914         next if $opt !~ m/^net(\d+)$/;
1915         my $net = parse_net($settings->{$opt});
1916         next if !$net;
1917         $settings->{$opt} = print_net($net);
1918     }
1919 }
1920
1921 sub vm_is_volid_owner {
1922     my ($storecfg, $vmid, $volid) = @_;
1923
1924     if ($volid !~  m|^/|) {
1925         my ($path, $owner);
1926         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1927         if ($owner && ($owner == $vmid)) {
1928             return 1;
1929         }
1930     }
1931
1932     return;
1933 }
1934
1935 sub vmconfig_register_unused_drive {
1936     my ($storecfg, $vmid, $conf, $drive) = @_;
1937
1938     if (drive_is_cloudinit($drive)) {
1939         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1940         warn $@ if $@;
1941         delete $conf->{cloudinit};
1942     } elsif (!drive_is_cdrom($drive)) {
1943         my $volid = $drive->{file};
1944         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1945             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1946         }
1947     }
1948 }
1949
1950 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1951 my $smbios1_fmt = {
1952     uuid => {
1953         type => 'string',
1954         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1955         format_description => 'UUID',
1956         description => "Set SMBIOS1 UUID.",
1957         optional => 1,
1958     },
1959     version => {
1960         type => 'string',
1961         pattern => '[A-Za-z0-9+\/]+={0,2}',
1962         format_description => 'Base64 encoded string',
1963         description => "Set SMBIOS1 version.",
1964         optional => 1,
1965     },
1966     serial => {
1967         type => 'string',
1968         pattern => '[A-Za-z0-9+\/]+={0,2}',
1969         format_description => 'Base64 encoded string',
1970         description => "Set SMBIOS1 serial number.",
1971         optional => 1,
1972     },
1973     manufacturer => {
1974         type => 'string',
1975         pattern => '[A-Za-z0-9+\/]+={0,2}',
1976         format_description => 'Base64 encoded string',
1977         description => "Set SMBIOS1 manufacturer.",
1978         optional => 1,
1979     },
1980     product => {
1981         type => 'string',
1982         pattern => '[A-Za-z0-9+\/]+={0,2}',
1983         format_description => 'Base64 encoded string',
1984         description => "Set SMBIOS1 product ID.",
1985         optional => 1,
1986     },
1987     sku => {
1988         type => 'string',
1989         pattern => '[A-Za-z0-9+\/]+={0,2}',
1990         format_description => 'Base64 encoded string',
1991         description => "Set SMBIOS1 SKU string.",
1992         optional => 1,
1993     },
1994     family => {
1995         type => 'string',
1996         pattern => '[A-Za-z0-9+\/]+={0,2}',
1997         format_description => 'Base64 encoded string',
1998         description => "Set SMBIOS1 family string.",
1999         optional => 1,
2000     },
2001     base64 => {
2002         type => 'boolean',
2003         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2004         optional => 1,
2005     },
2006 };
2007
2008 sub parse_smbios1 {
2009     my ($data) = @_;
2010
2011     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2012     warn $@ if $@;
2013     return $res;
2014 }
2015
2016 sub print_smbios1 {
2017     my ($smbios1) = @_;
2018     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2019 }
2020
2021 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2022
2023 sub parse_watchdog {
2024     my ($value) = @_;
2025
2026     return if !$value;
2027
2028     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2029     warn $@ if $@;
2030     return $res;
2031 }
2032
2033 sub parse_guest_agent {
2034     my ($conf) = @_;
2035
2036     return {} if !defined($conf->{agent});
2037
2038     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2039     warn $@ if $@;
2040
2041     # if the agent is disabled ignore the other potentially set properties
2042     return {} if !$res->{enabled};
2043     return $res;
2044 }
2045
2046 sub get_qga_key {
2047     my ($conf, $key) = @_;
2048     return undef if !defined($conf->{agent});
2049
2050     my $agent = parse_guest_agent($conf);
2051     return $agent->{$key};
2052 }
2053
2054 sub parse_vga {
2055     my ($value) = @_;
2056
2057     return {} if !$value;
2058     my $res = eval { parse_property_string($vga_fmt, $value) };
2059     warn $@ if $@;
2060     return $res;
2061 }
2062
2063 sub parse_rng {
2064     my ($value) = @_;
2065
2066     return if !$value;
2067
2068     my $res = eval { parse_property_string($rng_fmt, $value) };
2069     warn $@ if $@;
2070     return $res;
2071 }
2072
2073 sub parse_meta_info {
2074     my ($value) = @_;
2075
2076     return if !$value;
2077
2078     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2079     warn $@ if $@;
2080     return $res;
2081 }
2082
2083 sub new_meta_info_string {
2084     my () = @_; # for now do not allow to override any value
2085
2086     return PVE::JSONSchema::print_property_string(
2087         {
2088             'creation-qemu' => kvm_user_version(),
2089             ctime => "". int(time()),
2090         },
2091         $meta_info_fmt
2092     );
2093 }
2094
2095 sub qemu_created_version_fixups {
2096     my ($conf, $forcemachine, $kvmver) = @_;
2097
2098     my $meta = parse_meta_info($conf->{meta}) // {};
2099     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2100
2101     # check if we need to apply some handling for VMs that always use the latest machine version but
2102     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2103     # would be required (we do not want to pin machine version for non-windows OS type)
2104     if (
2105         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2106         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2107         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2108         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2109     ) {
2110         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2111         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2112             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2113             # and thus with the predictable interface naming of systemd
2114             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2115         }
2116     }
2117     return;
2118 }
2119
2120 # add JSON properties for create and set function
2121 sub json_config_properties {
2122     my ($prop, $with_disk_alloc) = @_;
2123
2124     my $skip_json_config_opts = {
2125         parent => 1,
2126         snaptime => 1,
2127         vmstate => 1,
2128         runningmachine => 1,
2129         runningcpu => 1,
2130         meta => 1,
2131     };
2132
2133     foreach my $opt (keys %$confdesc) {
2134         next if $skip_json_config_opts->{$opt};
2135
2136         if ($with_disk_alloc && is_valid_drivename($opt)) {
2137             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2138         } else {
2139             $prop->{$opt} = $confdesc->{$opt};
2140         }
2141     }
2142
2143     return $prop;
2144 }
2145
2146 # Properties that we can read from an OVF file
2147 sub json_ovf_properties {
2148     my $prop = {};
2149
2150     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2151         $prop->{$device} = {
2152             type => 'string',
2153             format => 'pve-volume-id-or-absolute-path',
2154             description => "Disk image that gets imported to $device",
2155             optional => 1,
2156         };
2157     }
2158
2159     $prop->{cores} = {
2160         type => 'integer',
2161         description => "The number of CPU cores.",
2162         optional => 1,
2163     };
2164     $prop->{memory} = {
2165         type => 'integer',
2166         description => "Amount of RAM for the VM in MB.",
2167         optional => 1,
2168     };
2169     $prop->{name} = {
2170         type => 'string',
2171         description => "Name of the VM.",
2172         optional => 1,
2173     };
2174
2175     return $prop;
2176 }
2177
2178 # return copy of $confdesc_cloudinit to generate documentation
2179 sub cloudinit_config_properties {
2180
2181     return dclone($confdesc_cloudinit);
2182 }
2183
2184 sub cloudinit_pending_properties {
2185     my $p = {
2186         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2187         name => 1,
2188     };
2189     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2190     return $p;
2191 }
2192
2193 sub check_type {
2194     my ($key, $value) = @_;
2195
2196     die "unknown setting '$key'\n" if !$confdesc->{$key};
2197
2198     my $type = $confdesc->{$key}->{type};
2199
2200     if (!defined($value)) {
2201         die "got undefined value\n";
2202     }
2203
2204     if ($value =~ m/[\n\r]/) {
2205         die "property contains a line feed\n";
2206     }
2207
2208     if ($type eq 'boolean') {
2209         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2210         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2211         die "type check ('boolean') failed - got '$value'\n";
2212     } elsif ($type eq 'integer') {
2213         return int($1) if $value =~ m/^(\d+)$/;
2214         die "type check ('integer') failed - got '$value'\n";
2215     } elsif ($type eq 'number') {
2216         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2217         die "type check ('number') failed - got '$value'\n";
2218     } elsif ($type eq 'string') {
2219         if (my $fmt = $confdesc->{$key}->{format}) {
2220             PVE::JSONSchema::check_format($fmt, $value);
2221             return $value;
2222         }
2223         $value =~ s/^\"(.*)\"$/$1/;
2224         return $value;
2225     } else {
2226         die "internal error"
2227     }
2228 }
2229
2230 sub destroy_vm {
2231     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2232
2233     my $conf = PVE::QemuConfig->load_config($vmid);
2234
2235     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2236         PVE::QemuConfig->check_lock($conf);
2237     }
2238
2239     if ($conf->{template}) {
2240         # check if any base image is still used by a linked clone
2241         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2242                 my ($ds, $drive) = @_;
2243                 return if drive_is_cdrom($drive);
2244
2245                 my $volid = $drive->{file};
2246                 return if !$volid || $volid =~ m|^/|;
2247
2248                 die "base volume '$volid' is still in use by linked cloned\n"
2249                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2250
2251         });
2252     }
2253
2254     my $volids = {};
2255     my $remove_owned_drive = sub {
2256         my ($ds, $drive) = @_;
2257         return if drive_is_cdrom($drive, 1);
2258
2259         my $volid = $drive->{file};
2260         return if !$volid || $volid =~ m|^/|;
2261         return if $volids->{$volid};
2262
2263         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2264         return if !$path || !$owner || ($owner != $vmid);
2265
2266         $volids->{$volid} = 1;
2267         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2268         warn "Could not remove disk '$volid', check manually: $@" if $@;
2269     };
2270
2271     # only remove disks owned by this VM (referenced in the config)
2272     my $include_opts = {
2273         include_unused => 1,
2274         extra_keys => ['vmstate'],
2275     };
2276     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2277
2278     for my $snap (values %{$conf->{snapshots}}) {
2279         next if !defined($snap->{vmstate});
2280         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2281         next if !defined($drive);
2282         $remove_owned_drive->('vmstate', $drive);
2283     }
2284
2285     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2286
2287     if ($purge_unreferenced) { # also remove unreferenced disk
2288         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2289         PVE::Storage::foreach_volid($vmdisks, sub {
2290             my ($volid, $sid, $volname, $d) = @_;
2291             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2292             warn $@ if $@;
2293         });
2294     }
2295
2296     eval { delete_ifaces_ipams_ips($conf, $vmid)};
2297     warn $@ if $@;
2298
2299     if (defined $replacement_conf) {
2300         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2301     } else {
2302         PVE::QemuConfig->destroy_config($vmid);
2303     }
2304 }
2305
2306 sub parse_vm_config {
2307     my ($filename, $raw, $strict) = @_;
2308
2309     return if !defined($raw);
2310
2311     my $res = {
2312         digest => Digest::SHA::sha1_hex($raw),
2313         snapshots => {},
2314         pending => {},
2315         cloudinit => {},
2316     };
2317
2318     my $handle_error = sub {
2319         my ($msg) = @_;
2320
2321         if ($strict) {
2322             die $msg;
2323         } else {
2324             warn $msg;
2325         }
2326     };
2327
2328     $filename =~ m|/qemu-server/(\d+)\.conf$|
2329         || die "got strange filename '$filename'";
2330
2331     my $vmid = $1;
2332
2333     my $conf = $res;
2334     my $descr;
2335     my $finish_description = sub {
2336         if (defined($descr)) {
2337             $descr =~ s/\s+$//;
2338             $conf->{description} = $descr;
2339         }
2340         $descr = undef;
2341     };
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             $finish_description->();
2351             $conf = $res->{$section} = {};
2352             next;
2353         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2354             $section = 'cloudinit';
2355             $finish_description->();
2356             $conf = $res->{$section} = {};
2357             next;
2358
2359         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2360             $section = $1;
2361             $finish_description->();
2362             $conf = $res->{snapshots}->{$section} = {};
2363             next;
2364         }
2365
2366         if ($line =~ m/^\#(.*)$/) {
2367             $descr = '' if !defined($descr);
2368             $descr .= PVE::Tools::decode_text($1) . "\n";
2369             next;
2370         }
2371
2372         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2373             $descr = '' if !defined($descr);
2374             $descr .= PVE::Tools::decode_text($2);
2375         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2376             $conf->{snapstate} = $1;
2377         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2378             my $key = $1;
2379             my $value = $2;
2380             $conf->{$key} = $value;
2381         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2382             my $value = $1;
2383             if ($section eq 'pending') {
2384                 $conf->{delete} = $value; # we parse this later
2385             } else {
2386                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2387             }
2388         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             if ($section eq 'cloudinit') {
2392                 # ignore validation only used for informative purpose
2393                 $conf->{$key} = $value;
2394                 next;
2395             }
2396             eval { $value = check_type($key, $value); };
2397             if ($@) {
2398                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2399             } else {
2400                 $key = 'ide2' if $key eq 'cdrom';
2401                 my $fmt = $confdesc->{$key}->{format};
2402                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2403                     my $v = parse_drive($key, $value);
2404                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2405                         $v->{file} = $volid;
2406                         $value = print_drive($v);
2407                     } else {
2408                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2409                         next;
2410                     }
2411                 }
2412
2413                 $conf->{$key} = $value;
2414             }
2415         } else {
2416             $handle_error->("vm $vmid - unable to parse config: $line\n");
2417         }
2418     }
2419
2420     $finish_description->();
2421     delete $res->{snapstate}; # just to be sure
2422
2423     return $res;
2424 }
2425
2426 sub write_vm_config {
2427     my ($filename, $conf) = @_;
2428
2429     delete $conf->{snapstate}; # just to be sure
2430
2431     if ($conf->{cdrom}) {
2432         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2433         $conf->{ide2} = $conf->{cdrom};
2434         delete $conf->{cdrom};
2435     }
2436
2437     # we do not use 'smp' any longer
2438     if ($conf->{sockets}) {
2439         delete $conf->{smp};
2440     } elsif ($conf->{smp}) {
2441         $conf->{sockets} = $conf->{smp};
2442         delete $conf->{cores};
2443         delete $conf->{smp};
2444     }
2445
2446     my $used_volids = {};
2447
2448     my $cleanup_config = sub {
2449         my ($cref, $pending, $snapname) = @_;
2450
2451         foreach my $key (keys %$cref) {
2452             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2453                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2454             my $value = $cref->{$key};
2455             if ($key eq 'delete') {
2456                 die "propertry 'delete' is only allowed in [PENDING]\n"
2457                     if !$pending;
2458                 # fixme: check syntax?
2459                 next;
2460             }
2461             eval { $value = check_type($key, $value); };
2462             die "unable to parse value of '$key' - $@" if $@;
2463
2464             $cref->{$key} = $value;
2465
2466             if (!$snapname && is_valid_drivename($key)) {
2467                 my $drive = parse_drive($key, $value);
2468                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2469             }
2470         }
2471     };
2472
2473     &$cleanup_config($conf);
2474
2475     &$cleanup_config($conf->{pending}, 1);
2476
2477     foreach my $snapname (keys %{$conf->{snapshots}}) {
2478         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2479         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2480     }
2481
2482     # remove 'unusedX' settings if we re-add a volume
2483     foreach my $key (keys %$conf) {
2484         my $value = $conf->{$key};
2485         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2486             delete $conf->{$key};
2487         }
2488     }
2489
2490     my $generate_raw_config = sub {
2491         my ($conf, $pending) = @_;
2492
2493         my $raw = '';
2494
2495         # add description as comment to top of file
2496         if (defined(my $descr = $conf->{description})) {
2497             if ($descr) {
2498                 foreach my $cl (split(/\n/, $descr)) {
2499                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2500                 }
2501             } else {
2502                 $raw .= "#\n" if $pending;
2503             }
2504         }
2505
2506         foreach my $key (sort keys %$conf) {
2507             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2508             $raw .= "$key: $conf->{$key}\n";
2509         }
2510         return $raw;
2511     };
2512
2513     my $raw = &$generate_raw_config($conf);
2514
2515     if (scalar(keys %{$conf->{pending}})){
2516         $raw .= "\n[PENDING]\n";
2517         $raw .= &$generate_raw_config($conf->{pending}, 1);
2518     }
2519
2520     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2521         $raw .= "\n[special:cloudinit]\n";
2522         $raw .= &$generate_raw_config($conf->{cloudinit});
2523     }
2524
2525     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2526         $raw .= "\n[$snapname]\n";
2527         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2528     }
2529
2530     return $raw;
2531 }
2532
2533 sub load_defaults {
2534
2535     my $res = {};
2536
2537     # we use static defaults from our JSON schema configuration
2538     foreach my $key (keys %$confdesc) {
2539         if (defined(my $default = $confdesc->{$key}->{default})) {
2540             $res->{$key} = $default;
2541         }
2542     }
2543
2544     return $res;
2545 }
2546
2547 sub config_list {
2548     my $vmlist = PVE::Cluster::get_vmlist();
2549     my $res = {};
2550     return $res if !$vmlist || !$vmlist->{ids};
2551     my $ids = $vmlist->{ids};
2552     my $nodename = nodename();
2553
2554     foreach my $vmid (keys %$ids) {
2555         my $d = $ids->{$vmid};
2556         next if !$d->{node} || $d->{node} ne $nodename;
2557         next if !$d->{type} || $d->{type} ne 'qemu';
2558         $res->{$vmid}->{exists} = 1;
2559     }
2560     return $res;
2561 }
2562
2563 # test if VM uses local resources (to prevent migration)
2564 sub check_local_resources {
2565     my ($conf, $noerr) = @_;
2566
2567     my @loc_res = ();
2568     my $mapped_res = [];
2569
2570     my $nodelist = PVE::Cluster::get_nodelist();
2571     my $pci_map = PVE::Mapping::PCI::config();
2572     my $usb_map = PVE::Mapping::USB::config();
2573
2574     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2575
2576     my $add_missing_mapping = sub {
2577         my ($type, $key, $id) = @_;
2578         for my $node (@$nodelist) {
2579             my $entry;
2580             if ($type eq 'pci') {
2581                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2582             } elsif ($type eq 'usb') {
2583                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2584             }
2585             if (!scalar($entry->@*)) {
2586                 push @{$missing_mappings_by_node->{$node}}, $key;
2587             }
2588         }
2589     };
2590
2591     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2592     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2593
2594     push @loc_res, "ivshmem" if $conf->{ivshmem};
2595
2596     foreach my $k (keys %$conf) {
2597         if ($k =~ m/^usb/) {
2598             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2599             next if $entry->{host} =~ m/^spice$/i;
2600             if ($entry->{mapping}) {
2601                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2602                 push @$mapped_res, $k;
2603             }
2604         }
2605         if ($k =~ m/^hostpci/) {
2606             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2607             if ($entry->{mapping}) {
2608                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2609                 push @$mapped_res, $k;
2610             }
2611         }
2612         # sockets are safe: they will recreated be on the target side post-migrate
2613         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2614         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2615     }
2616
2617     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2618
2619     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2620 }
2621
2622 # check if used storages are available on all nodes (use by migrate)
2623 sub check_storage_availability {
2624     my ($storecfg, $conf, $node) = @_;
2625
2626     PVE::QemuConfig->foreach_volume($conf, sub {
2627         my ($ds, $drive) = @_;
2628
2629         my $volid = $drive->{file};
2630         return if !$volid;
2631
2632         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2633         return if !$sid;
2634
2635         # check if storage is available on both nodes
2636         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2637         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2638
2639         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2640
2641         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2642             if !$scfg->{content}->{$vtype};
2643    });
2644 }
2645
2646 # list nodes where all VM images are available (used by has_feature API)
2647 sub shared_nodes {
2648     my ($conf, $storecfg) = @_;
2649
2650     my $nodelist = PVE::Cluster::get_nodelist();
2651     my $nodehash = { map { $_ => 1 } @$nodelist };
2652     my $nodename = nodename();
2653
2654     PVE::QemuConfig->foreach_volume($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         if ($storeid) {
2662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2663             if ($scfg->{disable}) {
2664                 $nodehash = {};
2665             } elsif (my $avail = $scfg->{nodes}) {
2666                 foreach my $node (keys %$nodehash) {
2667                     delete $nodehash->{$node} if !$avail->{$node};
2668                 }
2669             } elsif (!$scfg->{shared}) {
2670                 foreach my $node (keys %$nodehash) {
2671                     delete $nodehash->{$node} if $node ne $nodename
2672                 }
2673             }
2674         }
2675     });
2676
2677     return $nodehash
2678 }
2679
2680 sub check_local_storage_availability {
2681     my ($conf, $storecfg) = @_;
2682
2683     my $nodelist = PVE::Cluster::get_nodelist();
2684     my $nodehash = { map { $_ => {} } @$nodelist };
2685
2686     PVE::QemuConfig->foreach_volume($conf, sub {
2687         my ($ds, $drive) = @_;
2688
2689         my $volid = $drive->{file};
2690         return if !$volid;
2691
2692         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2693         if ($storeid) {
2694             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2695
2696             if ($scfg->{disable}) {
2697                 foreach my $node (keys %$nodehash) {
2698                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2699                 }
2700             } elsif (my $avail = $scfg->{nodes}) {
2701                 foreach my $node (keys %$nodehash) {
2702                     if (!$avail->{$node}) {
2703                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2704                     }
2705                 }
2706             }
2707         }
2708     });
2709
2710     foreach my $node (values %$nodehash) {
2711         if (my $unavail = $node->{unavailable_storages}) {
2712             $node->{unavailable_storages} = [ sort keys %$unavail ];
2713         }
2714     }
2715
2716     return $nodehash
2717 }
2718
2719 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2720 sub check_running {
2721     my ($vmid, $nocheck, $node) = @_;
2722
2723     # $nocheck is set when called during a migration, in which case the config
2724     # file might still or already reside on the *other* node
2725     # - because rename has already happened, and current node is source
2726     # - because rename hasn't happened yet, and current node is target
2727     # - because rename has happened, current node is target, but hasn't yet
2728     # processed it yet
2729     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2730     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2731 }
2732
2733 sub vzlist {
2734
2735     my $vzlist = config_list();
2736
2737     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2738
2739     while (defined(my $de = $fd->read)) {
2740         next if $de !~ m/^(\d+)\.pid$/;
2741         my $vmid = $1;
2742         next if !defined($vzlist->{$vmid});
2743         if (my $pid = check_running($vmid)) {
2744             $vzlist->{$vmid}->{pid} = $pid;
2745         }
2746     }
2747
2748     return $vzlist;
2749 }
2750
2751 our $vmstatus_return_properties = {
2752     vmid => get_standard_option('pve-vmid'),
2753     status => {
2754         description => "QEMU process status.",
2755         type => 'string',
2756         enum => ['stopped', 'running'],
2757     },
2758     maxmem => {
2759         description => "Maximum memory in bytes.",
2760         type => 'integer',
2761         optional => 1,
2762         renderer => 'bytes',
2763     },
2764     maxdisk => {
2765         description => "Root disk size in bytes.",
2766         type => 'integer',
2767         optional => 1,
2768         renderer => 'bytes',
2769     },
2770     name => {
2771         description => "VM name.",
2772         type => 'string',
2773         optional => 1,
2774     },
2775     qmpstatus => {
2776         description => "VM run state from the 'query-status' QMP monitor command.",
2777         type => 'string',
2778         optional => 1,
2779     },
2780     pid => {
2781         description => "PID of running qemu process.",
2782         type => 'integer',
2783         optional => 1,
2784     },
2785     uptime => {
2786         description => "Uptime.",
2787         type => 'integer',
2788         optional => 1,
2789         renderer => 'duration',
2790     },
2791     cpus => {
2792         description => "Maximum usable CPUs.",
2793         type => 'number',
2794         optional => 1,
2795     },
2796     lock => {
2797         description => "The current config lock, if any.",
2798         type => 'string',
2799         optional => 1,
2800     },
2801     tags => {
2802         description  => "The current configured tags, if any",
2803         type => 'string',
2804         optional => 1,
2805     },
2806     'running-machine' => {
2807         description => "The currently running machine type (if running).",
2808         type => 'string',
2809         optional => 1,
2810     },
2811     'running-qemu' => {
2812         description => "The currently running QEMU version (if running).",
2813         type => 'string',
2814         optional => 1,
2815     },
2816 };
2817
2818 my $last_proc_pid_stat;
2819
2820 # get VM status information
2821 # This must be fast and should not block ($full == false)
2822 # We only query KVM using QMP if $full == true (this can be slow)
2823 sub vmstatus {
2824     my ($opt_vmid, $full) = @_;
2825
2826     my $res = {};
2827
2828     my $storecfg = PVE::Storage::config();
2829
2830     my $list = vzlist();
2831     my $defaults = load_defaults();
2832
2833     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2834
2835     my $cpucount = $cpuinfo->{cpus} || 1;
2836
2837     foreach my $vmid (keys %$list) {
2838         next if $opt_vmid && ($vmid ne $opt_vmid);
2839
2840         my $conf = PVE::QemuConfig->load_config($vmid);
2841
2842         my $d = { vmid => int($vmid) };
2843         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2844
2845         # fixme: better status?
2846         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2847
2848         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2849         if (defined($size)) {
2850             $d->{disk} = 0; # no info available
2851             $d->{maxdisk} = $size;
2852         } else {
2853             $d->{disk} = 0;
2854             $d->{maxdisk} = 0;
2855         }
2856
2857         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2858             * ($conf->{cores} || $defaults->{cores});
2859         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2860         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2861
2862         $d->{name} = $conf->{name} || "VM $vmid";
2863         $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2864
2865         if ($conf->{balloon}) {
2866             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2867             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2868                 : $defaults->{shares};
2869         }
2870
2871         $d->{uptime} = 0;
2872         $d->{cpu} = 0;
2873         $d->{mem} = 0;
2874
2875         $d->{netout} = 0;
2876         $d->{netin} = 0;
2877
2878         $d->{diskread} = 0;
2879         $d->{diskwrite} = 0;
2880
2881         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2882
2883         $d->{serial} = 1 if conf_has_serial($conf);
2884         $d->{lock} = $conf->{lock} if $conf->{lock};
2885         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2886
2887         $res->{$vmid} = $d;
2888     }
2889
2890     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2891     foreach my $dev (keys %$netdev) {
2892         next if $dev !~ m/^tap([1-9]\d*)i/;
2893         my $vmid = $1;
2894         my $d = $res->{$vmid};
2895         next if !$d;
2896
2897         $d->{netout} += $netdev->{$dev}->{receive};
2898         $d->{netin} += $netdev->{$dev}->{transmit};
2899
2900         if ($full) {
2901             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2902             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2903         }
2904
2905     }
2906
2907     my $ctime = gettimeofday;
2908
2909     foreach my $vmid (keys %$list) {
2910
2911         my $d = $res->{$vmid};
2912         my $pid = $d->{pid};
2913         next if !$pid;
2914
2915         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2916         next if !$pstat; # not running
2917
2918         my $used = $pstat->{utime} + $pstat->{stime};
2919
2920         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2921
2922         if ($pstat->{vsize}) {
2923             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2924         }
2925
2926         my $old = $last_proc_pid_stat->{$pid};
2927         if (!$old) {
2928             $last_proc_pid_stat->{$pid} = {
2929                 time => $ctime,
2930                 used => $used,
2931                 cpu => 0,
2932             };
2933             next;
2934         }
2935
2936         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2937
2938         if ($dtime > 1000) {
2939             my $dutime = $used -  $old->{used};
2940
2941             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2942             $last_proc_pid_stat->{$pid} = {
2943                 time => $ctime,
2944                 used => $used,
2945                 cpu => $d->{cpu},
2946             };
2947         } else {
2948             $d->{cpu} = $old->{cpu};
2949         }
2950     }
2951
2952     return $res if !$full;
2953
2954     my $qmpclient = PVE::QMPClient->new();
2955
2956     my $ballooncb = sub {
2957         my ($vmid, $resp) = @_;
2958
2959         my $info = $resp->{'return'};
2960         return if !$info->{max_mem};
2961
2962         my $d = $res->{$vmid};
2963
2964         # use memory assigned to VM
2965         $d->{maxmem} = $info->{max_mem};
2966         $d->{balloon} = $info->{actual};
2967
2968         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2969             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2970             $d->{freemem} = $info->{free_mem};
2971         }
2972
2973         $d->{ballooninfo} = $info;
2974     };
2975
2976     my $blockstatscb = sub {
2977         my ($vmid, $resp) = @_;
2978         my $data = $resp->{'return'} || [];
2979         my $totalrdbytes = 0;
2980         my $totalwrbytes = 0;
2981
2982         for my $blockstat (@$data) {
2983             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2984             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2985
2986             $blockstat->{device} =~ s/drive-//;
2987             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2988         }
2989         $res->{$vmid}->{diskread} = $totalrdbytes;
2990         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2991     };
2992
2993     my $machinecb = sub {
2994         my ($vmid, $resp) = @_;
2995         my $data = $resp->{'return'} || [];
2996
2997         $res->{$vmid}->{'running-machine'} =
2998             PVE::QemuServer::Machine::current_from_query_machines($data);
2999     };
3000
3001     my $versioncb = sub {
3002         my ($vmid, $resp) = @_;
3003         my $data = $resp->{'return'} // {};
3004         my $version = 'unknown';
3005
3006         if (my $v = $data->{qemu}) {
3007             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3008         }
3009
3010         $res->{$vmid}->{'running-qemu'} = $version;
3011     };
3012
3013     my $statuscb = sub {
3014         my ($vmid, $resp) = @_;
3015
3016         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3017         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3018         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3019         # this fails if ballon driver is not loaded, so this must be
3020         # the last commnand (following command are aborted if this fails).
3021         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3022
3023         my $status = 'unknown';
3024         if (!defined($status = $resp->{'return'}->{status})) {
3025             warn "unable to get VM status\n";
3026             return;
3027         }
3028
3029         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3030     };
3031
3032     foreach my $vmid (keys %$list) {
3033         next if $opt_vmid && ($vmid ne $opt_vmid);
3034         next if !$res->{$vmid}->{pid}; # not running
3035         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3036     }
3037
3038     $qmpclient->queue_execute(undef, 2);
3039
3040     foreach my $vmid (keys %$list) {
3041         next if $opt_vmid && ($vmid ne $opt_vmid);
3042         next if !$res->{$vmid}->{pid}; #not running
3043
3044         # we can't use the $qmpclient since it might have already aborted on
3045         # 'query-balloon', but this might also fail for older versions...
3046         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3047         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3048     }
3049
3050     foreach my $vmid (keys %$list) {
3051         next if $opt_vmid && ($vmid ne $opt_vmid);
3052         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3053     }
3054
3055     return $res;
3056 }
3057
3058 sub conf_has_serial {
3059     my ($conf) = @_;
3060
3061     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3062         if ($conf->{"serial$i"}) {
3063             return 1;
3064         }
3065     }
3066
3067     return 0;
3068 }
3069
3070 sub conf_has_audio {
3071     my ($conf, $id) = @_;
3072
3073     $id //= 0;
3074     my $audio = $conf->{"audio$id"};
3075     return if !defined($audio);
3076
3077     my $audioproperties = parse_property_string($audio_fmt, $audio);
3078     my $audiodriver = $audioproperties->{driver} // 'spice';
3079
3080     return {
3081         dev => $audioproperties->{device},
3082         dev_id => "audiodev$id",
3083         backend => $audiodriver,
3084         backend_id => "$audiodriver-backend${id}",
3085     };
3086 }
3087
3088 sub audio_devs {
3089     my ($audio, $audiopciaddr, $machine_version) = @_;
3090
3091     my $devs = [];
3092
3093     my $id = $audio->{dev_id};
3094     my $audiodev = "";
3095     if (min_version($machine_version, 4, 2)) {
3096         $audiodev = ",audiodev=$audio->{backend_id}";
3097     }
3098
3099     if ($audio->{dev} eq 'AC97') {
3100         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3101     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3102         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3103         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3104         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3105     } else {
3106         die "unkown audio device '$audio->{dev}', implement me!";
3107     }
3108
3109     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3110
3111     return $devs;
3112 }
3113
3114 sub get_tpm_paths {
3115     my ($vmid) = @_;
3116     return {
3117         socket => "/var/run/qemu-server/$vmid.swtpm",
3118         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3119     };
3120 }
3121
3122 sub add_tpm_device {
3123     my ($vmid, $devices, $conf) = @_;
3124
3125     return if !$conf->{tpmstate0};
3126
3127     my $paths = get_tpm_paths($vmid);
3128
3129     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3130     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3131     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3132 }
3133
3134 sub start_swtpm {
3135     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3136
3137     return if !$tpmdrive;
3138
3139     my $state;
3140     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3141     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3142     if ($storeid) {
3143         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3144     } else {
3145         $state = $tpm->{file};
3146     }
3147
3148     my $paths = get_tpm_paths($vmid);
3149
3150     # during migration, we will get state from remote
3151     #
3152     if (!$migration) {
3153         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3154         my $setup_cmd = [
3155             "swtpm_setup",
3156             "--tpmstate",
3157             "file://$state",
3158             "--createek",
3159             "--create-ek-cert",
3160             "--create-platform-cert",
3161             "--lock-nvram",
3162             "--config",
3163             "/etc/swtpm_setup.conf", # do not use XDG configs
3164             "--runas",
3165             "0", # force creation as root, error if not possible
3166             "--not-overwrite", # ignore existing state, do not modify
3167         ];
3168
3169         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3170         # TPM 2.0 supports ECC crypto, use if possible
3171         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3172
3173         run_command($setup_cmd, outfunc => sub {
3174             print "swtpm_setup: $1\n";
3175         });
3176     }
3177
3178     # Used to distinguish different invocations in the log.
3179     my $log_prefix = "[id=" . int(time()) . "] ";
3180
3181     my $emulator_cmd = [
3182         "swtpm",
3183         "socket",
3184         "--tpmstate",
3185         "backend-uri=file://$state,mode=0600",
3186         "--ctrl",
3187         "type=unixio,path=$paths->{socket},mode=0600",
3188         "--pid",
3189         "file=$paths->{pid}",
3190         "--terminate", # terminate on QEMU disconnect
3191         "--daemon",
3192         "--log",
3193         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3194     ];
3195     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3196     run_command($emulator_cmd, outfunc => sub { print $1; });
3197
3198     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3199     while (! -e $paths->{pid}) {
3200         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3201         usleep(50_000);
3202     }
3203
3204     # return untainted PID of swtpm daemon so it can be killed on error
3205     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3206     return $1;
3207 }
3208
3209 sub vga_conf_has_spice {
3210     my ($vga) = @_;
3211
3212     my $vgaconf = parse_vga($vga);
3213     my $vgatype = $vgaconf->{type};
3214     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3215
3216     return $1 || 1;
3217 }
3218
3219 sub get_vm_arch {
3220     my ($conf) = @_;
3221     return $conf->{arch} // get_host_arch();
3222 }
3223
3224 my $default_machines = {
3225     x86_64 => 'pc',
3226     aarch64 => 'virt',
3227 };
3228
3229 sub get_installed_machine_version {
3230     my ($kvmversion) = @_;
3231     $kvmversion = kvm_user_version() if !defined($kvmversion);
3232     $kvmversion =~ m/^(\d+\.\d+)/;
3233     return $1;
3234 }
3235
3236 sub windows_get_pinned_machine_version {
3237     my ($machine, $base_version, $kvmversion) = @_;
3238
3239     my $pin_version = $base_version;
3240     if (!defined($base_version) ||
3241         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3242     ) {
3243         $pin_version = get_installed_machine_version($kvmversion);
3244     }
3245     if (!$machine || $machine eq 'pc') {
3246         $machine = "pc-i440fx-$pin_version";
3247     } elsif ($machine eq 'q35') {
3248         $machine = "pc-q35-$pin_version";
3249     } elsif ($machine eq 'virt') {
3250         $machine = "virt-$pin_version";
3251     } else {
3252         warn "unknown machine type '$machine', not touching that!\n";
3253     }
3254
3255     return $machine;
3256 }
3257
3258 sub get_vm_machine {
3259     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3260
3261     my $machine = $forcemachine || $conf->{machine};
3262
3263     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3264         $kvmversion //= kvm_user_version();
3265         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3266         # layout which confuses windows quite a bit and may result in various regressions..
3267         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3268         if (windows_version($conf->{ostype})) {
3269             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3270         }
3271         $arch //= 'x86_64';
3272         $machine ||= $default_machines->{$arch};
3273         if ($add_pve_version) {
3274             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3275             $machine .= "+pve$pvever";
3276         }
3277     }
3278
3279     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3280         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3281         $machine = $1 if $is_pxe;
3282
3283         # for version-pinned machines that do not include a pve-version (e.g.
3284         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3285         $machine .= '+pve0';
3286
3287         $machine .= '.pxe' if $is_pxe;
3288     }
3289
3290     return $machine;
3291 }
3292
3293 sub get_ovmf_files($$$) {
3294     my ($arch, $efidisk, $smm) = @_;
3295
3296     my $types = $OVMF->{$arch}
3297         or die "no OVMF images known for architecture '$arch'\n";
3298
3299     my $type = 'default';
3300     if ($arch eq 'x86_64') {
3301         if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3302             $type = $smm ? "4m" : "4m-no-smm";
3303             $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3304         } else {
3305             # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3306         }
3307     }
3308
3309     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3310     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3311     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3312
3313     return ($ovmf_code, $ovmf_vars);
3314 }
3315
3316 my $Arch2Qemu = {
3317     aarch64 => '/usr/bin/qemu-system-aarch64',
3318     x86_64 => '/usr/bin/qemu-system-x86_64',
3319 };
3320 sub get_command_for_arch($) {
3321     my ($arch) = @_;
3322     return '/usr/bin/kvm' if is_native_arch($arch);
3323
3324     my $cmd = $Arch2Qemu->{$arch}
3325         or die "don't know how to emulate architecture '$arch'\n";
3326     return $cmd;
3327 }
3328
3329 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3330 # to use in a QEMU command line (-cpu element), first array_intersect the result
3331 # of query_supported_ with query_understood_. This is necessary because:
3332 #
3333 # a) query_understood_ returns flags the host cannot use and
3334 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3335 #    flags, but CPU settings - with most of them being flags. Those settings
3336 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3337 #
3338 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3339 # expensive. If you need the value returned from this, you can get it much
3340 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3341 # $accel being 'kvm' or 'tcg'.
3342 #
3343 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3344 # changes, automatically populating pmxcfs.
3345 #
3346 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3347 # since kvm and tcg machines support different flags
3348 #
3349 sub query_supported_cpu_flags {
3350     my ($arch) = @_;
3351
3352     $arch //= get_host_arch();
3353     my $default_machine = $default_machines->{$arch};
3354
3355     my $flags = {};
3356
3357     # FIXME: Once this is merged, the code below should work for ARM as well:
3358     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3359     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3360         $arch eq "aarch64";
3361
3362     my $kvm_supported = defined(kvm_version());
3363     my $qemu_cmd = get_command_for_arch($arch);
3364     my $fakevmid = -1;
3365     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3366
3367     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3368     my $query_supported_run_qemu = sub {
3369         my ($kvm) = @_;
3370
3371         my $flags = {};
3372         my $cmd = [
3373             $qemu_cmd,
3374             '-machine', $default_machine,
3375             '-display', 'none',
3376             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3377             '-mon', 'chardev=qmp,mode=control',
3378             '-pidfile', $pidfile,
3379             '-S', '-daemonize'
3380         ];
3381
3382         if (!$kvm) {
3383             push @$cmd, '-accel', 'tcg';
3384         }
3385
3386         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3387         die "QEMU flag querying VM exited with code " . $rc if $rc;
3388
3389         eval {
3390             my $cmd_result = mon_cmd(
3391                 $fakevmid,
3392                 'query-cpu-model-expansion',
3393                 type => 'full',
3394                 model => { name => 'host' }
3395             );
3396
3397             my $props = $cmd_result->{model}->{props};
3398             foreach my $prop (keys %$props) {
3399                 next if $props->{$prop} ne '1';
3400                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3401                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3402                 # We only keep those with underscores, to match /proc/cpuinfo
3403                 $prop =~ s/\.|-/_/g;
3404                 $flags->{$prop} = 1;
3405             }
3406         };
3407         my $err = $@;
3408
3409         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3410         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3411
3412         die $err if $err;
3413
3414         return [ sort keys %$flags ];
3415     };
3416
3417     # We need to query QEMU twice, since KVM and TCG have different supported flags
3418     PVE::QemuConfig->lock_config($fakevmid, sub {
3419         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3420         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3421
3422         if ($kvm_supported) {
3423             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3424             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3425         }
3426     });
3427
3428     return $flags;
3429 }
3430
3431 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3432 my $understood_cpu_flag_dir = "/usr/share/kvm";
3433 sub query_understood_cpu_flags {
3434     my $arch = get_host_arch();
3435     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3436
3437     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3438         if ! -e $filepath;
3439
3440     my $raw = file_get_contents($filepath);
3441     $raw =~ s/^\s+|\s+$//g;
3442     my @flags = split(/\s+/, $raw);
3443
3444     return \@flags;
3445 }
3446
3447 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3448 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3449 my sub should_disable_smm {
3450     my ($conf, $vga, $machine) = @_;
3451
3452     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3453
3454     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3455         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3456 }
3457
3458 my sub print_ovmf_drive_commandlines {
3459     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3460
3461     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3462
3463     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3464
3465     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3466     if ($d) {
3467         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3468         my ($path, $format) = $d->@{'file', 'format'};
3469         if ($storeid) {
3470             $path = PVE::Storage::path($storecfg, $d->{file});
3471             if (!defined($format)) {
3472                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3473                 $format = qemu_img_format($scfg, $volname);
3474             }
3475         } elsif (!defined($format)) {
3476             die "efidisk format must be specified\n";
3477         }
3478         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3479         if ($path =~ m/^rbd:/) {
3480             $var_drive_str .= ',cache=writeback';
3481             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3482         }
3483         $var_drive_str .= ",format=$format,file=$path";
3484
3485         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3486         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3487     } else {
3488         log_warn("no efidisk configured! Using temporary efivars disk.");
3489         my $path = "/tmp/$vmid-ovmf.fd";
3490         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3491         $var_drive_str .= ",format=raw,file=$path";
3492         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3493     }
3494
3495     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3496 }
3497
3498 sub config_to_command {
3499     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3500         $pbs_backing) = @_;
3501
3502     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3503     my $devices = [];
3504     my $bridges = {};
3505     my $ostype = $conf->{ostype};
3506     my $winversion = windows_version($ostype);
3507     my $kvm = $conf->{kvm};
3508     my $nodename = nodename();
3509
3510     my $arch = get_vm_arch($conf);
3511     my $kvm_binary = get_command_for_arch($arch);
3512     my $kvmver = kvm_user_version($kvm_binary);
3513
3514     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3515         $kvmver //= "undefined";
3516         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3517     }
3518
3519     my $add_pve_version = min_version($kvmver, 4, 1);
3520
3521     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3522     my $machine_version = extract_version($machine_type, $kvmver);
3523     $kvm //= 1 if is_native_arch($arch);
3524
3525     $machine_version =~ m/(\d+)\.(\d+)/;
3526     my ($machine_major, $machine_minor) = ($1, $2);
3527
3528     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3529         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3530     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3531         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3532             ." please upgrade node '$nodename'\n"
3533     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3534         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3535         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3536             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3537             ." node '$nodename'\n";
3538     }
3539
3540     # if a specific +pve version is required for a feature, use $version_guard
3541     # instead of min_version to allow machines to be run with the minimum
3542     # required version
3543     my $required_pve_version = 0;
3544     my $version_guard = sub {
3545         my ($major, $minor, $pve) = @_;
3546         return 0 if !min_version($machine_version, $major, $minor, $pve);
3547         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3548         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3549         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3550         return 1;
3551     };
3552
3553     if ($kvm && !defined kvm_version()) {
3554         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3555             ." or enable in BIOS.\n";
3556     }
3557
3558     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3559     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3560     my $use_old_bios_files = undef;
3561     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3562
3563     my $cmd = [];
3564     if ($conf->{affinity}) {
3565         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3566     }
3567
3568     push @$cmd, $kvm_binary;
3569
3570     push @$cmd, '-id', $vmid;
3571
3572     my $vmname = $conf->{name} || "vm$vmid";
3573
3574     push @$cmd, '-name', "$vmname,debug-threads=on";
3575
3576     push @$cmd, '-no-shutdown';
3577
3578     my $use_virtio = 0;
3579
3580     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3581     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3582     push @$cmd, '-mon', "chardev=qmp,mode=control";
3583
3584     if (min_version($machine_version, 2, 12)) {
3585         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3586         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3587     }
3588
3589     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3590
3591     push @$cmd, '-daemonize';
3592
3593     if ($conf->{smbios1}) {
3594         my $smbios_conf = parse_smbios1($conf->{smbios1});
3595         if ($smbios_conf->{base64}) {
3596             # Do not pass base64 flag to qemu
3597             delete $smbios_conf->{base64};
3598             my $smbios_string = "";
3599             foreach my $key (keys %$smbios_conf) {
3600                 my $value;
3601                 if ($key eq "uuid") {
3602                     $value = $smbios_conf->{uuid}
3603                 } else {
3604                     $value = decode_base64($smbios_conf->{$key});
3605                 }
3606                 # qemu accepts any binary data, only commas need escaping by double comma
3607                 $value =~ s/,/,,/g;
3608                 $smbios_string .= "," . $key . "=" . $value if $value;
3609             }
3610             push @$cmd, '-smbios', "type=1" . $smbios_string;
3611         } else {
3612             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3613         }
3614     }
3615
3616     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3617         die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3618             if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3619
3620         my ($code_drive_str, $var_drive_str) =
3621             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3622         push $cmd->@*, '-drive', $code_drive_str;
3623         push $cmd->@*, '-drive', $var_drive_str;
3624     }
3625
3626     if ($q35) { # tell QEMU to load q35 config early
3627         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628         if (min_version($machine_version, 4, 0)) {
3629             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3630         } else {
3631             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632         }
3633     }
3634
3635     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3636         push @$cmd, $fixups->@*;
3637     }
3638
3639     if ($conf->{vmgenid}) {
3640         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3641     }
3642
3643     # add usb controllers
3644     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3645         $conf, $bridges, $arch, $machine_type, $machine_version);
3646     push @$devices, @usbcontrollers if @usbcontrollers;
3647     my $vga = parse_vga($conf->{vga});
3648
3649     my $qxlnum = vga_conf_has_spice($conf->{vga});
3650     $vga->{type} = 'qxl' if $qxlnum;
3651
3652     if (!$vga->{type}) {
3653         if ($arch eq 'aarch64') {
3654             $vga->{type} = 'virtio';
3655         } elsif (min_version($machine_version, 2, 9)) {
3656             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657         } else {
3658             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659         }
3660     }
3661
3662     # enable absolute mouse coordinates (needed by vnc)
3663     my $tablet = $conf->{tablet};
3664     if (!defined($tablet)) {
3665         $tablet = $defaults->{tablet};
3666         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668     }
3669
3670     if ($tablet) {
3671         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3672         my $kbd = print_keyboarddevice_full($conf, $arch);
3673         push @$devices, '-device', $kbd if defined($kbd);
3674     }
3675
3676     my $bootorder = device_bootorder($conf);
3677
3678     # host pci device passthrough
3679     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3680         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3681
3682     # usb devices
3683     my $usb_dev_features = {};
3684     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3685
3686     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3687         $conf, $usb_dev_features, $bootorder, $machine_version);
3688     push @$devices, @usbdevices if @usbdevices;
3689
3690     # serial devices
3691     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3692         my $path = $conf->{"serial$i"} or next;
3693         if ($path eq 'socket') {
3694             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696             # On aarch64, serial0 is the UART device. QEMU only allows
3697             # connecting UART devices via the '-serial' command line, as
3698             # the device has a fixed slot on the hardware...
3699             if ($arch eq 'aarch64' && $i == 0) {
3700                 push @$devices, '-serial', "chardev:serial$i";
3701             } else {
3702                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703             }
3704         } else {
3705             die "no such serial device\n" if ! -c $path;
3706             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3707             push @$devices, '-device', "isa-serial,chardev=serial$i";
3708         }
3709     }
3710
3711     # parallel devices
3712     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3713         if (my $path = $conf->{"parallel$i"}) {
3714             die "no such parallel device\n" if ! -c $path;
3715             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3716             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718         }
3719     }
3720
3721     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3722         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3723         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3724         push @$devices, @$audio_devs;
3725     }
3726
3727     # Add a TPM only if the VM is not a template,
3728     # to support backing up template VMs even if the TPM disk is write-protected.
3729     add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3730
3731     my $sockets = 1;
3732     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3733     $sockets = $conf->{sockets} if  $conf->{sockets};
3734
3735     my $cores = $conf->{cores} || 1;
3736
3737     my $maxcpus = $sockets * $cores;
3738
3739     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3740
3741     my $allowed_vcpus = $cpuinfo->{cpus};
3742
3743     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3744
3745     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3746         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747         for (my $i = 2; $i <= $vcpus; $i++)  {
3748             my $cpustr = print_cpu_device($conf, $arch, $i);
3749             push @$cmd, '-device', $cpustr;
3750         }
3751
3752     } else {
3753
3754         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755     }
3756     push @$cmd, '-nodefaults';
3757
3758     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3759
3760     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3761
3762     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3763
3764     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3765         push @$devices, '-device', print_vga_device(
3766             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3767
3768         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3769
3770         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3771         push @$cmd,  '-vnc', "unix:$socket,password=on";
3772     } else {
3773         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3774         push @$cmd, '-nographic';
3775     }
3776
3777     # time drift fix
3778     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3779     my $useLocaltime = $conf->{localtime};
3780
3781     if ($winversion >= 5) { # windows
3782         $useLocaltime = 1 if !defined($conf->{localtime});
3783
3784         # use time drift fix when acpi is enabled
3785         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3786             $tdf = 1 if !defined($conf->{tdf});
3787         }
3788     }
3789
3790     if ($winversion >= 6) {
3791         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3792         push @$machineFlags, 'hpet=off';
3793     }
3794
3795     push @$rtcFlags, 'driftfix=slew' if $tdf;
3796
3797     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3798         push @$rtcFlags, "base=$conf->{startdate}";
3799     } elsif ($useLocaltime) {
3800         push @$rtcFlags, 'base=localtime';
3801     }
3802
3803     if ($forcecpu) {
3804         push @$cmd, '-cpu', $forcecpu;
3805     } else {
3806         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3807     }
3808
3809     PVE::QemuServer::Memory::config(
3810         $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3811
3812     push @$cmd, '-S' if $conf->{freeze};
3813
3814     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3815
3816     my $guest_agent = parse_guest_agent($conf);
3817
3818     if ($guest_agent->{enabled}) {
3819         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3820         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3821
3822         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3823             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3824             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3825             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3826         } elsif ($guest_agent->{type} eq 'isa') {
3827             push @$devices, '-device', "isa-serial,chardev=qga0";
3828         }
3829     }
3830
3831     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3832     if ($rng && $version_guard->(4, 1, 2)) {
3833         check_rng_source($rng->{source});
3834
3835         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3836         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3837         my $limiter_str = "";
3838         if ($max_bytes) {
3839             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3840         }
3841
3842         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3843         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3844         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3845     }
3846
3847     my $spice_port;
3848
3849     assert_clipboard_config($vga);
3850     my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3851
3852     if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3853         if ($qxlnum > 1) {
3854             if ($winversion){
3855                 for (my $i = 1; $i < $qxlnum; $i++){
3856                     push @$devices, '-device', print_vga_device(
3857                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3858                 }
3859             } else {
3860                 # assume other OS works like Linux
3861                 my ($ram, $vram) = ("134217728", "67108864");
3862                 if ($vga->{memory}) {
3863                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3864                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3865                 }
3866                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3867                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3868             }
3869         }
3870
3871         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3872
3873         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3874         if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3875             push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3876         } else {
3877             push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3878         }
3879         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880
3881         if ($is_spice) {
3882             my $pfamily = PVE::Tools::get_host_address_family($nodename);
3883             my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3884             die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3885
3886             my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3887             $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3888
3889             my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3890             my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3891             if ($spice_enhancement->{foldersharing}) {
3892                 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893                 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3894             }
3895
3896             my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897             $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898                 if $spice_enhancement->{videostreaming};
3899             push @$devices, '-spice', "$spice_opts";
3900         }
3901     }
3902
3903     # enable balloon by default, unless explicitly disabled
3904     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3905         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3906         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3908         push @$devices, '-device', $ballooncmd;
3909     }
3910
3911     if ($conf->{watchdog}) {
3912         my $wdopts = parse_watchdog($conf->{watchdog});
3913         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3914         my $watchdog = $wdopts->{model} || 'i6300esb';
3915         push @$devices, '-device', "$watchdog$pciaddr";
3916         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3917     }
3918
3919     my $vollist = [];
3920     my $scsicontroller = {};
3921     my $ahcicontroller = {};
3922     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3923
3924     # Add iscsi initiator name if available
3925     if (my $initiator = get_initiator_name()) {
3926         push @$devices, '-iscsi', "initiator-name=$initiator";
3927     }
3928
3929     PVE::QemuConfig->foreach_volume($conf, sub {
3930         my ($ds, $drive) = @_;
3931
3932         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3933             check_volume_storage_type($storecfg, $drive->{file});
3934             push @$vollist, $drive->{file};
3935         }
3936
3937         # ignore efidisk here, already added in bios/fw handling code above
3938         return if $drive->{interface} eq 'efidisk';
3939         # similar for TPM
3940         return if $drive->{interface} eq 'tpmstate';
3941
3942         $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3945
3946         if ($drive->{interface} eq 'virtio'){
3947            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3948         }
3949
3950         if ($drive->{interface} eq 'scsi') {
3951
3952             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3953
3954             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3956
3957             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3959
3960             my $iothread = '';
3961             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3962                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964             } elsif ($drive->{iothread}) {
3965                 log_warn(
3966                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3967                 );
3968             }
3969
3970             my $queues = '';
3971             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3972                 $queues = ",num_queues=$drive->{queues}";
3973             }
3974
3975             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3976                 if !$scsicontroller->{$controller};
3977             $scsicontroller->{$controller}=1;
3978         }
3979
3980         if ($drive->{interface} eq 'sata') {
3981             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3982             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3983             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3984                 if !$ahcicontroller->{$controller};
3985             $ahcicontroller->{$controller}=1;
3986         }
3987
3988         my $pbs_conf = $pbs_backing->{$ds};
3989         my $pbs_name = undef;
3990         if ($pbs_conf) {
3991             $pbs_name = "drive-$ds-pbs";
3992             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3993         }
3994
3995         my $drive_cmd = print_drive_commandline_full(
3996             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3997
3998         # extra protection for templates, but SATA and IDE don't support it..
3999         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4000
4001         push @$devices, '-drive',$drive_cmd;
4002         push @$devices, '-device', print_drivedevice_full(
4003             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4004     });
4005
4006     for (my $i = 0; $i < $MAX_NETS; $i++) {
4007         my $netname = "net$i";
4008
4009         next if !$conf->{$netname};
4010         my $d = parse_net($conf->{$netname});
4011         next if !$d;
4012         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4013
4014         $use_virtio = 1 if $d->{model} eq 'virtio';
4015
4016         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4017
4018         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4019         push @$devices, '-netdev', $netdevfull;
4020
4021         my $netdevicefull = print_netdevice_full(
4022             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4023
4024         push @$devices, '-device', $netdevicefull;
4025     }
4026
4027     if ($conf->{ivshmem}) {
4028         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4029
4030         my $bus;
4031         if ($q35) {
4032             $bus = print_pcie_addr("ivshmem");
4033         } else {
4034             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4035         }
4036
4037         my $ivshmem_name = $ivshmem->{name} // $vmid;
4038         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4039
4040         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4042             .",size=$ivshmem->{size}M";
4043     }
4044
4045     # pci.4 is nested in pci.1
4046     $bridges->{1} = 1 if $bridges->{4};
4047
4048     if (!$q35) { # add pci bridges
4049         if (min_version($machine_version, 2, 3)) {
4050            $bridges->{1} = 1;
4051            $bridges->{2} = 1;
4052         }
4053         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4054     }
4055
4056     for my $k (sort {$b cmp $a} keys %$bridges) {
4057         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4058
4059         my $k_name = $k;
4060         if ($k == 2 && $legacy_igd) {
4061             $k_name = "$k-igd";
4062         }
4063         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4064         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4065
4066         if ($q35) { # add after -readconfig pve-q35.cfg
4067             splice @$devices, 2, 0, '-device', $devstr;
4068         } else {
4069             unshift @$devices, '-device', $devstr if $k > 0;
4070         }
4071     }
4072
4073     if (!$kvm) {
4074         push @$machineFlags, 'accel=tcg';
4075     }
4076
4077     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4078
4079     my $machine_type_min = $machine_type;
4080     if ($add_pve_version) {
4081         $machine_type_min =~ s/\+pve\d+$//;
4082         $machine_type_min .= "+pve$required_pve_version";
4083     }
4084     push @$machineFlags, "type=${machine_type_min}";
4085
4086     push @$cmd, @$devices;
4087     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4088     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4089     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4090
4091     if (my $vmstate = $conf->{vmstate}) {
4092         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4093         push @$vollist, $vmstate;
4094         push @$cmd, '-loadstate', $statepath;
4095         print "activating and using '$vmstate' as vmstate\n";
4096     }
4097
4098     if (PVE::QemuConfig->is_template($conf)) {
4099         # needed to workaround base volumes being read-only
4100         push @$cmd, '-snapshot';
4101     }
4102
4103     # add custom args
4104     if ($conf->{args}) {
4105         my $aa = PVE::Tools::split_args($conf->{args});
4106         push @$cmd, @$aa;
4107     }
4108
4109     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4110 }
4111
4112 sub check_rng_source {
4113     my ($source) = @_;
4114
4115     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4116     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4117         if ! -e $source;
4118
4119     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4120     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4121         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4122         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4123         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4124             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4125             ." to the host.\n";
4126     }
4127 }
4128
4129 sub spice_port {
4130     my ($vmid) = @_;
4131
4132     my $res = mon_cmd($vmid, 'query-spice');
4133
4134     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4135 }
4136
4137 sub vm_devices_list {
4138     my ($vmid) = @_;
4139
4140     my $res = mon_cmd($vmid, 'query-pci');
4141     my $devices_to_check = [];
4142     my $devices = {};
4143     foreach my $pcibus (@$res) {
4144         push @$devices_to_check, @{$pcibus->{devices}},
4145     }
4146
4147     while (@$devices_to_check) {
4148         my $to_check = [];
4149         for my $d (@$devices_to_check) {
4150             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4151             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4152
4153             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4154             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4155         }
4156         $devices_to_check = $to_check;
4157     }
4158
4159     my $resblock = mon_cmd($vmid, 'query-block');
4160     foreach my $block (@$resblock) {
4161         if($block->{device} =~ m/^drive-(\S+)/){
4162                 $devices->{$1} = 1;
4163         }
4164     }
4165
4166     my $resmice = mon_cmd($vmid, 'query-mice');
4167     foreach my $mice (@$resmice) {
4168         if ($mice->{name} eq 'QEMU HID Tablet') {
4169             $devices->{tablet} = 1;
4170             last;
4171         }
4172     }
4173
4174     # for usb devices there is no query-usb
4175     # but we can iterate over the entries in
4176     # qom-list path=/machine/peripheral
4177     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4178     foreach my $per (@$resperipheral) {
4179         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4180             $devices->{$per->{name}} = 1;
4181         }
4182     }
4183
4184     return $devices;
4185 }
4186
4187 sub vm_deviceplug {
4188     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4189
4190     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4191
4192     my $devices_list = vm_devices_list($vmid);
4193     return 1 if defined($devices_list->{$deviceid});
4194
4195     # add PCI bridge if we need it for the device
4196     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4197
4198     if ($deviceid eq 'tablet') {
4199         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4200     } elsif ($deviceid eq 'keyboard') {
4201         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4202     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4203         my $id = $1;
4204         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4205         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4206     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4207         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4208     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4209         qemu_iothread_add($vmid, $deviceid, $device);
4210
4211         qemu_driveadd($storecfg, $vmid, $device);
4212         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4213
4214         qemu_deviceadd($vmid, $devicefull);
4215         eval { qemu_deviceaddverify($vmid, $deviceid); };
4216         if (my $err = $@) {
4217             eval { qemu_drivedel($vmid, $deviceid); };
4218             warn $@ if $@;
4219             die $err;
4220         }
4221     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4222         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4223         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4224         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4225
4226         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4227
4228         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4229             qemu_iothread_add($vmid, $deviceid, $device);
4230             $devicefull .= ",iothread=iothread-$deviceid";
4231         }
4232
4233         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4234             $devicefull .= ",num_queues=$device->{queues}";
4235         }
4236
4237         qemu_deviceadd($vmid, $devicefull);
4238         qemu_deviceaddverify($vmid, $deviceid);
4239     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4240         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4241         qemu_driveadd($storecfg, $vmid, $device);
4242
4243         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4244         eval { qemu_deviceadd($vmid, $devicefull); };
4245         if (my $err = $@) {
4246             eval { qemu_drivedel($vmid, $deviceid); };
4247             warn $@ if $@;
4248             die $err;
4249         }
4250     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4251         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4252
4253         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4254         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4255         my $use_old_bios_files = undef;
4256         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4257
4258         my $netdevicefull = print_netdevice_full(
4259             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4260         qemu_deviceadd($vmid, $netdevicefull);
4261         eval {
4262             qemu_deviceaddverify($vmid, $deviceid);
4263             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4264         };
4265         if (my $err = $@) {
4266             eval { qemu_netdevdel($vmid, $deviceid); };
4267             warn $@ if $@;
4268             die $err;
4269         }
4270     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4271         my $bridgeid = $2;
4272         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4273         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4274
4275         qemu_deviceadd($vmid, $devicefull);
4276         qemu_deviceaddverify($vmid, $deviceid);
4277     } else {
4278         die "can't hotplug device '$deviceid'\n";
4279     }
4280
4281     return 1;
4282 }
4283
4284 # fixme: this should raise exceptions on error!
4285 sub vm_deviceunplug {
4286     my ($vmid, $conf, $deviceid) = @_;
4287
4288     my $devices_list = vm_devices_list($vmid);
4289     return 1 if !defined($devices_list->{$deviceid});
4290
4291     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4292     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4293
4294     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4295         qemu_devicedel($vmid, $deviceid);
4296     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4297         qemu_devicedel($vmid, $deviceid);
4298         qemu_devicedelverify($vmid, $deviceid);
4299     } elsif ($deviceid =~ m/^usb\d+$/) {
4300         qemu_devicedel($vmid, $deviceid);
4301         qemu_devicedelverify($vmid, $deviceid);
4302     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4303         my $device = parse_drive($deviceid, $conf->{$deviceid});
4304
4305         qemu_devicedel($vmid, $deviceid);
4306         qemu_devicedelverify($vmid, $deviceid);
4307         qemu_drivedel($vmid, $deviceid);
4308         qemu_iothread_del($vmid, $deviceid, $device);
4309     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4310         qemu_devicedel($vmid, $deviceid);
4311         qemu_devicedelverify($vmid, $deviceid);
4312     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4313         my $device = parse_drive($deviceid, $conf->{$deviceid});
4314
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_drivedel($vmid, $deviceid);
4318         qemu_deletescsihw($conf, $vmid, $deviceid);
4319
4320         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4321             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4322     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4323         qemu_devicedel($vmid, $deviceid);
4324         qemu_devicedelverify($vmid, $deviceid);
4325         qemu_netdevdel($vmid, $deviceid);
4326     } else {
4327         die "can't unplug device '$deviceid'\n";
4328     }
4329
4330     return 1;
4331 }
4332
4333 sub qemu_spice_usbredir_chardev_add {
4334     my ($vmid, $id) = @_;
4335
4336     mon_cmd($vmid, "chardev-add" , (
4337         id => $id,
4338         backend => {
4339             type => 'spicevmc',
4340             data => {
4341                 type => "usbredir",
4342             },
4343         },
4344     ));
4345 }
4346
4347 sub qemu_iothread_add {
4348     my ($vmid, $deviceid, $device) = @_;
4349
4350     if ($device->{iothread}) {
4351         my $iothreads = vm_iothreads_list($vmid);
4352         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4353     }
4354 }
4355
4356 sub qemu_iothread_del {
4357     my ($vmid, $deviceid, $device) = @_;
4358
4359     if ($device->{iothread}) {
4360         my $iothreads = vm_iothreads_list($vmid);
4361         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4362     }
4363 }
4364
4365 sub qemu_driveadd {
4366     my ($storecfg, $vmid, $device) = @_;
4367
4368     my $kvmver = get_running_qemu_version($vmid);
4369     my $io_uring = min_version($kvmver, 6, 0);
4370     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4371     $drive =~ s/\\/\\\\/g;
4372     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4373
4374     # If the command succeeds qemu prints: "OK"
4375     return 1 if $ret =~ m/OK/s;
4376
4377     die "adding drive failed: $ret\n";
4378 }
4379
4380 sub qemu_drivedel {
4381     my ($vmid, $deviceid) = @_;
4382
4383     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4384     $ret =~ s/^\s+//;
4385
4386     return 1 if $ret eq "";
4387
4388     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4389     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4390
4391     die "deleting drive $deviceid failed : $ret\n";
4392 }
4393
4394 sub qemu_deviceaddverify {
4395     my ($vmid, $deviceid) = @_;
4396
4397     for (my $i = 0; $i <= 5; $i++) {
4398          my $devices_list = vm_devices_list($vmid);
4399          return 1 if defined($devices_list->{$deviceid});
4400          sleep 1;
4401     }
4402
4403     die "error on hotplug device '$deviceid'\n";
4404 }
4405
4406
4407 sub qemu_devicedelverify {
4408     my ($vmid, $deviceid) = @_;
4409
4410     # need to verify that the device is correctly removed as device_del
4411     # is async and empty return is not reliable
4412
4413     for (my $i = 0; $i <= 5; $i++) {
4414          my $devices_list = vm_devices_list($vmid);
4415          return 1 if !defined($devices_list->{$deviceid});
4416          sleep 1;
4417     }
4418
4419     die "error on hot-unplugging device '$deviceid'\n";
4420 }
4421
4422 sub qemu_findorcreatescsihw {
4423     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4424
4425     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4426
4427     my $scsihwid="$controller_prefix$controller";
4428     my $devices_list = vm_devices_list($vmid);
4429
4430     if (!defined($devices_list->{$scsihwid})) {
4431         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4432     }
4433
4434     return 1;
4435 }
4436
4437 sub qemu_deletescsihw {
4438     my ($conf, $vmid, $opt) = @_;
4439
4440     my $device = parse_drive($opt, $conf->{$opt});
4441
4442     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4443         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4444         return 1;
4445     }
4446
4447     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4448
4449     my $devices_list = vm_devices_list($vmid);
4450     foreach my $opt (keys %{$devices_list}) {
4451         if (is_valid_drivename($opt)) {
4452             my $drive = parse_drive($opt, $conf->{$opt});
4453             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4454                 return 1;
4455             }
4456         }
4457     }
4458
4459     my $scsihwid="scsihw$controller";
4460
4461     vm_deviceunplug($vmid, $conf, $scsihwid);
4462
4463     return 1;
4464 }
4465
4466 sub qemu_add_pci_bridge {
4467     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4468
4469     my $bridges = {};
4470
4471     my $bridgeid;
4472
4473     print_pci_addr($device, $bridges, $arch, $machine_type);
4474
4475     while (my ($k, $v) = each %$bridges) {
4476         $bridgeid = $k;
4477     }
4478     return 1 if !defined($bridgeid) || $bridgeid < 1;
4479
4480     my $bridge = "pci.$bridgeid";
4481     my $devices_list = vm_devices_list($vmid);
4482
4483     if (!defined($devices_list->{$bridge})) {
4484         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4485     }
4486
4487     return 1;
4488 }
4489
4490 sub qemu_set_link_status {
4491     my ($vmid, $device, $up) = @_;
4492
4493     mon_cmd($vmid, "set_link", name => $device,
4494                up => $up ? JSON::true : JSON::false);
4495 }
4496
4497 sub qemu_netdevadd {
4498     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4499
4500     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4501     my %options =  split(/[=,]/, $netdev);
4502
4503     if (defined(my $vhost = $options{vhost})) {
4504         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4505     }
4506
4507     if (defined(my $queues = $options{queues})) {
4508         $options{queues} = $queues + 0;
4509     }
4510
4511     mon_cmd($vmid, "netdev_add",  %options);
4512     return 1;
4513 }
4514
4515 sub qemu_netdevdel {
4516     my ($vmid, $deviceid) = @_;
4517
4518     mon_cmd($vmid, "netdev_del", id => $deviceid);
4519 }
4520
4521 sub qemu_usb_hotplug {
4522     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4523
4524     return if !$device;
4525
4526     # remove the old one first
4527     vm_deviceunplug($vmid, $conf, $deviceid);
4528
4529     # check if xhci controller is necessary and available
4530     my $devicelist = vm_devices_list($vmid);
4531
4532     if (!$devicelist->{xhci}) {
4533         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4534         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4535     }
4536
4537     # add the new one
4538     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4539 }
4540
4541 sub qemu_cpu_hotplug {
4542     my ($vmid, $conf, $vcpus) = @_;
4543
4544     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4545
4546     my $sockets = 1;
4547     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4548     $sockets = $conf->{sockets} if  $conf->{sockets};
4549     my $cores = $conf->{cores} || 1;
4550     my $maxcpus = $sockets * $cores;
4551
4552     $vcpus = $maxcpus if !$vcpus;
4553
4554     die "you can't add more vcpus than maxcpus\n"
4555         if $vcpus > $maxcpus;
4556
4557     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4558
4559     if ($vcpus < $currentvcpus) {
4560
4561         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4562
4563             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4564                 qemu_devicedel($vmid, "cpu$i");
4565                 my $retry = 0;
4566                 my $currentrunningvcpus = undef;
4567                 while (1) {
4568                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4569                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4570                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4571                     $retry++;
4572                     sleep 1;
4573                 }
4574                 #update conf after each succesfull cpu unplug
4575                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4576                 PVE::QemuConfig->write_config($vmid, $conf);
4577             }
4578         } else {
4579             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4580         }
4581
4582         return;
4583     }
4584
4585     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4586     die "vcpus in running vm does not match its configuration\n"
4587         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4588
4589     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4590         my $arch = get_vm_arch($conf);
4591
4592         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593             my $cpustr = print_cpu_device($conf, $arch, $i);
4594             qemu_deviceadd($vmid, $cpustr);
4595
4596             my $retry = 0;
4597             my $currentrunningvcpus = undef;
4598             while (1) {
4599                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                 last if scalar(@{$currentrunningvcpus}) == $i;
4601                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4602                 sleep 1;
4603                 $retry++;
4604             }
4605             #update conf after each succesfull cpu hotplug
4606             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607             PVE::QemuConfig->write_config($vmid, $conf);
4608         }
4609     } else {
4610
4611         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612             mon_cmd($vmid, "cpu-add", id => int($i));
4613         }
4614     }
4615 }
4616
4617 sub qemu_block_set_io_throttle {
4618     my ($vmid, $deviceid,
4619         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4623
4624     return if !check_running($vmid) ;
4625
4626     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4627         bps => int($bps),
4628         bps_rd => int($bps_rd),
4629         bps_wr => int($bps_wr),
4630         iops => int($iops),
4631         iops_rd => int($iops_rd),
4632         iops_wr => int($iops_wr),
4633         bps_max => int($bps_max),
4634         bps_rd_max => int($bps_rd_max),
4635         bps_wr_max => int($bps_wr_max),
4636         iops_max => int($iops_max),
4637         iops_rd_max => int($iops_rd_max),
4638         iops_wr_max => int($iops_wr_max),
4639         bps_max_length => int($bps_max_length),
4640         bps_rd_max_length => int($bps_rd_max_length),
4641         bps_wr_max_length => int($bps_wr_max_length),
4642         iops_max_length => int($iops_max_length),
4643         iops_rd_max_length => int($iops_rd_max_length),
4644         iops_wr_max_length => int($iops_wr_max_length),
4645     );
4646
4647 }
4648
4649 sub qemu_block_resize {
4650     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4655
4656     return if !$running;
4657
4658     my $padding = (1024 - $size % 1024) % 1024;
4659     $size = $size + $padding;
4660
4661     mon_cmd(
4662         $vmid,
4663         "block_resize",
4664         device => $deviceid,
4665         size => int($size),
4666         timeout => 60,
4667     );
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686     my $attached_deviceid;
4687
4688     if ($running) {
4689         my $conf = PVE::QemuConfig->load_config($vmid);
4690         PVE::QemuConfig->foreach_volume($conf, sub {
4691             my ($ds, $drive) = @_;
4692             $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4693         });
4694     }
4695
4696     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4697         mon_cmd(
4698             $vmid,
4699             'blockdev-snapshot-delete-internal-sync',
4700             device => $attached_deviceid,
4701             name => $snap,
4702         );
4703     } else {
4704         PVE::Storage::volume_snapshot_delete(
4705             $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4706     }
4707 }
4708
4709 sub set_migration_caps {
4710     my ($vmid, $savevm) = @_;
4711
4712     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4713
4714     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4715     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4716
4717     my $cap_ref = [];
4718
4719     my $enabled_cap = {
4720         "auto-converge" => 1,
4721         "xbzrle" => 1,
4722         "x-rdma-pin-all" => 0,
4723         "zero-blocks" => 0,
4724         "compress" => 0,
4725         "dirty-bitmaps" => $dirty_bitmaps,
4726     };
4727
4728     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4729
4730     for my $supported_capability (@$supported_capabilities) {
4731         push @$cap_ref, {
4732             capability => $supported_capability->{capability},
4733             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4734         };
4735     }
4736
4737     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4738 }
4739
4740 sub foreach_volid {
4741     my ($conf, $func, @param) = @_;
4742
4743     my $volhash = {};
4744
4745     my $test_volid = sub {
4746         my ($key, $drive, $snapname, $pending) = @_;
4747
4748         my $volid = $drive->{file};
4749         return if !$volid;
4750
4751         $volhash->{$volid}->{cdrom} //= 1;
4752         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4753
4754         my $replicate = $drive->{replicate} // 1;
4755         $volhash->{$volid}->{replicate} //= 0;
4756         $volhash->{$volid}->{replicate} = 1 if $replicate;
4757
4758         $volhash->{$volid}->{shared} //= 0;
4759         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4760
4761         $volhash->{$volid}->{is_unused} //= 0;
4762         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4763
4764         $volhash->{$volid}->{is_attached} //= 0;
4765         $volhash->{$volid}->{is_attached} = 1
4766             if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4767
4768         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4769             if defined($snapname);
4770
4771         $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4772
4773         my $size = $drive->{size};
4774         $volhash->{$volid}->{size} //= $size if $size;
4775
4776         $volhash->{$volid}->{is_vmstate} //= 0;
4777         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4778
4779         $volhash->{$volid}->{is_tpmstate} //= 0;
4780         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4781
4782         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4783     };
4784
4785     my $include_opts = {
4786         extra_keys => ['vmstate'],
4787         include_unused => 1,
4788     };
4789
4790     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4791
4792     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4793         if defined($conf->{pending}) && $conf->{pending}->%*;
4794
4795     foreach my $snapname (keys %{$conf->{snapshots}}) {
4796         my $snap = $conf->{snapshots}->{$snapname};
4797         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4798     }
4799
4800     foreach my $volid (keys %$volhash) {
4801         &$func($volid, $volhash->{$volid}, @param);
4802     }
4803 }
4804
4805 my $fast_plug_option = {
4806     'description' => 1,
4807     'hookscript' => 1,
4808     'lock' => 1,
4809     'migrate_downtime' => 1,
4810     'migrate_speed' => 1,
4811     'name' => 1,
4812     'onboot' => 1,
4813     'protection' => 1,
4814     'shares' => 1,
4815     'startup' => 1,
4816     'tags' => 1,
4817     'vmstatestorage' => 1,
4818 };
4819
4820 for my $opt (keys %$confdesc_cloudinit) {
4821     $fast_plug_option->{$opt} = 1;
4822 };
4823
4824 # hotplug changes in [PENDING]
4825 # $selection hash can be used to only apply specified options, for
4826 # example: { cores => 1 } (only apply changed 'cores')
4827 # $errors ref is used to return error messages
4828 sub vmconfig_hotplug_pending {
4829     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4830
4831     my $defaults = load_defaults();
4832     my $arch = get_vm_arch($conf);
4833     my $machine_type = get_vm_machine($conf, undef, $arch);
4834
4835     # commit values which do not have any impact on running VM first
4836     # Note: those option cannot raise errors, we we do not care about
4837     # $selection and always apply them.
4838
4839     my $add_error = sub {
4840         my ($opt, $msg) = @_;
4841         $errors->{$opt} = "hotplug problem - $msg";
4842     };
4843
4844     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4845
4846     my $cloudinit_record_changed = sub {
4847         my ($conf, $opt, $old, $new) = @_;
4848         return if !$cloudinit_pending_properties->{$opt};
4849
4850         my $ci = ($conf->{cloudinit} //= {});
4851
4852         my $recorded = $ci->{$opt};
4853         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4854
4855         if (defined($new)) {
4856             if (defined($old)) {
4857                 # an existing value is being modified
4858                 if (defined($recorded)) {
4859                     # the value was already not in sync
4860                     if ($new eq $recorded) {
4861                         # a value is being reverted to the cloud-init state:
4862                         delete $ci->{$opt};
4863                         delete $added{$opt};
4864                     } else {
4865                         # the value was changed multiple times, do nothing
4866                     }
4867                 } elsif ($added{$opt}) {
4868                     # the value had been marked as added and is being changed, do nothing
4869                 } else {
4870                     # the value is new, record it:
4871                     $ci->{$opt} = $old;
4872                 }
4873             } else {
4874                 # a new value is being added
4875                 if (defined($recorded)) {
4876                     # it was already not in sync
4877                     if ($new eq $recorded) {
4878                         # a value is being reverted to the cloud-init state:
4879                         delete $ci->{$opt};
4880                         delete $added{$opt};
4881                     } else {
4882                         # the value had temporarily been removed, do nothing
4883                     }
4884                 } elsif ($added{$opt}) {
4885                     # the value had been marked as added already, do nothing
4886                 } else {
4887                     # the value is new, add it
4888                     $added{$opt} = 1;
4889                 }
4890             }
4891         } elsif (!defined($old)) {
4892             # a non-existent value is being removed? ignore...
4893         } else {
4894             # a value is being deleted
4895             if (defined($recorded)) {
4896                 # a value was already recorded, just keep it
4897             } elsif ($added{$opt}) {
4898                 # the value was marked as added, remove it
4899                 delete $added{$opt};
4900             } else {
4901                 # a previously unrecorded value is being removed, record the old value:
4902                 $ci->{$opt} = $old;
4903             }
4904         }
4905
4906         my $added = join(',', sort keys %added);
4907         $ci->{added} = $added if length($added);
4908     };
4909
4910     my $changes = 0;
4911     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4912         if ($fast_plug_option->{$opt}) {
4913             my $new = delete $conf->{pending}->{$opt};
4914             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4915             $conf->{$opt} = $new;
4916             $changes = 1;
4917         }
4918     }
4919
4920     if ($changes) {
4921         PVE::QemuConfig->write_config($vmid, $conf);
4922     }
4923
4924     my $ostype = $conf->{ostype};
4925     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4926     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4927     my $usb_hotplug = $hotplug_features->{usb}
4928         && min_version($version, 7, 1)
4929         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4930
4931     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4932     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4933
4934     foreach my $opt (sort keys %$pending_delete_hash) {
4935         next if $selection && !$selection->{$opt};
4936         my $force = $pending_delete_hash->{$opt}->{force};
4937         eval {
4938             if ($opt eq 'hotplug') {
4939                 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4940             } elsif ($opt eq 'tablet') {
4941                 die "skip\n" if !$hotplug_features->{usb};
4942                 if ($defaults->{tablet}) {
4943                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4944                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4945                         if $arch eq 'aarch64';
4946                 } else {
4947                     vm_deviceunplug($vmid, $conf, 'tablet');
4948                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4949                 }
4950             } elsif ($opt =~ m/^usb(\d+)$/) {
4951                 my $index = $1;
4952                 die "skip\n" if !$usb_hotplug;
4953                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif ($opt eq 'vcpus') {
4956                 die "skip\n" if !$hotplug_features->{cpu};
4957                 qemu_cpu_hotplug($vmid, $conf, undef);
4958             } elsif ($opt eq 'balloon') {
4959                 # enable balloon device is not hotpluggable
4960                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4961                 # here we reset the ballooning value to memory
4962                 my $balloon = get_current_memory($conf->{memory});
4963                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4964             } elsif ($fast_plug_option->{$opt}) {
4965                 # do nothing
4966             } elsif ($opt =~ m/^net(\d+)$/) {
4967                 die "skip\n" if !$hotplug_features->{network};
4968                 vm_deviceunplug($vmid, $conf, $opt);
4969                 if($have_sdn) {
4970                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
4971                     PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4972                 }
4973             } elsif (is_valid_drivename($opt)) {
4974                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4975                 vm_deviceunplug($vmid, $conf, $opt);
4976                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4977             } elsif ($opt =~ m/^memory$/) {
4978                 die "skip\n" if !$hotplug_features->{memory};
4979                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4980             } elsif ($opt eq 'cpuunits') {
4981                 $cgroup->change_cpu_shares(undef);
4982             } elsif ($opt eq 'cpulimit') {
4983                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4984             } else {
4985                 die "skip\n";
4986             }
4987         };
4988         if (my $err = $@) {
4989             &$add_error($opt, $err) if $err ne "skip\n";
4990         } else {
4991             my $old = delete $conf->{$opt};
4992             $cloudinit_record_changed->($conf, $opt, $old, undef);
4993             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4994         }
4995     }
4996
4997     my $cloudinit_opt;
4998     foreach my $opt (keys %{$conf->{pending}}) {
4999         next if $selection && !$selection->{$opt};
5000         my $value = $conf->{pending}->{$opt};
5001         eval {
5002             if ($opt eq 'hotplug') {
5003                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5004                 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5005             } elsif ($opt eq 'tablet') {
5006                 die "skip\n" if !$hotplug_features->{usb};
5007                 if ($value == 1) {
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5010                         if $arch eq 'aarch64';
5011                 } elsif ($value == 0) {
5012                     vm_deviceunplug($vmid, $conf, 'tablet');
5013                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5014                 }
5015             } elsif ($opt =~ m/^usb(\d+)$/) {
5016                 my $index = $1;
5017                 die "skip\n" if !$usb_hotplug;
5018                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5019                 my $id = $opt;
5020                 if ($d->{host} =~ m/^spice$/i)  {
5021                     $id = "usbredirdev$index";
5022                 }
5023                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5024             } elsif ($opt eq 'vcpus') {
5025                 die "skip\n" if !$hotplug_features->{cpu};
5026                 qemu_cpu_hotplug($vmid, $conf, $value);
5027             } elsif ($opt eq 'balloon') {
5028                 # enable/disable balloning device is not hotpluggable
5029                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5030                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5031                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5032
5033                 # allow manual ballooning if shares is set to zero
5034                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5035                     my $memory = get_current_memory($conf->{memory});
5036                     my $balloon = $conf->{pending}->{balloon} || $memory;
5037                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5038                 }
5039             } elsif ($opt =~ m/^net(\d+)$/) {
5040                 # some changes can be done without hotplug
5041                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5042                                     $vmid, $opt, $value, $arch, $machine_type);
5043             } elsif (is_valid_drivename($opt)) {
5044                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5045                 # some changes can be done without hotplug
5046                 my $drive = parse_drive($opt, $value);
5047                 if (drive_is_cloudinit($drive)) {
5048                     $cloudinit_opt = [$opt, $drive];
5049                     # apply all the other changes first, then generate the cloudinit disk
5050                     die "skip\n";
5051                 }
5052                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5053                                      $vmid, $opt, $value, $arch, $machine_type);
5054             } elsif ($opt =~ m/^memory$/) { #dimms
5055                 die "skip\n" if !$hotplug_features->{memory};
5056                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5057             } elsif ($opt eq 'cpuunits') {
5058                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5059                 $cgroup->change_cpu_shares($new_cpuunits);
5060             } elsif ($opt eq 'cpulimit') {
5061                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5062                 $cgroup->change_cpu_quota($cpulimit, 100000);
5063             } elsif ($opt eq 'agent') {
5064                 vmconfig_update_agent($conf, $opt, $value);
5065             } else {
5066                 die "skip\n";  # skip non-hot-pluggable options
5067             }
5068         };
5069         if (my $err = $@) {
5070             &$add_error($opt, $err) if $err ne "skip\n";
5071         } else {
5072             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5073             $conf->{$opt} = $value;
5074             delete $conf->{pending}->{$opt};
5075         }
5076     }
5077
5078     if (defined($cloudinit_opt)) {
5079         my ($opt, $drive) = @$cloudinit_opt;
5080         my $value = $conf->{pending}->{$opt};
5081         eval {
5082             my $temp = {%$conf, $opt => $value};
5083             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5084             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5085                                  $vmid, $opt, $value, $arch, $machine_type);
5086         };
5087         if (my $err = $@) {
5088             &$add_error($opt, $err) if $err ne "skip\n";
5089         } else {
5090             $conf->{$opt} = $value;
5091             delete $conf->{pending}->{$opt};
5092         }
5093     }
5094
5095     # unplug xhci controller if no usb device is left
5096     if ($usb_hotplug) {
5097         my $has_usb = 0;
5098         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5099             next if !defined($conf->{"usb$i"});
5100             $has_usb = 1;
5101             last;
5102         }
5103         if (!$has_usb) {
5104             vm_deviceunplug($vmid, $conf, 'xhci');
5105         }
5106     }
5107
5108     PVE::QemuConfig->write_config($vmid, $conf);
5109
5110     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5111         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5112     }
5113 }
5114
5115 sub try_deallocate_drive {
5116     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5117
5118     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5119         my $volid = $drive->{file};
5120         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5121             my $sid = PVE::Storage::parse_volume_id($volid);
5122             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5123
5124             # check if the disk is really unused
5125             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5126                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5127             PVE::Storage::vdisk_free($storecfg, $volid);
5128             return 1;
5129         } else {
5130             # If vm is not owner of this disk remove from config
5131             return 1;
5132         }
5133     }
5134
5135     return;
5136 }
5137
5138 sub vmconfig_delete_or_detach_drive {
5139     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5140
5141     my $drive = parse_drive($opt, $conf->{$opt});
5142
5143     my $rpcenv = PVE::RPCEnvironment::get();
5144     my $authuser = $rpcenv->get_user();
5145
5146     if ($force) {
5147         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5148         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5149     } else {
5150         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5151     }
5152 }
5153
5154
5155
5156 sub vmconfig_apply_pending {
5157     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5158
5159     return if !scalar(keys %{$conf->{pending}});
5160
5161     my $add_apply_error = sub {
5162         my ($opt, $msg) = @_;
5163         my $err_msg = "unable to apply pending change $opt : $msg";
5164         $errors->{$opt} = $err_msg;
5165         warn $err_msg;
5166     };
5167
5168     # cold plug
5169
5170     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5171     foreach my $opt (sort keys %$pending_delete_hash) {
5172         my $force = $pending_delete_hash->{$opt}->{force};
5173         eval {
5174             if ($opt =~ m/^unused/) {
5175                 die "internal error";
5176             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5177                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5178             } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5179                 if($have_sdn) {
5180                     my $net = PVE::QemuServer::parse_net($conf->{$opt});
5181                     eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5182                     warn if $@;
5183                 }
5184             }
5185         };
5186         if (my $err = $@) {
5187             $add_apply_error->($opt, $err);
5188         } else {
5189             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5190             delete $conf->{$opt};
5191         }
5192     }
5193
5194     PVE::QemuConfig->cleanup_pending($conf);
5195
5196     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5197
5198     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5199         next if $opt eq 'delete'; # just to be sure
5200         eval {
5201             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5202                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5203             } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5204                 return if !$have_sdn; # return from eval if SDN is not available
5205
5206                 my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5207                 if ($conf->{$opt}) {
5208                     my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5209
5210                     if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (
5211                         safe_string_ne($old_net->{bridge}, $new_net->{bridge}) ||
5212                         safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})
5213                     )) {
5214                         PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5215                     }
5216                 }
5217                 #fixme: reuse ip if mac change && same bridge
5218                 PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5219             }
5220         };
5221         if (my $err = $@) {
5222             $add_apply_error->($opt, $err);
5223         } else {
5224
5225             if (is_valid_drivename($opt)) {
5226                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5227                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5228             }
5229
5230             $conf->{$opt} = delete $conf->{pending}->{$opt};
5231         }
5232     }
5233
5234     # write all changes at once to avoid unnecessary i/o
5235     PVE::QemuConfig->write_config($vmid, $conf);
5236     if ($generate_cloudinit) {
5237         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5238             # After successful generation and if there were changes to be applied, update the
5239             # config to drop the {cloudinit} entry.
5240             PVE::QemuConfig->write_config($vmid, $conf);
5241         }
5242     }
5243 }
5244
5245 sub vmconfig_update_net {
5246     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5247
5248     my $newnet = parse_net($value);
5249
5250     if ($conf->{$opt}) {
5251         my $oldnet = parse_net($conf->{$opt});
5252
5253         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5254             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5255             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5256             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5257             !($newnet->{bridge} && $oldnet->{bridge})
5258         ) { # bridge/nat mode change
5259
5260             # for non online change, we try to hot-unplug
5261             die "skip\n" if !$hotplug;
5262             vm_deviceunplug($vmid, $conf, $opt);
5263
5264             if ($have_sdn) {
5265                 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5266             }
5267
5268         } else {
5269
5270             die "internal error" if $opt !~ m/net(\d+)/;
5271             my $iface = "tap${vmid}i$1";
5272
5273             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5274                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5275                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5276                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})
5277             ) {
5278                 PVE::Network::tap_unplug($iface);
5279
5280                 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5281                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5282                     safe_num_ne($oldnet->{tag}, $newnet->{tag})
5283                 ) {
5284                     qemu_set_link_status($vmid, $opt, 0);
5285                 }
5286
5287                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5288                     if ($have_sdn) {
5289                         PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5290                         PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5291                     }
5292                 }
5293
5294                 if ($have_sdn) {
5295                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5296                 } else {
5297                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5298                 }
5299
5300                 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5301                 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5302                     safe_num_ne($oldnet->{tag}, $newnet->{tag})
5303                 ) {
5304                     qemu_set_link_status($vmid, $opt, 1);
5305                 }
5306
5307             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5308                 # Rate can be applied on its own but any change above needs to
5309                 # include the rate in tap_plug since OVS resets everything.
5310                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5311             }
5312
5313             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5314                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5315             }
5316
5317             return 1;
5318         }
5319     }
5320
5321     if ($hotplug) {
5322         if ($have_sdn) {
5323             PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5324             PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5325         }
5326         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5327     } else {
5328         die "skip\n";
5329     }
5330 }
5331
5332 sub vmconfig_update_agent {
5333     my ($conf, $opt, $value) = @_;
5334
5335     die "skip\n" if !$conf->{$opt};
5336
5337     my $hotplug_options = { fstrim_cloned_disks => 1 };
5338
5339     my $old_agent = parse_guest_agent($conf);
5340     my $agent = parse_guest_agent({$opt => $value});
5341
5342     for my $option (keys %$agent) { # added/changed options
5343         next if defined($hotplug_options->{$option});
5344         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5345     }
5346
5347     for my $option (keys %$old_agent) { # removed options
5348         next if defined($hotplug_options->{$option});
5349         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5350     }
5351
5352     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5353 }
5354
5355 sub vmconfig_update_disk {
5356     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5357
5358     my $drive = parse_drive($opt, $value);
5359
5360     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5361         my $media = $drive->{media} || 'disk';
5362         my $oldmedia = $old_drive->{media} || 'disk';
5363         die "unable to change media type\n" if $media ne $oldmedia;
5364
5365         if (!drive_is_cdrom($old_drive)) {
5366
5367             if ($drive->{file} ne $old_drive->{file}) {
5368
5369                 die "skip\n" if !$hotplug;
5370
5371                 # unplug and register as unused
5372                 vm_deviceunplug($vmid, $conf, $opt);
5373                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5374
5375             } else {
5376                 # update existing disk
5377
5378                 # skip non hotpluggable value
5379                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5380                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5381                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5382                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5383                     safe_string_ne($drive->{product}, $old_drive->{product}) ||
5384                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5385                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5386                     safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5387                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5388                     die "skip\n";
5389                 }
5390
5391                 # apply throttle
5392                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5393                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5394                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5395                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5396                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5397                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5398                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5399                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5400                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5401                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5402                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5403                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5404                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5405                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5406                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5407                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5408                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5409                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5410
5411                     qemu_block_set_io_throttle(
5412                         $vmid,"drive-$opt",
5413                         ($drive->{mbps} || 0)*1024*1024,
5414                         ($drive->{mbps_rd} || 0)*1024*1024,
5415                         ($drive->{mbps_wr} || 0)*1024*1024,
5416                          $drive->{iops} || 0,
5417                          $drive->{iops_rd} || 0,
5418                          $drive->{iops_wr} || 0,
5419                         ($drive->{mbps_max} || 0)*1024*1024,
5420                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5421                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5422                          $drive->{iops_max} || 0,
5423                          $drive->{iops_rd_max} || 0,
5424                          $drive->{iops_wr_max} || 0,
5425                          $drive->{bps_max_length} || 1,
5426                          $drive->{bps_rd_max_length} || 1,
5427                          $drive->{bps_wr_max_length} || 1,
5428                          $drive->{iops_max_length} || 1,
5429                          $drive->{iops_rd_max_length} || 1,
5430                          $drive->{iops_wr_max_length} || 1,
5431                     );
5432
5433                 }
5434
5435                 return 1;
5436             }
5437
5438         } else { # cdrom
5439
5440             if ($drive->{file} eq 'none') {
5441                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5442                 if (drive_is_cloudinit($old_drive)) {
5443                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5444                 }
5445             } else {
5446                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5447
5448                 # force eject if locked
5449                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5450
5451                 if ($path) {
5452                     mon_cmd($vmid, "blockdev-change-medium",
5453                         id => "$opt", filename => "$path");
5454                 }
5455             }
5456
5457             return 1;
5458         }
5459     }
5460
5461     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5462     # hotplug new disks
5463     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5464     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5465 }
5466
5467 sub vmconfig_update_cloudinit_drive {
5468     my ($storecfg, $conf, $vmid) = @_;
5469
5470     my $cloudinit_ds = undef;
5471     my $cloudinit_drive = undef;
5472
5473     PVE::QemuConfig->foreach_volume($conf, sub {
5474         my ($ds, $drive) = @_;
5475         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5476             $cloudinit_ds = $ds;
5477             $cloudinit_drive = $drive;
5478         }
5479     });
5480
5481     return if !$cloudinit_drive;
5482
5483     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5484         PVE::QemuConfig->write_config($vmid, $conf);
5485     }
5486
5487     my $running = PVE::QemuServer::check_running($vmid);
5488
5489     if ($running) {
5490         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5491         if ($path) {
5492             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5493             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5494         }
5495     }
5496 }
5497
5498 # called in locked context by incoming migration
5499 sub vm_migrate_get_nbd_disks {
5500     my ($storecfg, $conf, $replicated_volumes) = @_;
5501
5502     my $local_volumes = {};
5503     PVE::QemuConfig->foreach_volume($conf, sub {
5504         my ($ds, $drive) = @_;
5505
5506         return if drive_is_cdrom($drive);
5507         return if $ds eq 'tpmstate0';
5508
5509         my $volid = $drive->{file};
5510
5511         return if !$volid;
5512
5513         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5514
5515         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5516         return if $scfg->{shared};
5517
5518         my $format = qemu_img_format($scfg, $volname);
5519
5520         # replicated disks re-use existing state via bitmap
5521         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5522         $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5523     });
5524     return $local_volumes;
5525 }
5526
5527 # called in locked context by incoming migration
5528 sub vm_migrate_alloc_nbd_disks {
5529     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5530
5531     my $nbd = {};
5532     foreach my $opt (sort keys %$source_volumes) {
5533         my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5534
5535         if ($use_existing) {
5536             $nbd->{$opt}->{drivestr} = print_drive($drive);
5537             $nbd->{$opt}->{volid} = $volid;
5538             $nbd->{$opt}->{replicated} = 1;
5539             next;
5540         }
5541
5542         $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5543
5544         # order of precedence, filtered by whether storage supports it:
5545         # 1. explicit requested format
5546         # 2. default format of storage
5547         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5548         $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5549
5550         my $size = $drive->{size} / 1024;
5551         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5552         my $newdrive = $drive;
5553         $newdrive->{format} = $format;
5554         $newdrive->{file} = $newvolid;
5555         my $drivestr = print_drive($newdrive);
5556         $nbd->{$opt}->{drivestr} = $drivestr;
5557         $nbd->{$opt}->{volid} = $newvolid;
5558     }
5559
5560     return $nbd;
5561 }
5562
5563 # see vm_start_nolock for parameters, additionally:
5564 # migrate_opts:
5565 #   storagemap = parsed storage map for allocating NBD disks
5566 sub vm_start {
5567     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5568
5569     return PVE::QemuConfig->lock_config($vmid, sub {
5570         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5571
5572         die "you can't start a vm if it's a template\n"
5573             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5574
5575         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5576         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5577
5578         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5579
5580         if ($has_backup_lock && $running) {
5581             # a backup is currently running, attempt to start the guest in the
5582             # existing QEMU instance
5583             return vm_resume($vmid);
5584         }
5585
5586         PVE::QemuConfig->check_lock($conf)
5587             if !($params->{skiplock} || $has_suspended_lock);
5588
5589         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5590
5591         die "VM $vmid already running\n" if $running;
5592
5593         if (my $storagemap = $migrate_opts->{storagemap}) {
5594             my $replicated = $migrate_opts->{replicated_volumes};
5595             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5596             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5597
5598             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5599                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5600             }
5601         }
5602
5603         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5604     });
5605 }
5606
5607
5608 # params:
5609 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5610 #   skiplock => 0/1, skip checking for config lock
5611 #   skiptemplate => 0/1, skip checking whether VM is template
5612 #   forcemachine => to force QEMU machine (rollback/migration)
5613 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5614 #   timeout => in seconds
5615 #   paused => start VM in paused state (backup)
5616 #   resume => resume from hibernation
5617 #   pbs-backing => {
5618 #      sata0 => {
5619 #         repository
5620 #         snapshot
5621 #         keyfile
5622 #         archive
5623 #      },
5624 #      virtio2 => ...
5625 #   }
5626 # migrate_opts:
5627 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5628 #   migratedfrom => source node
5629 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5630 #   network => CIDR of migration network
5631 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5632 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5633 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5634 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5635 #       contained in config
5636 sub vm_start_nolock {
5637     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5638
5639     my $statefile = $params->{statefile};
5640     my $resume = $params->{resume};
5641
5642     my $migratedfrom = $migrate_opts->{migratedfrom};
5643     my $migration_type = $migrate_opts->{type};
5644
5645     my $res = {};
5646
5647     # clean up leftover reboot request files
5648     eval { clear_reboot_request($vmid); };
5649     warn $@ if $@;
5650
5651     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5652         vmconfig_apply_pending($vmid, $conf, $storecfg);
5653         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5654     }
5655
5656     # don't regenerate the ISO if the VM is started as part of a live migration
5657     # this way we can reuse the old ISO with the correct config
5658     if (!$migratedfrom) {
5659         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5660             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5661             # $conf->{cloudinit}, so we could just not do this?
5662             # But we do it above, so for now let's be consistent.
5663             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5664         }
5665     }
5666
5667     # override offline migrated volumes, conf is out of date still
5668     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5669         for my $key (sort keys $offline_volumes->%*) {
5670             my $parsed = parse_drive($key, $conf->{$key});
5671             $parsed->{file} = $offline_volumes->{$key};
5672             $conf->{$key} = print_drive($parsed);
5673         }
5674     }
5675
5676     my $defaults = load_defaults();
5677
5678     # set environment variable useful inside network script
5679     # for remote migration the config is available on the target node!
5680     if (!$migrate_opts->{remote_node}) {
5681         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5682     }
5683
5684     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5685
5686     my $forcemachine = $params->{forcemachine};
5687     my $forcecpu = $params->{forcecpu};
5688     if ($resume) {
5689         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5690         $forcemachine = $conf->{runningmachine};
5691         $forcecpu = $conf->{runningcpu};
5692         print "Resuming suspended VM\n";
5693     }
5694
5695     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5696         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5697
5698     my $migration_ip;
5699     my $get_migration_ip = sub {
5700         my ($nodename) = @_;
5701
5702         return $migration_ip if defined($migration_ip);
5703
5704         my $cidr = $migrate_opts->{network};
5705
5706         if (!defined($cidr)) {
5707             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5708             $cidr = $dc_conf->{migration}->{network};
5709         }
5710
5711         if (defined($cidr)) {
5712             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5713
5714             die "could not get IP: no address configured on local " .
5715                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5716
5717             die "could not get IP: multiple addresses configured on local " .
5718                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5719
5720             $migration_ip = @$ips[0];
5721         }
5722
5723         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5724             if !defined($migration_ip);
5725
5726         return $migration_ip;
5727     };
5728
5729     if ($statefile) {
5730         if ($statefile eq 'tcp') {
5731             my $migrate = $res->{migrate} = { proto => 'tcp' };
5732             $migrate->{addr} = "localhost";
5733             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5734             my $nodename = nodename();
5735
5736             if (!defined($migration_type)) {
5737                 if (defined($datacenterconf->{migration}->{type})) {
5738                     $migration_type = $datacenterconf->{migration}->{type};
5739                 } else {
5740                     $migration_type = 'secure';
5741                 }
5742             }
5743
5744             if ($migration_type eq 'insecure') {
5745                 $migrate->{addr} = $get_migration_ip->($nodename);
5746                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5747             }
5748
5749             # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5750             # via QMP later
5751             push @$cmd, '-incoming', 'defer';
5752             push @$cmd, '-S';
5753
5754         } elsif ($statefile eq 'unix') {
5755             # should be default for secure migrations as a ssh TCP forward
5756             # tunnel is not deterministic reliable ready and fails regurarly
5757             # to set up in time, so use UNIX socket forwards
5758             my $migrate = $res->{migrate} = { proto => 'unix' };
5759             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5760             unlink $migrate->{addr};
5761
5762             $migrate->{uri} = "unix:$migrate->{addr}";
5763             push @$cmd, '-incoming', $migrate->{uri};
5764             push @$cmd, '-S';
5765
5766         } elsif (-e $statefile) {
5767             push @$cmd, '-loadstate', $statefile;
5768         } else {
5769             my $statepath = PVE::Storage::path($storecfg, $statefile);
5770             push @$vollist, $statefile;
5771             push @$cmd, '-loadstate', $statepath;
5772         }
5773     } elsif ($params->{paused}) {
5774         push @$cmd, '-S';
5775     }
5776
5777     my $memory = get_current_memory($conf->{memory});
5778     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5779
5780     my $pci_reserve_list = [];
5781     for my $device (values $pci_devices->%*) {
5782         next if $device->{mdev}; # we don't reserve for mdev devices
5783         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5784     }
5785
5786     # reserve all PCI IDs before actually doing anything with them
5787     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5788
5789     eval {
5790         my $uuid;
5791         for my $id (sort keys %$pci_devices) {
5792             my $d = $pci_devices->{$id};
5793             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5794
5795             my $chosen_mdev;
5796             for my $dev ($d->{ids}->@*) {
5797                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5798                 if ($d->{mdev}) {
5799                     warn $@ if $@;
5800                     $chosen_mdev = $info;
5801                     last if $chosen_mdev; # if successful, we're done
5802                 } else {
5803                     die $@ if $@;
5804                 }
5805             }
5806
5807             next if !$d->{mdev};
5808             die "could not create mediated device\n" if !defined($chosen_mdev);
5809
5810             # nvidia grid needs the uuid of the mdev as qemu parameter
5811             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5812                 if (defined($conf->{smbios1})) {
5813                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5814                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5815                 }
5816                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5817             }
5818         }
5819         push @$cmd, '-uuid', $uuid if defined($uuid);
5820     };
5821     if (my $err = $@) {
5822         eval { cleanup_pci_devices($vmid, $conf) };
5823         warn $@ if $@;
5824         die $err;
5825     }
5826
5827     PVE::Storage::activate_volumes($storecfg, $vollist);
5828
5829
5830     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5831     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5832     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5833     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5834     # timeout should be more than enough here...
5835     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5836
5837     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5838
5839     my %run_params = (
5840         timeout => $statefile ? undef : $start_timeout,
5841         umask => 0077,
5842         noerr => 1,
5843     );
5844
5845     # when migrating, prefix QEMU output so other side can pick up any
5846     # errors that might occur and show the user
5847     if ($migratedfrom) {
5848         $run_params{quiet} = 1;
5849         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5850     }
5851
5852     my %systemd_properties = (
5853         Slice => 'qemu.slice',
5854         KillMode => 'process',
5855         SendSIGKILL => 0,
5856         TimeoutStopUSec => ULONG_MAX, # infinity
5857     );
5858
5859     if (PVE::CGroup::cgroup_mode() == 2) {
5860         $systemd_properties{CPUWeight} = $cpuunits;
5861     } else {
5862         $systemd_properties{CPUShares} = $cpuunits;
5863     }
5864
5865     if (my $cpulimit = $conf->{cpulimit}) {
5866         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5867     }
5868     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5869
5870     my $run_qemu = sub {
5871         PVE::Tools::run_fork sub {
5872             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5873
5874             my $tpmpid;
5875             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5876                 # start the TPM emulator so QEMU can connect on start
5877                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5878             }
5879
5880             my $exitcode = run_command($cmd, %run_params);
5881             if ($exitcode) {
5882                 if ($tpmpid) {
5883                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5884                     kill 'TERM', $tpmpid;
5885                 }
5886                 die "QEMU exited with code $exitcode\n";
5887             }
5888         };
5889     };
5890
5891     if ($conf->{hugepages}) {
5892
5893         my $code = sub {
5894             my $hotplug_features =
5895                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5896             my $hugepages_topology =
5897                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5898
5899             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5900
5901             PVE::QemuServer::Memory::hugepages_mount();
5902             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5903
5904             eval { $run_qemu->() };
5905             if (my $err = $@) {
5906                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5907                     if !$conf->{keephugepages};
5908                 die $err;
5909             }
5910
5911             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5912                 if !$conf->{keephugepages};
5913         };
5914         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5915
5916     } else {
5917         eval { $run_qemu->() };
5918     }
5919
5920     if (my $err = $@) {
5921         # deactivate volumes if start fails
5922         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5923         warn $@ if $@;
5924         eval { cleanup_pci_devices($vmid, $conf) };
5925         warn $@ if $@;
5926
5927         die "start failed: $err";
5928     }
5929
5930     # re-reserve all PCI IDs now that we can know the actual VM PID
5931     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5932     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5933     warn $@ if $@;
5934
5935     if (defined(my $migrate = $res->{migrate})) {
5936         if ($migrate->{proto} eq 'tcp') {
5937             my $nodename = nodename();
5938             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5939             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5940             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5941             mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5942         }
5943         print "migration listens on $migrate->{uri}\n";
5944     } elsif ($statefile) {
5945         eval { mon_cmd($vmid, "cont"); };
5946         warn $@ if $@;
5947     }
5948
5949     #start nbd server for storage migration
5950     if (my $nbd = $migrate_opts->{nbd}) {
5951         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5952
5953         my $migrate_storage_uri;
5954         # nbd_protocol_version > 0 for unix socket support
5955         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5956             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5957             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5958             $migrate_storage_uri = "nbd:unix:$socket_path";
5959             $res->{migrate}->{unix_sockets} = [$socket_path];
5960         } else {
5961             my $nodename = nodename();
5962             my $localip = $get_migration_ip->($nodename);
5963             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5964             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5965
5966             mon_cmd($vmid, "nbd-server-start", addr => {
5967                 type => 'inet',
5968                 data => {
5969                     host => "${localip}",
5970                     port => "${storage_migrate_port}",
5971                 },
5972             });
5973             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5974             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5975         }
5976
5977         my $block_info = mon_cmd($vmid, "query-block");
5978         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5979
5980         foreach my $opt (sort keys %$nbd) {
5981             my $drivestr = $nbd->{$opt}->{drivestr};
5982             my $volid = $nbd->{$opt}->{volid};
5983
5984             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5985
5986             mon_cmd(
5987                 $vmid,
5988                 "block-export-add",
5989                 id => "drive-$opt",
5990                 'node-name' => $block_node,
5991                 writable => JSON::true,
5992                 type => "nbd",
5993                 name => "drive-$opt", # NBD export name
5994             );
5995
5996             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5997             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5998             print "re-using replicated volume: $opt - $volid\n"
5999                 if $nbd->{$opt}->{replicated};
6000
6001             $res->{drives}->{$opt} = $nbd->{$opt};
6002             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6003         }
6004     }
6005
6006     if ($migratedfrom) {
6007         eval {
6008             set_migration_caps($vmid);
6009         };
6010         warn $@ if $@;
6011
6012         if ($spice_port) {
6013             print "spice listens on port $spice_port\n";
6014             $res->{spice_port} = $spice_port;
6015             if ($migrate_opts->{spice_ticket}) {
6016                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6017                     $migrate_opts->{spice_ticket});
6018                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6019             }
6020         }
6021
6022     } else {
6023         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6024             if !$statefile && $conf->{balloon};
6025
6026         foreach my $opt (keys %$conf) {
6027             next if $opt !~  m/^net\d+$/;
6028             my $nicconf = parse_net($conf->{$opt});
6029             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6030         }
6031         add_nets_bridge_fdb($conf, $vmid);
6032     }
6033
6034    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6035         eval {
6036             mon_cmd(
6037                 $vmid,
6038                 'qom-set',
6039                 path => "machine/peripheral/balloon0",
6040                 property => "guest-stats-polling-interval",
6041                 value => 2
6042             );
6043         };
6044         log_warn("could not set polling interval for ballooning - $@") if $@;
6045     }
6046
6047     if ($resume) {
6048         print "Resumed VM, removing state\n";
6049         if (my $vmstate = $conf->{vmstate}) {
6050             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6051             PVE::Storage::vdisk_free($storecfg, $vmstate);
6052         }
6053         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6054         PVE::QemuConfig->write_config($vmid, $conf);
6055     }
6056
6057     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6058
6059     my ($current_machine, $is_deprecated) =
6060         PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6061     if ($is_deprecated) {
6062         log_warn(
6063             "current machine version '$current_machine' is deprecated - see the documentation and ".
6064             "change to a newer one",
6065         );
6066     }
6067
6068     return $res;
6069 }
6070
6071 sub vm_commandline {
6072     my ($storecfg, $vmid, $snapname) = @_;
6073
6074     my $conf = PVE::QemuConfig->load_config($vmid);
6075
6076     my ($forcemachine, $forcecpu);
6077     if ($snapname) {
6078         my $snapshot = $conf->{snapshots}->{$snapname};
6079         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6080
6081         # check for machine or CPU overrides in snapshot
6082         $forcemachine = $snapshot->{runningmachine};
6083         $forcecpu = $snapshot->{runningcpu};
6084
6085         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6086
6087         $conf = $snapshot;
6088     }
6089
6090     my $defaults = load_defaults();
6091
6092     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6093
6094     return PVE::Tools::cmd2string($cmd);
6095 }
6096
6097 sub vm_reset {
6098     my ($vmid, $skiplock) = @_;
6099
6100     PVE::QemuConfig->lock_config($vmid, sub {
6101
6102         my $conf = PVE::QemuConfig->load_config($vmid);
6103
6104         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6105
6106         mon_cmd($vmid, "system_reset");
6107     });
6108 }
6109
6110 sub get_vm_volumes {
6111     my ($conf) = @_;
6112
6113     my $vollist = [];
6114     foreach_volid($conf, sub {
6115         my ($volid, $attr) = @_;
6116
6117         return if $volid =~ m|^/|;
6118
6119         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6120         return if !$sid;
6121
6122         push @$vollist, $volid;
6123     });
6124
6125     return $vollist;
6126 }
6127
6128 sub cleanup_pci_devices {
6129     my ($vmid, $conf) = @_;
6130
6131     foreach my $key (keys %$conf) {
6132         next if $key !~ m/^hostpci(\d+)$/;
6133         my $hostpciindex = $1;
6134         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6135         my $d = parse_hostpci($conf->{$key});
6136         if ($d->{mdev}) {
6137             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6138             # don't want to break ABI just for this two liner
6139             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6140
6141             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6142             # out when we do it first. so wait for up to 10 seconds and then try it manually
6143             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6144                 my $count = 0;
6145                 while (-e $dev_sysfs_dir && $count < 10) {
6146                     sleep 1;
6147                     $count++;
6148                 }
6149                 print "waited $count seconds for mediated device driver finishing clean up\n";
6150             }
6151
6152             if (-e $dev_sysfs_dir) {
6153                 print "actively clean up mediated device with UUID $uuid\n";
6154                 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6155             }
6156         }
6157     }
6158     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6159 }
6160
6161 sub vm_stop_cleanup {
6162     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6163
6164     eval {
6165
6166         if (!$keepActive) {
6167             my $vollist = get_vm_volumes($conf);
6168             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6169
6170             if (my $tpmdrive = $conf->{tpmstate0}) {
6171                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6172                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6173                 if ($storeid) {
6174                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6175                 }
6176             }
6177         }
6178
6179         foreach my $ext (qw(mon qmp pid vnc qga)) {
6180             unlink "/var/run/qemu-server/${vmid}.$ext";
6181         }
6182
6183         if ($conf->{ivshmem}) {
6184             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6185             # just delete it for now, VMs which have this already open do not
6186             # are affected, but new VMs will get a separated one. If this
6187             # becomes an issue we either add some sort of ref-counting or just
6188             # add a "don't delete on stop" flag to the ivshmem format.
6189             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6190         }
6191
6192         cleanup_pci_devices($vmid, $conf);
6193
6194         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6195     };
6196     warn $@ if $@; # avoid errors - just warn
6197 }
6198
6199 # call only in locked context
6200 sub _do_vm_stop {
6201     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6202
6203     my $pid = check_running($vmid, $nocheck);
6204     return if !$pid;
6205
6206     my $conf;
6207     if (!$nocheck) {
6208         $conf = PVE::QemuConfig->load_config($vmid);
6209         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6210         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6211             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6212             $timeout = $opts->{down} if $opts->{down};
6213         }
6214         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6215     }
6216
6217     eval {
6218         if ($shutdown) {
6219             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6220                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6221             } else {
6222                 mon_cmd($vmid, "system_powerdown");
6223             }
6224         } else {
6225             mon_cmd($vmid, "quit");
6226         }
6227     };
6228     my $err = $@;
6229
6230     if (!$err) {
6231         $timeout = 60 if !defined($timeout);
6232
6233         my $count = 0;
6234         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6235             $count++;
6236             sleep 1;
6237         }
6238
6239         if ($count >= $timeout) {
6240             if ($force) {
6241                 warn "VM still running - terminating now with SIGTERM\n";
6242                 kill 15, $pid;
6243             } else {
6244                 die "VM quit/powerdown failed - got timeout\n";
6245             }
6246         } else {
6247             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6248             return;
6249         }
6250     } else {
6251        if (!check_running($vmid, $nocheck)) {
6252             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6253             return;
6254        }
6255        if ($force) {
6256             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6257             kill 15, $pid;
6258         } else {
6259             die "VM quit/powerdown failed\n";
6260         }
6261     }
6262
6263     # wait again
6264     $timeout = 10;
6265
6266     my $count = 0;
6267     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6268         $count++;
6269         sleep 1;
6270     }
6271
6272     if ($count >= $timeout) {
6273         warn "VM still running - terminating now with SIGKILL\n";
6274         kill 9, $pid;
6275         sleep 1;
6276     }
6277
6278     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6279 }
6280
6281 # Note: use $nocheck to skip tests if VM configuration file exists.
6282 # We need that when migration VMs to other nodes (files already moved)
6283 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6284 sub vm_stop {
6285     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6286
6287     $force = 1 if !defined($force) && !$shutdown;
6288
6289     if ($migratedfrom){
6290         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6291         kill 15, $pid if $pid;
6292         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6293         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6294         return;
6295     }
6296
6297     PVE::QemuConfig->lock_config($vmid, sub {
6298         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6299    });
6300 }
6301
6302 sub vm_reboot {
6303     my ($vmid, $timeout) = @_;
6304
6305     PVE::QemuConfig->lock_config($vmid, sub {
6306         eval {
6307
6308             # only reboot if running, as qmeventd starts it again on a stop event
6309             return if !check_running($vmid);
6310
6311             create_reboot_request($vmid);
6312
6313             my $storecfg = PVE::Storage::config();
6314             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6315
6316         };
6317         if (my $err = $@) {
6318             # avoid that the next normal shutdown will be confused for a reboot
6319             clear_reboot_request($vmid);
6320             die $err;
6321         }
6322    });
6323 }
6324
6325 # note: if using the statestorage parameter, the caller has to check privileges
6326 sub vm_suspend {
6327     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6328
6329     my $conf;
6330     my $path;
6331     my $storecfg;
6332     my $vmstate;
6333
6334     PVE::QemuConfig->lock_config($vmid, sub {
6335
6336         $conf = PVE::QemuConfig->load_config($vmid);
6337
6338         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6339         PVE::QemuConfig->check_lock($conf)
6340             if !($skiplock || $is_backing_up);
6341
6342         die "cannot suspend to disk during backup\n"
6343             if $is_backing_up && $includestate;
6344
6345         if ($includestate) {
6346             $conf->{lock} = 'suspending';
6347             my $date = strftime("%Y-%m-%d", localtime(time()));
6348             $storecfg = PVE::Storage::config();
6349             if (!$statestorage) {
6350                 $statestorage = find_vmstate_storage($conf, $storecfg);
6351                 # check permissions for the storage
6352                 my $rpcenv = PVE::RPCEnvironment::get();
6353                 if ($rpcenv->{type} ne 'cli') {
6354                     my $authuser = $rpcenv->get_user();
6355                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6356                 }
6357             }
6358
6359
6360             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6361                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6362             $path = PVE::Storage::path($storecfg, $vmstate);
6363             PVE::QemuConfig->write_config($vmid, $conf);
6364         } else {
6365             mon_cmd($vmid, "stop");
6366         }
6367     });
6368
6369     if ($includestate) {
6370         # save vm state
6371         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6372
6373         eval {
6374             set_migration_caps($vmid, 1);
6375             mon_cmd($vmid, "savevm-start", statefile => $path);
6376             for(;;) {
6377                 my $state = mon_cmd($vmid, "query-savevm");
6378                 if (!$state->{status}) {
6379                     die "savevm not active\n";
6380                 } elsif ($state->{status} eq 'active') {
6381                     sleep(1);
6382                     next;
6383                 } elsif ($state->{status} eq 'completed') {
6384                     print "State saved, quitting\n";
6385                     last;
6386                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6387                     die "query-savevm failed with error '$state->{error}'\n"
6388                 } else {
6389                     die "query-savevm returned status '$state->{status}'\n";
6390                 }
6391             }
6392         };
6393         my $err = $@;
6394
6395         PVE::QemuConfig->lock_config($vmid, sub {
6396             $conf = PVE::QemuConfig->load_config($vmid);
6397             if ($err) {
6398                 # cleanup, but leave suspending lock, to indicate something went wrong
6399                 eval {
6400                     mon_cmd($vmid, "savevm-end");
6401                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6402                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6403                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6404                     PVE::QemuConfig->write_config($vmid, $conf);
6405                 };
6406                 warn $@ if $@;
6407                 die $err;
6408             }
6409
6410             die "lock changed unexpectedly\n"
6411                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6412
6413             mon_cmd($vmid, "quit");
6414             $conf->{lock} = 'suspended';
6415             PVE::QemuConfig->write_config($vmid, $conf);
6416         });
6417     }
6418 }
6419
6420 # $nocheck is set when called as part of a migration - in this context the
6421 # location of the config file (source or target node) is not deterministic,
6422 # since migration cannot wait for pmxcfs to process the rename
6423 sub vm_resume {
6424     my ($vmid, $skiplock, $nocheck) = @_;
6425
6426     PVE::QemuConfig->lock_config($vmid, sub {
6427         my $res = mon_cmd($vmid, 'query-status');
6428         my $resume_cmd = 'cont';
6429         my $reset = 0;
6430         my $conf;
6431         if ($nocheck) {
6432             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6433             if ($@) {
6434                 my $vmlist = PVE::Cluster::get_vmlist();
6435                 if (exists($vmlist->{ids}->{$vmid})) {
6436                     my $node = $vmlist->{ids}->{$vmid}->{node};
6437                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6438                 }
6439                 if (!$conf) {
6440                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6441                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6442                 }
6443             }
6444         } else {
6445             $conf = PVE::QemuConfig->load_config($vmid);
6446         }
6447
6448         if ($res->{status}) {
6449             return if $res->{status} eq 'running'; # job done, go home
6450             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6451             $reset = 1 if $res->{status} eq 'shutdown';
6452         }
6453
6454         if (!$nocheck) {
6455             PVE::QemuConfig->check_lock($conf)
6456                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6457         }
6458
6459         if ($reset) {
6460             # required if a VM shuts down during a backup and we get a resume
6461             # request before the backup finishes for example
6462             mon_cmd($vmid, "system_reset");
6463         }
6464
6465         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6466
6467         mon_cmd($vmid, $resume_cmd);
6468     });
6469 }
6470
6471 sub vm_sendkey {
6472     my ($vmid, $skiplock, $key) = @_;
6473
6474     PVE::QemuConfig->lock_config($vmid, sub {
6475
6476         my $conf = PVE::QemuConfig->load_config($vmid);
6477
6478         # there is no qmp command, so we use the human monitor command
6479         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6480         die $res if $res ne '';
6481     });
6482 }
6483
6484 sub check_bridge_access {
6485     my ($rpcenv, $authuser, $conf) = @_;
6486
6487     return 1 if $authuser eq 'root@pam';
6488
6489     for my $opt (sort keys $conf->%*) {
6490         next if $opt !~ m/^net\d+$/;
6491         my $net = parse_net($conf->{$opt});
6492         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6493         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6494     }
6495     return 1;
6496 };
6497
6498 sub check_mapping_access {
6499     my ($rpcenv, $user, $conf) = @_;
6500
6501     for my $opt (keys $conf->%*) {
6502         if ($opt =~ m/^usb\d+$/) {
6503             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6504             if (my $host = $device->{host}) {
6505                 die "only root can set '$opt' config for real devices\n"
6506                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6507             } elsif ($device->{mapping}) {
6508                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6509             } else {
6510                 die "either 'host' or 'mapping' must be set.\n";
6511             }
6512         } elsif ($opt =~ m/^hostpci\d+$/) {
6513             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6514             if ($device->{host}) {
6515                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6516             } elsif ($device->{mapping}) {
6517                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6518             } else {
6519                 die "either 'host' or 'mapping' must be set.\n";
6520             }
6521        }
6522    }
6523 };
6524
6525 sub check_restore_permissions {
6526     my ($rpcenv, $user, $conf) = @_;
6527
6528     check_bridge_access($rpcenv, $user, $conf);
6529     check_mapping_access($rpcenv, $user, $conf);
6530 }
6531 # vzdump restore implementaion
6532
6533 sub tar_archive_read_firstfile {
6534     my $archive = shift;
6535
6536     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6537
6538     # try to detect archive type first
6539     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6540         die "unable to open file '$archive'\n";
6541     my $firstfile = <$fh>;
6542     kill 15, $pid;
6543     close $fh;
6544
6545     die "ERROR: archive contaions no data\n" if !$firstfile;
6546     chomp $firstfile;
6547
6548     return $firstfile;
6549 }
6550
6551 sub tar_restore_cleanup {
6552     my ($storecfg, $statfile) = @_;
6553
6554     print STDERR "starting cleanup\n";
6555
6556     if (my $fd = IO::File->new($statfile, "r")) {
6557         while (defined(my $line = <$fd>)) {
6558             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6559                 my $volid = $2;
6560                 eval {
6561                     if ($volid =~ m|^/|) {
6562                         unlink $volid || die 'unlink failed\n';
6563                     } else {
6564                         PVE::Storage::vdisk_free($storecfg, $volid);
6565                     }
6566                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6567                 };
6568                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6569             } else {
6570                 print STDERR "unable to parse line in statfile - $line";
6571             }
6572         }
6573         $fd->close();
6574     }
6575 }
6576
6577 sub restore_file_archive {
6578     my ($archive, $vmid, $user, $opts) = @_;
6579
6580     return restore_vma_archive($archive, $vmid, $user, $opts)
6581         if $archive eq '-';
6582
6583     my $info = PVE::Storage::archive_info($archive);
6584     my $format = $opts->{format} // $info->{format};
6585     my $comp = $info->{compression};
6586
6587     # try to detect archive format
6588     if ($format eq 'tar') {
6589         return restore_tar_archive($archive, $vmid, $user, $opts);
6590     } else {
6591         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6592     }
6593 }
6594
6595 # hepler to remove disks that will not be used after restore
6596 my $restore_cleanup_oldconf = sub {
6597     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6598
6599     my $kept_disks = {};
6600
6601     PVE::QemuConfig->foreach_volume($oldconf, sub {
6602         my ($ds, $drive) = @_;
6603
6604         return if drive_is_cdrom($drive, 1);
6605
6606         my $volid = $drive->{file};
6607         return if !$volid || $volid =~ m|^/|;
6608
6609         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6610         return if !$path || !$owner || ($owner != $vmid);
6611
6612         # Note: only delete disk we want to restore
6613         # other volumes will become unused
6614         if ($virtdev_hash->{$ds}) {
6615             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6616             if (my $err = $@) {
6617                 warn $err;
6618             }
6619         } else {
6620             $kept_disks->{$volid} = 1;
6621         }
6622     });
6623
6624     # after the restore we have no snapshots anymore
6625     for my $snapname (keys $oldconf->{snapshots}->%*) {
6626         my $snap = $oldconf->{snapshots}->{$snapname};
6627         if ($snap->{vmstate}) {
6628             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6629             if (my $err = $@) {
6630                 warn $err;
6631             }
6632         }
6633
6634         for my $volid (keys $kept_disks->%*) {
6635             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6636             warn $@ if $@;
6637         }
6638     }
6639 };
6640
6641 # Helper to parse vzdump backup device hints
6642 #
6643 # $rpcenv: Environment, used to ckeck storage permissions
6644 # $user: User ID, to check storage permissions
6645 # $storecfg: Storage configuration
6646 # $fh: the file handle for reading the configuration
6647 # $devinfo: should contain device sizes for all backu-up'ed devices
6648 # $options: backup options (pool, default storage)
6649 #
6650 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6651 my $parse_backup_hints = sub {
6652     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6653
6654     my $check_storage = sub { # assert if an image can be allocate
6655         my ($storeid, $scfg) = @_;
6656         die "Content type 'images' is not available on storage '$storeid'\n"
6657             if !$scfg->{content}->{images};
6658         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6659             if $user ne 'root@pam';
6660     };
6661
6662     my $virtdev_hash = {};
6663     while (defined(my $line = <$fh>)) {
6664         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6665             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6666             die "archive does not contain data for drive '$virtdev'\n"
6667                 if !$devinfo->{$devname};
6668
6669             if (defined($options->{storage})) {
6670                 $storeid = $options->{storage} || 'local';
6671             } elsif (!$storeid) {
6672                 $storeid = 'local';
6673             }
6674             $format = 'raw' if !$format;
6675             $devinfo->{$devname}->{devname} = $devname;
6676             $devinfo->{$devname}->{virtdev} = $virtdev;
6677             $devinfo->{$devname}->{format} = $format;
6678             $devinfo->{$devname}->{storeid} = $storeid;
6679
6680             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6681             $check_storage->($storeid, $scfg); # permission and content type check
6682
6683             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6684         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6685             my $virtdev = $1;
6686             my $drive = parse_drive($virtdev, $2);
6687
6688             if (drive_is_cloudinit($drive)) {
6689                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6690                 $storeid = $options->{storage} if defined ($options->{storage});
6691                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6692                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6693
6694                 $check_storage->($storeid, $scfg); # permission and content type check
6695
6696                 $virtdev_hash->{$virtdev} = {
6697                     format => $format,
6698                     storeid => $storeid,
6699                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6700                     is_cloudinit => 1,
6701                 };
6702             }
6703         }
6704     }
6705
6706     return $virtdev_hash;
6707 };
6708
6709 # Helper to allocate and activate all volumes required for a restore
6710 #
6711 # $storecfg: Storage configuration
6712 # $virtdev_hash: as returned by parse_backup_hints()
6713 #
6714 # Returns: { $virtdev => $volid }
6715 my $restore_allocate_devices = sub {
6716     my ($storecfg, $virtdev_hash, $vmid) = @_;
6717
6718     my $map = {};
6719     foreach my $virtdev (sort keys %$virtdev_hash) {
6720         my $d = $virtdev_hash->{$virtdev};
6721         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6722         my $storeid = $d->{storeid};
6723         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6724
6725         # test if requested format is supported
6726         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6727         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6728         $d->{format} = $defFormat if !$supported;
6729
6730         my $name;
6731         if ($d->{is_cloudinit}) {
6732             $name = "vm-$vmid-cloudinit";
6733             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6734             if ($scfg->{path}) {
6735                 $name .= ".$d->{format}";
6736             }
6737         }
6738
6739         my $volid = PVE::Storage::vdisk_alloc(
6740             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6741
6742         print STDERR "new volume ID is '$volid'\n";
6743         $d->{volid} = $volid;
6744
6745         PVE::Storage::activate_volumes($storecfg, [$volid]);
6746
6747         $map->{$virtdev} = $volid;
6748     }
6749
6750     return $map;
6751 };
6752
6753 sub restore_update_config_line {
6754     my ($cookie, $map, $line, $unique) = @_;
6755
6756     return '' if $line =~ m/^\#qmdump\#/;
6757     return '' if $line =~ m/^\#vzdump\#/;
6758     return '' if $line =~ m/^lock:/;
6759     return '' if $line =~ m/^unused\d+:/;
6760     return '' if $line =~ m/^parent:/;
6761
6762     my $res = '';
6763
6764     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6765     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6766         # try to convert old 1.X settings
6767         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6768         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6769             my ($model, $macaddr) = split(/\=/, $devconfig);
6770             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6771             my $net = {
6772                 model => $model,
6773                 bridge => "vmbr$ind",
6774                 macaddr => $macaddr,
6775             };
6776             my $netstr = print_net($net);
6777
6778             $res .= "net$cookie->{netcount}: $netstr\n";
6779             $cookie->{netcount}++;
6780         }
6781     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6782         my ($id, $netstr) = ($1, $2);
6783         my $net = parse_net($netstr);
6784         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6785         $netstr = print_net($net);
6786         $res .= "$id: $netstr\n";
6787     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6788         my $virtdev = $1;
6789         my $value = $3;
6790         my $di = parse_drive($virtdev, $value);
6791         if (defined($di->{backup}) && !$di->{backup}) {
6792             $res .= "#$line";
6793         } elsif ($map->{$virtdev}) {
6794             delete $di->{format}; # format can change on restore
6795             $di->{file} = $map->{$virtdev};
6796             $value = print_drive($di);
6797             $res .= "$virtdev: $value\n";
6798         } else {
6799             $res .= $line;
6800         }
6801     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6802         my $vmgenid = $1;
6803         if ($vmgenid ne '0') {
6804             # always generate a new vmgenid if there was a valid one setup
6805             $vmgenid = generate_uuid();
6806         }
6807         $res .= "vmgenid: $vmgenid\n";
6808     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6809         my ($uuid, $uuid_str);
6810         UUID::generate($uuid);
6811         UUID::unparse($uuid, $uuid_str);
6812         my $smbios1 = parse_smbios1($2);
6813         $smbios1->{uuid} = $uuid_str;
6814         $res .= $1.print_smbios1($smbios1)."\n";
6815     } else {
6816         $res .= $line;
6817     }
6818
6819     return $res;
6820 }
6821
6822 my $restore_deactivate_volumes = sub {
6823     my ($storecfg, $virtdev_hash) = @_;
6824
6825     my $vollist = [];
6826     for my $dev (values $virtdev_hash->%*) {
6827         push $vollist->@*, $dev->{volid} if $dev->{volid};
6828     }
6829
6830     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6831     print STDERR $@ if $@;
6832 };
6833
6834 my $restore_destroy_volumes = sub {
6835     my ($storecfg, $virtdev_hash) = @_;
6836
6837     for my $dev (values $virtdev_hash->%*) {
6838         my $volid = $dev->{volid} or next;
6839         eval {
6840             PVE::Storage::vdisk_free($storecfg, $volid);
6841             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6842         };
6843         print STDERR "unable to cleanup '$volid' - $@" if $@;
6844     }
6845 };
6846
6847 sub restore_merge_config {
6848     my ($filename, $backup_conf_raw, $override_conf) = @_;
6849
6850     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6851     for my $key (keys $override_conf->%*) {
6852         $backup_conf->{$key} = $override_conf->{$key};
6853     }
6854
6855     return $backup_conf;
6856 }
6857
6858 sub scan_volids {
6859     my ($cfg, $vmid) = @_;
6860
6861     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6862
6863     my $volid_hash = {};
6864     foreach my $storeid (keys %$info) {
6865         foreach my $item (@{$info->{$storeid}}) {
6866             next if !($item->{volid} && $item->{size});
6867             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6868             $volid_hash->{$item->{volid}} = $item;
6869         }
6870     }
6871
6872     return $volid_hash;
6873 }
6874
6875 sub update_disk_config {
6876     my ($vmid, $conf, $volid_hash) = @_;
6877
6878     my $changes;
6879     my $prefix = "VM $vmid";
6880
6881     # used and unused disks
6882     my $referenced = {};
6883
6884     # Note: it is allowed to define multiple storages with same path (alias), so
6885     # we need to check both 'volid' and real 'path' (two different volid can point
6886     # to the same path).
6887
6888     my $referencedpath = {};
6889
6890     # update size info
6891     PVE::QemuConfig->foreach_volume($conf, sub {
6892         my ($opt, $drive) = @_;
6893
6894         my $volid = $drive->{file};
6895         return if !$volid;
6896         my $volume = $volid_hash->{$volid};
6897
6898         # mark volid as "in-use" for next step
6899         $referenced->{$volid} = 1;
6900         if ($volume && (my $path = $volume->{path})) {
6901             $referencedpath->{$path} = 1;
6902         }
6903
6904         return if drive_is_cdrom($drive);
6905         return if !$volume;
6906
6907         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6908         if (defined($updated)) {
6909             $changes = 1;
6910             $conf->{$opt} = print_drive($updated);
6911             print "$prefix ($opt): $msg\n";
6912         }
6913     });
6914
6915     # remove 'unusedX' entry if volume is used
6916     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6917         my ($opt, $drive) = @_;
6918
6919         my $volid = $drive->{file};
6920         return if !$volid;
6921
6922         my $path;
6923         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6924         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6925             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6926             $changes = 1;
6927             delete $conf->{$opt};
6928         }
6929
6930         $referenced->{$volid} = 1;
6931         $referencedpath->{$path} = 1 if $path;
6932     });
6933
6934     foreach my $volid (sort keys %$volid_hash) {
6935         next if $volid =~ m/vm-$vmid-state-/;
6936         next if $referenced->{$volid};
6937         my $path = $volid_hash->{$volid}->{path};
6938         next if !$path; # just to be sure
6939         next if $referencedpath->{$path};
6940         $changes = 1;
6941         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6942         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6943         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6944     }
6945
6946     return $changes;
6947 }
6948
6949 sub rescan {
6950     my ($vmid, $nolock, $dryrun) = @_;
6951
6952     my $cfg = PVE::Storage::config();
6953
6954     print "rescan volumes...\n";
6955     my $volid_hash = scan_volids($cfg, $vmid);
6956
6957     my $updatefn =  sub {
6958         my ($vmid) = @_;
6959
6960         my $conf = PVE::QemuConfig->load_config($vmid);
6961
6962         PVE::QemuConfig->check_lock($conf);
6963
6964         my $vm_volids = {};
6965         foreach my $volid (keys %$volid_hash) {
6966             my $info = $volid_hash->{$volid};
6967             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6968         }
6969
6970         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6971
6972         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6973     };
6974
6975     if (defined($vmid)) {
6976         if ($nolock) {
6977             &$updatefn($vmid);
6978         } else {
6979             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6980         }
6981     } else {
6982         my $vmlist = config_list();
6983         foreach my $vmid (keys %$vmlist) {
6984             if ($nolock) {
6985                 &$updatefn($vmid);
6986             } else {
6987                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6988             }
6989         }
6990     }
6991 }
6992
6993 sub restore_proxmox_backup_archive {
6994     my ($archive, $vmid, $user, $options) = @_;
6995
6996     my $storecfg = PVE::Storage::config();
6997
6998     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6999     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7000
7001     my $fingerprint = $scfg->{fingerprint};
7002     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7003
7004     my $repo = PVE::PBSClient::get_repository($scfg);
7005     my $namespace = $scfg->{namespace};
7006
7007     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7008     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7009     local $ENV{PBS_PASSWORD} = $password;
7010     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7011
7012     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7013         PVE::Storage::parse_volname($storecfg, $archive);
7014
7015     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7016
7017     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7018
7019     my $tmpdir = "/var/tmp/vzdumptmp$$";
7020     rmtree $tmpdir;
7021     mkpath $tmpdir;
7022
7023     my $conffile = PVE::QemuConfig->config_file($vmid);
7024      # disable interrupts (always do cleanups)
7025     local $SIG{INT} =
7026         local $SIG{TERM} =
7027         local $SIG{QUIT} =
7028         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7029
7030     # Note: $oldconf is undef if VM does not exists
7031     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7032     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7033     my $new_conf_raw = '';
7034
7035     my $rpcenv = PVE::RPCEnvironment::get();
7036     my $devinfo = {}; # info about drives included in backup
7037     my $virtdev_hash = {}; # info about allocated drives
7038
7039     eval {
7040         # enable interrupts
7041         local $SIG{INT} =
7042             local $SIG{TERM} =
7043             local $SIG{QUIT} =
7044             local $SIG{HUP} =
7045             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7046
7047         my $cfgfn = "$tmpdir/qemu-server.conf";
7048         my $firewall_config_fn = "$tmpdir/fw.conf";
7049         my $index_fn = "$tmpdir/index.json";
7050
7051         my $cmd = "restore";
7052
7053         my $param = [$pbs_backup_name, "index.json", $index_fn];
7054         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7055         my $index = PVE::Tools::file_get_contents($index_fn);
7056         $index = decode_json($index);
7057
7058         foreach my $info (@{$index->{files}}) {
7059             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7060                 my $devname = $1;
7061                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7062                     $devinfo->{$devname}->{size} = $1;
7063                 } else {
7064                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7065                 }
7066             }
7067         }
7068
7069         my $is_qemu_server_backup = scalar(
7070             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7071         );
7072         if (!$is_qemu_server_backup) {
7073             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7074         }
7075         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7076
7077         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7078         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7079
7080         if ($has_firewall_config) {
7081             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7082             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7083
7084             my $pve_firewall_dir = '/etc/pve/firewall';
7085             mkdir $pve_firewall_dir; # make sure the dir exists
7086             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7087         }
7088
7089         my $fh = IO::File->new($cfgfn, "r") ||
7090             die "unable to read qemu-server.conf - $!\n";
7091
7092         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7093
7094         # fixme: rate limit?
7095
7096         # create empty/temp config
7097         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7098
7099         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7100
7101         # allocate volumes
7102         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7103
7104         foreach my $virtdev (sort keys %$virtdev_hash) {
7105             my $d = $virtdev_hash->{$virtdev};
7106             next if $d->{is_cloudinit}; # no need to restore cloudinit
7107
7108             # this fails if storage is unavailable
7109             my $volid = $d->{volid};
7110             my $path = PVE::Storage::path($storecfg, $volid);
7111
7112             # for live-restore we only want to preload the efidisk and TPM state
7113             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7114
7115             my @ns_arg;
7116             if (defined(my $ns = $scfg->{namespace})) {
7117                 @ns_arg = ('--ns', $ns);
7118             }
7119
7120             my $pbs_restore_cmd = [
7121                 '/usr/bin/pbs-restore',
7122                 '--repository', $repo,
7123                 @ns_arg,
7124                 $pbs_backup_name,
7125                 "$d->{devname}.img.fidx",
7126                 $path,
7127                 '--verbose',
7128                 ];
7129
7130             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7131             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7132
7133             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7134                 push @$pbs_restore_cmd, '--skip-zero';
7135             }
7136
7137             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7138             print "restore proxmox backup image: $dbg_cmdstring\n";
7139             run_command($pbs_restore_cmd);
7140         }
7141
7142         $fh->seek(0, 0) || die "seek failed - $!\n";
7143
7144         my $cookie = { netcount => 0 };
7145         while (defined(my $line = <$fh>)) {
7146             $new_conf_raw .= restore_update_config_line(
7147                 $cookie,
7148                 $map,
7149                 $line,
7150                 $options->{unique},
7151             );
7152         }
7153
7154         $fh->close();
7155     };
7156     my $err = $@;
7157
7158     if ($err || !$options->{live}) {
7159         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7160     }
7161
7162     rmtree $tmpdir;
7163
7164     if ($err) {
7165         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7166         die $err;
7167     }
7168
7169     if ($options->{live}) {
7170         # keep lock during live-restore
7171         $new_conf_raw .= "\nlock: create";
7172     }
7173
7174     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7175     check_restore_permissions($rpcenv, $user, $new_conf);
7176     PVE::QemuConfig->write_config($vmid, $new_conf);
7177
7178     eval { rescan($vmid, 1); };
7179     warn $@ if $@;
7180
7181     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7182
7183     if ($options->{live}) {
7184         # enable interrupts
7185         local $SIG{INT} =
7186             local $SIG{TERM} =
7187             local $SIG{QUIT} =
7188             local $SIG{HUP} =
7189             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7190
7191         my $conf = PVE::QemuConfig->load_config($vmid);
7192         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7193
7194         # these special drives are already restored before start
7195         delete $devinfo->{'drive-efidisk0'};
7196         delete $devinfo->{'drive-tpmstate0-backup'};
7197
7198         my $pbs_opts = {
7199             repo => $repo,
7200             keyfile => $keyfile,
7201             snapshot => $pbs_backup_name,
7202             namespace => $namespace,
7203         };
7204         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7205
7206         PVE::QemuConfig->remove_lock($vmid, "create");
7207     }
7208 }
7209
7210 sub pbs_live_restore {
7211     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7212
7213     print "starting VM for live-restore\n";
7214     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7215
7216     my $pbs_backing = {};
7217     for my $ds (keys %$restored_disks) {
7218         $ds =~ m/^drive-(.*)$/;
7219         my $confname = $1;
7220         $pbs_backing->{$confname} = {
7221             repository => $opts->{repo},
7222             snapshot => $opts->{snapshot},
7223             archive => "$ds.img.fidx",
7224         };
7225         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7226         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7227
7228         my $drive = parse_drive($confname, $conf->{$confname});
7229         print "restoring '$ds' to '$drive->{file}'\n";
7230     }
7231
7232     my $drives_streamed = 0;
7233     eval {
7234         # make sure HA doesn't interrupt our restore by stopping the VM
7235         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7236             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7237         }
7238
7239         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7240         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7241         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7242
7243         my $qmeventd_fd = register_qmeventd_handle($vmid);
7244
7245         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7246         # this will effectively collapse the backing image chain consisting of
7247         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7248         # removes itself once all backing images vanish with 'auto-remove=on')
7249         my $jobs = {};
7250         for my $ds (sort keys %$restored_disks) {
7251             my $job_id = "restore-$ds";
7252             mon_cmd($vmid, 'block-stream',
7253                 'job-id' => $job_id,
7254                 device => "$ds",
7255             );
7256             $jobs->{$job_id} = {};
7257         }
7258
7259         mon_cmd($vmid, 'cont');
7260         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7261
7262         print "restore-drive jobs finished successfully, removing all tracking block devices"
7263             ." to disconnect from Proxmox Backup Server\n";
7264
7265         for my $ds (sort keys %$restored_disks) {
7266             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7267         }
7268
7269         close($qmeventd_fd);
7270     };
7271
7272     my $err = $@;
7273
7274     if ($err) {
7275         warn "An error occurred during live-restore: $err\n";
7276         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7277         die "live-restore failed\n";
7278     }
7279 }
7280
7281 sub restore_vma_archive {
7282     my ($archive, $vmid, $user, $opts, $comp) = @_;
7283
7284     my $readfrom = $archive;
7285
7286     my $cfg = PVE::Storage::config();
7287     my $commands = [];
7288     my $bwlimit = $opts->{bwlimit};
7289
7290     my $dbg_cmdstring = '';
7291     my $add_pipe = sub {
7292         my ($cmd) = @_;
7293         push @$commands, $cmd;
7294         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7295         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7296         $readfrom = '-';
7297     };
7298
7299     my $input = undef;
7300     if ($archive eq '-') {
7301         $input = '<&STDIN';
7302     } else {
7303         # If we use a backup from a PVE defined storage we also consider that
7304         # storage's rate limit:
7305         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7306         if (defined($volid)) {
7307             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7308             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7309             if ($readlimit) {
7310                 print STDERR "applying read rate limit: $readlimit\n";
7311                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7312                 $add_pipe->($cstream);
7313             }
7314         }
7315     }
7316
7317     if ($comp) {
7318         my $info = PVE::Storage::decompressor_info('vma', $comp);
7319         my $cmd = $info->{decompressor};
7320         push @$cmd, $readfrom;
7321         $add_pipe->($cmd);
7322     }
7323
7324     my $tmpdir = "/var/tmp/vzdumptmp$$";
7325     rmtree $tmpdir;
7326
7327     # disable interrupts (always do cleanups)
7328     local $SIG{INT} =
7329         local $SIG{TERM} =
7330         local $SIG{QUIT} =
7331         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7332
7333     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7334     POSIX::mkfifo($mapfifo, 0600);
7335     my $fifofh;
7336     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7337
7338     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7339
7340     my $devinfo = {}; # info about drives included in backup
7341     my $virtdev_hash = {}; # info about allocated drives
7342
7343     my $rpcenv = PVE::RPCEnvironment::get();
7344
7345     my $conffile = PVE::QemuConfig->config_file($vmid);
7346
7347     # Note: $oldconf is undef if VM does not exist
7348     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7349     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7350     my $new_conf_raw = '';
7351
7352     my %storage_limits;
7353
7354     my $print_devmap = sub {
7355         my $cfgfn = "$tmpdir/qemu-server.conf";
7356
7357         # we can read the config - that is already extracted
7358         my $fh = IO::File->new($cfgfn, "r") ||
7359             die "unable to read qemu-server.conf - $!\n";
7360
7361         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7362         if (-f $fwcfgfn) {
7363             my $pve_firewall_dir = '/etc/pve/firewall';
7364             mkdir $pve_firewall_dir; # make sure the dir exists
7365             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7366         }
7367
7368         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7369
7370         foreach my $info (values %{$virtdev_hash}) {
7371             my $storeid = $info->{storeid};
7372             next if defined($storage_limits{$storeid});
7373
7374             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7375             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7376             $storage_limits{$storeid} = $limit * 1024;
7377         }
7378
7379         foreach my $devname (keys %$devinfo) {
7380             die "found no device mapping information for device '$devname'\n"
7381                 if !$devinfo->{$devname}->{virtdev};
7382         }
7383
7384         # create empty/temp config
7385         if ($oldconf) {
7386             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7387             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7388         }
7389
7390         # allocate volumes
7391         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7392
7393         # print restore information to $fifofh
7394         foreach my $virtdev (sort keys %$virtdev_hash) {
7395             my $d = $virtdev_hash->{$virtdev};
7396             next if $d->{is_cloudinit}; # no need to restore cloudinit
7397
7398             my $storeid = $d->{storeid};
7399             my $volid = $d->{volid};
7400
7401             my $map_opts = '';
7402             if (my $limit = $storage_limits{$storeid}) {
7403                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7404             }
7405
7406             my $write_zeros = 1;
7407             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7408                 $write_zeros = 0;
7409             }
7410
7411             my $path = PVE::Storage::path($cfg, $volid);
7412
7413             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7414
7415             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7416         }
7417
7418         $fh->seek(0, 0) || die "seek failed - $!\n";
7419
7420         my $cookie = { netcount => 0 };
7421         while (defined(my $line = <$fh>)) {
7422             $new_conf_raw .= restore_update_config_line(
7423                 $cookie,
7424                 $map,
7425                 $line,
7426                 $opts->{unique},
7427             );
7428         }
7429
7430         $fh->close();
7431     };
7432
7433     my $oldtimeout;
7434
7435     eval {
7436         # enable interrupts
7437         local $SIG{INT} =
7438             local $SIG{TERM} =
7439             local $SIG{QUIT} =
7440             local $SIG{HUP} =
7441             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7442         local $SIG{ALRM} = sub { die "got timeout\n"; };
7443
7444         $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7445
7446         my $parser = sub {
7447             my $line = shift;
7448
7449             print "$line\n";
7450
7451             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7452                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7453                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7454             } elsif ($line =~ m/^CTIME: /) {
7455                 # we correctly received the vma config, so we can disable
7456                 # the timeout now for disk allocation
7457                 alarm($oldtimeout || 0);
7458                 $oldtimeout = undef;
7459                 &$print_devmap();
7460                 print $fifofh "done\n";
7461                 close($fifofh);
7462                 $fifofh = undef;
7463             }
7464         };
7465
7466         print "restore vma archive: $dbg_cmdstring\n";
7467         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7468     };
7469     my $err = $@;
7470
7471     alarm($oldtimeout) if $oldtimeout;
7472
7473     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7474
7475     close($fifofh) if $fifofh;
7476     unlink $mapfifo;
7477     rmtree $tmpdir;
7478
7479     if ($err) {
7480         $restore_destroy_volumes->($cfg, $virtdev_hash);
7481         die $err;
7482     }
7483
7484     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7485     check_restore_permissions($rpcenv, $user, $new_conf);
7486     PVE::QemuConfig->write_config($vmid, $new_conf);
7487
7488     eval { rescan($vmid, 1); };
7489     warn $@ if $@;
7490
7491     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7492 }
7493
7494 sub restore_tar_archive {
7495     my ($archive, $vmid, $user, $opts) = @_;
7496
7497     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7498         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7499         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7500     }
7501
7502     if ($archive ne '-') {
7503         my $firstfile = tar_archive_read_firstfile($archive);
7504         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7505             if $firstfile ne 'qemu-server.conf';
7506     }
7507
7508     my $storecfg = PVE::Storage::config();
7509
7510     # avoid zombie disks when restoring over an existing VM -> cleanup first
7511     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7512     # skiplock=1 because qmrestore has set the 'create' lock itself already
7513     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7514     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7515
7516     my $tocmd = "/usr/lib/qemu-server/qmextract";
7517
7518     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7519     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7520     $tocmd .= ' --prealloc' if $opts->{prealloc};
7521     $tocmd .= ' --info' if $opts->{info};
7522
7523     # tar option "xf" does not autodetect compression when read from STDIN,
7524     # so we pipe to zcat
7525     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7526         PVE::Tools::shellquote("--to-command=$tocmd");
7527
7528     my $tmpdir = "/var/tmp/vzdumptmp$$";
7529     mkpath $tmpdir;
7530
7531     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7532     local $ENV{VZDUMP_VMID} = $vmid;
7533     local $ENV{VZDUMP_USER} = $user;
7534
7535     my $conffile = PVE::QemuConfig->config_file($vmid);
7536     my $new_conf_raw = '';
7537
7538     # disable interrupts (always do cleanups)
7539     local $SIG{INT} =
7540         local $SIG{TERM} =
7541         local $SIG{QUIT} =
7542         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7543
7544     eval {
7545         # enable interrupts
7546         local $SIG{INT} =
7547             local $SIG{TERM} =
7548             local $SIG{QUIT} =
7549             local $SIG{HUP} =
7550             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7551
7552         if ($archive eq '-') {
7553             print "extracting archive from STDIN\n";
7554             run_command($cmd, input => "<&STDIN");
7555         } else {
7556             print "extracting archive '$archive'\n";
7557             run_command($cmd);
7558         }
7559
7560         return if $opts->{info};
7561
7562         # read new mapping
7563         my $map = {};
7564         my $statfile = "$tmpdir/qmrestore.stat";
7565         if (my $fd = IO::File->new($statfile, "r")) {
7566             while (defined (my $line = <$fd>)) {
7567                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7568                     $map->{$1} = $2 if $1;
7569                 } else {
7570                     print STDERR "unable to parse line in statfile - $line\n";
7571                 }
7572             }
7573             $fd->close();
7574         }
7575
7576         my $confsrc = "$tmpdir/qemu-server.conf";
7577
7578         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7579
7580         my $cookie = { netcount => 0 };
7581         while (defined (my $line = <$srcfd>)) {
7582             $new_conf_raw .= restore_update_config_line(
7583                 $cookie,
7584                 $map,
7585                 $line,
7586                 $opts->{unique},
7587             );
7588         }
7589
7590         $srcfd->close();
7591     };
7592     if (my $err = $@) {
7593         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7594         die $err;
7595     }
7596
7597     rmtree $tmpdir;
7598
7599     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7600
7601     PVE::Cluster::cfs_update(); # make sure we read new file
7602
7603     eval { rescan($vmid, 1); };
7604     warn $@ if $@;
7605 };
7606
7607 sub foreach_storage_used_by_vm {
7608     my ($conf, $func) = @_;
7609
7610     my $sidhash = {};
7611
7612     PVE::QemuConfig->foreach_volume($conf, sub {
7613         my ($ds, $drive) = @_;
7614         return if drive_is_cdrom($drive);
7615
7616         my $volid = $drive->{file};
7617
7618         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7619         $sidhash->{$sid} = $sid if $sid;
7620     });
7621
7622     foreach my $sid (sort keys %$sidhash) {
7623         &$func($sid);
7624     }
7625 }
7626
7627 my $qemu_snap_storage = {
7628     rbd => 1,
7629 };
7630 sub do_snapshots_with_qemu {
7631     my ($storecfg, $volid, $deviceid) = @_;
7632
7633     return if $deviceid =~ m/tpmstate0/;
7634
7635     my $storage_name = PVE::Storage::parse_volume_id($volid);
7636     my $scfg = $storecfg->{ids}->{$storage_name};
7637     die "could not find storage '$storage_name'\n" if !defined($scfg);
7638
7639     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7640         return 1;
7641     }
7642
7643     if ($volid =~ m/\.(qcow2|qed)$/){
7644         return 1;
7645     }
7646
7647     return;
7648 }
7649
7650 sub qga_check_running {
7651     my ($vmid, $nowarn) = @_;
7652
7653     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7654     if ($@) {
7655         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7656         return 0;
7657     }
7658     return 1;
7659 }
7660
7661 sub template_create {
7662     my ($vmid, $conf, $disk) = @_;
7663
7664     my $storecfg = PVE::Storage::config();
7665
7666     PVE::QemuConfig->foreach_volume($conf, sub {
7667         my ($ds, $drive) = @_;
7668
7669         return if drive_is_cdrom($drive);
7670         return if $disk && $ds ne $disk;
7671
7672         my $volid = $drive->{file};
7673         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7674
7675         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7676         $drive->{file} = $voliddst;
7677         $conf->{$ds} = print_drive($drive);
7678         PVE::QemuConfig->write_config($vmid, $conf);
7679     });
7680 }
7681
7682 sub convert_iscsi_path {
7683     my ($path) = @_;
7684
7685     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7686         my $portal = $1;
7687         my $target = $2;
7688         my $lun = $3;
7689
7690         my $initiator_name = get_initiator_name();
7691
7692         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7693                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7694     }
7695
7696     die "cannot convert iscsi path '$path', unkown format\n";
7697 }
7698
7699 sub qemu_img_convert {
7700     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7701
7702     my $storecfg = PVE::Storage::config();
7703     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7704     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7705
7706     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7707
7708     my $cachemode;
7709     my $src_path;
7710     my $src_is_iscsi = 0;
7711     my $src_format;
7712
7713     if ($src_storeid) {
7714         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7715         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7716         $src_format = qemu_img_format($src_scfg, $src_volname);
7717         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7718         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7719         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7720     } elsif (-f $src_volid || -b $src_volid) {
7721         $src_path = $src_volid;
7722         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7723             $src_format = $1;
7724         }
7725     }
7726
7727     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7728
7729     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7730     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7731     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7732     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7733
7734     my $cmd = [];
7735     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7736     push @$cmd, '-l', "snapshot.name=$snapname"
7737         if $snapname && $src_format && $src_format eq "qcow2";
7738     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7739     push @$cmd, '-T', $cachemode if defined($cachemode);
7740     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7741
7742     if ($src_is_iscsi) {
7743         push @$cmd, '--image-opts';
7744         $src_path = convert_iscsi_path($src_path);
7745     } elsif ($src_format) {
7746         push @$cmd, '-f', $src_format;
7747     }
7748
7749     if ($dst_is_iscsi) {
7750         push @$cmd, '--target-image-opts';
7751         $dst_path = convert_iscsi_path($dst_path);
7752     } else {
7753         push @$cmd, '-O', $dst_format;
7754     }
7755
7756     push @$cmd, $src_path;
7757
7758     if (!$dst_is_iscsi && $is_zero_initialized) {
7759         push @$cmd, "zeroinit:$dst_path";
7760     } else {
7761         push @$cmd, $dst_path;
7762     }
7763
7764     my $parser = sub {
7765         my $line = shift;
7766         if($line =~ m/\((\S+)\/100\%\)/){
7767             my $percent = $1;
7768             my $transferred = int($size * $percent / 100);
7769             my $total_h = render_bytes($size, 1);
7770             my $transferred_h = render_bytes($transferred, 1);
7771
7772             print "transferred $transferred_h of $total_h ($percent%)\n";
7773         }
7774
7775     };
7776
7777     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7778     my $err = $@;
7779     die "copy failed: $err" if $err;
7780 }
7781
7782 sub qemu_img_format {
7783     my ($scfg, $volname) = @_;
7784
7785     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7786         return $1;
7787     } else {
7788         return "raw";
7789     }
7790 }
7791
7792 sub qemu_drive_mirror {
7793     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7794
7795     $jobs = {} if !$jobs;
7796
7797     my $qemu_target;
7798     my $format;
7799     $jobs->{"drive-$drive"} = {};
7800
7801     if ($dst_volid =~ /^nbd:/) {
7802         $qemu_target = $dst_volid;
7803         $format = "nbd";
7804     } else {
7805         my $storecfg = PVE::Storage::config();
7806         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7807
7808         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7809
7810         $format = qemu_img_format($dst_scfg, $dst_volname);
7811
7812         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7813
7814         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7815     }
7816
7817     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7818     $opts->{format} = $format if $format;
7819
7820     if (defined($src_bitmap)) {
7821         $opts->{sync} = 'incremental';
7822         $opts->{bitmap} = $src_bitmap;
7823         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7824     }
7825
7826     if (defined($bwlimit)) {
7827         $opts->{speed} = $bwlimit * 1024;
7828         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7829     } else {
7830         print "drive mirror is starting for drive-$drive\n";
7831     }
7832
7833     # if a job already runs for this device we get an error, catch it for cleanup
7834     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7835     if (my $err = $@) {
7836         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7837         warn "$@\n" if $@;
7838         die "mirroring error: $err\n";
7839     }
7840
7841     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7842 }
7843
7844 # $completion can be either
7845 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7846 # 'cancel': wait until all jobs are ready, block-job-cancel them
7847 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7848 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7849 sub qemu_drive_mirror_monitor {
7850     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7851
7852     $completion //= 'complete';
7853     $op //= "mirror";
7854
7855     eval {
7856         my $err_complete = 0;
7857
7858         my $starttime = time ();
7859         while (1) {
7860             die "block job ('$op') timed out\n" if $err_complete > 300;
7861
7862             my $stats = mon_cmd($vmid, "query-block-jobs");
7863             my $ctime = time();
7864
7865             my $running_jobs = {};
7866             for my $stat (@$stats) {
7867                 next if $stat->{type} ne $op;
7868                 $running_jobs->{$stat->{device}} = $stat;
7869             }
7870
7871             my $readycounter = 0;
7872
7873             for my $job_id (sort keys %$jobs) {
7874                 my $job = $running_jobs->{$job_id};
7875
7876                 my $vanished = !defined($job);
7877                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7878                 if($complete || ($vanished && $completion eq 'auto')) {
7879                     print "$job_id: $op-job finished\n";
7880                     delete $jobs->{$job_id};
7881                     next;
7882                 }
7883
7884                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7885
7886                 my $busy = $job->{busy};
7887                 my $ready = $job->{ready};
7888                 if (my $total = $job->{len}) {
7889                     my $transferred = $job->{offset} || 0;
7890                     my $remaining = $total - $transferred;
7891                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7892
7893                     my $duration = $ctime - $starttime;
7894                     my $total_h = render_bytes($total, 1);
7895                     my $transferred_h = render_bytes($transferred, 1);
7896
7897                     my $status = sprintf(
7898                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7899                         render_duration($duration),
7900                     );
7901
7902                     if ($ready) {
7903                         if ($busy) {
7904                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7905                         } else {
7906                             $status .= ", ready";
7907                         }
7908                     }
7909                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7910                     $jobs->{$job_id}->{ready} = $ready;
7911                 }
7912
7913                 $readycounter++ if $job->{ready};
7914             }
7915
7916             last if scalar(keys %$jobs) == 0;
7917
7918             if ($readycounter == scalar(keys %$jobs)) {
7919                 print "all '$op' jobs are ready\n";
7920
7921                 # do the complete later (or has already been done)
7922                 last if $completion eq 'skip' || $completion eq 'auto';
7923
7924                 if ($vmiddst && $vmiddst != $vmid) {
7925                     my $agent_running = $qga && qga_check_running($vmid);
7926                     if ($agent_running) {
7927                         print "freeze filesystem\n";
7928                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7929                         warn $@ if $@;
7930                     } else {
7931                         print "suspend vm\n";
7932                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7933                         warn $@ if $@;
7934                     }
7935
7936                     # if we clone a disk for a new target vm, we don't switch the disk
7937                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7938
7939                     if ($agent_running) {
7940                         print "unfreeze filesystem\n";
7941                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7942                         warn $@ if $@;
7943                     } else {
7944                         print "resume vm\n";
7945                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7946                         warn $@ if $@;
7947                     }
7948
7949                     last;
7950                 } else {
7951
7952                     for my $job_id (sort keys %$jobs) {
7953                         # try to switch the disk if source and destination are on the same guest
7954                         print "$job_id: Completing block job_id...\n";
7955
7956                         my $op;
7957                         if ($completion eq 'complete') {
7958                             $op = 'block-job-complete';
7959                         } elsif ($completion eq 'cancel') {
7960                             $op = 'block-job-cancel';
7961                         } else {
7962                             die "invalid completion value: $completion\n";
7963                         }
7964                         eval { mon_cmd($vmid, $op, device => $job_id) };
7965                         if ($@ =~ m/cannot be completed/) {
7966                             print "$job_id: block job cannot be completed, trying again.\n";
7967                             $err_complete++;
7968                         }else {
7969                             print "$job_id: Completed successfully.\n";
7970                             $jobs->{$job_id}->{complete} = 1;
7971                         }
7972                     }
7973                 }
7974             }
7975             sleep 1;
7976         }
7977     };
7978     my $err = $@;
7979
7980     if ($err) {
7981         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7982         die "block job ($op) error: $err";
7983     }
7984 }
7985
7986 sub qemu_blockjobs_cancel {
7987     my ($vmid, $jobs) = @_;
7988
7989     foreach my $job (keys %$jobs) {
7990         print "$job: Cancelling block job\n";
7991         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7992         $jobs->{$job}->{cancel} = 1;
7993     }
7994
7995     while (1) {
7996         my $stats = mon_cmd($vmid, "query-block-jobs");
7997
7998         my $running_jobs = {};
7999         foreach my $stat (@$stats) {
8000             $running_jobs->{$stat->{device}} = $stat;
8001         }
8002
8003         foreach my $job (keys %$jobs) {
8004
8005             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8006                 print "$job: Done.\n";
8007                 delete $jobs->{$job};
8008             }
8009         }
8010
8011         last if scalar(keys %$jobs) == 0;
8012
8013         sleep 1;
8014     }
8015 }
8016
8017 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8018 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8019 my sub clone_disk_check_io_uring {
8020     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8021
8022     return if !$use_drive_mirror;
8023
8024     # Don't complain when not changing storage.
8025     # Assume if it works for the source, it'll work for the target too.
8026     return if $src_storeid eq $dst_storeid;
8027
8028     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8029     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8030
8031     my $cache_direct = drive_uses_cache_direct($src_drive);
8032
8033     my $src_uses_io_uring;
8034     if ($src_drive->{aio}) {
8035         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8036     } else {
8037         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8038     }
8039
8040     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8041         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8042 }
8043
8044 sub clone_disk {
8045     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8046
8047     my ($vmid, $running) = $source->@{qw(vmid running)};
8048     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8049
8050     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8051     my ($storage, $format) = $dest->@{qw(storage format)};
8052
8053     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8054
8055     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8056         die "cloning from/to EFI disk requires EFI disk\n"
8057             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8058         die "cloning from/to TPM state requires TPM state\n"
8059             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8060
8061         # This would lead to two device nodes in QEMU pointing to the same backing image!
8062         die "cannot change drive name when cloning disk from/to the same VM\n"
8063             if $use_drive_mirror && $vmid == $newvmid;
8064     }
8065
8066     die "cannot move TPM state while VM is running\n"
8067         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8068
8069     my $newvolid;
8070
8071     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8072     print "$src_drivename " if $src_drivename;
8073     print "($drive->{file})\n";
8074
8075     if (!$full) {
8076         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8077         push @$newvollist, $newvolid;
8078     } else {
8079         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8080         my $storeid = $storage || $src_storeid;
8081
8082         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8083
8084         my $name = undef;
8085         my $size = undef;
8086         if (drive_is_cloudinit($drive)) {
8087             $name = "vm-$newvmid-cloudinit";
8088             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8089             if ($scfg->{path}) {
8090                 $name .= ".$dst_format";
8091             }
8092             $snapname = undef;
8093             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8094         } elsif ($dst_drivename eq 'efidisk0') {
8095             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8096         } elsif ($dst_drivename eq 'tpmstate0') {
8097             $dst_format = 'raw';
8098             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8099         } else {
8100             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8101
8102             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8103         }
8104         $newvolid = PVE::Storage::vdisk_alloc(
8105             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8106         );
8107         push @$newvollist, $newvolid;
8108
8109         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8110
8111         if (drive_is_cloudinit($drive)) {
8112             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8113             # if this is the case, we have to complete any block-jobs still there from
8114             # previous drive-mirrors
8115             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8116                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8117             }
8118             goto no_data_clone;
8119         }
8120
8121         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8122         if ($use_drive_mirror) {
8123             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8124                 $completion, $qga, $bwlimit);
8125         } else {
8126             if ($dst_drivename eq 'efidisk0') {
8127                 # the relevant data on the efidisk may be smaller than the source
8128                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8129                 # that is given by the OVMF_VARS.fd
8130                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8131                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8132
8133                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8134
8135                 # better for Ceph if block size is not too small, see bug #3324
8136                 my $bs = 1024*1024;
8137
8138                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8139
8140                 if ($src_format eq 'qcow2' && $snapname) {
8141                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8142                         if !min_version(kvm_user_version(), 6, 2);
8143                     push $cmd->@*, '-l', $snapname;
8144                 }
8145                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8146                 run_command($cmd);
8147             } else {
8148                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8149             }
8150         }
8151     }
8152
8153 no_data_clone:
8154     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8155
8156     my $disk = dclone($drive);
8157     delete $disk->{format};
8158     $disk->{file} = $newvolid;
8159     $disk->{size} = $size if defined($size);
8160
8161     return $disk;
8162 }
8163
8164 sub get_running_qemu_version {
8165     my ($vmid) = @_;
8166     my $res = mon_cmd($vmid, "query-version");
8167     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8168 }
8169
8170 sub qemu_use_old_bios_files {
8171     my ($machine_type) = @_;
8172
8173     return if !$machine_type;
8174
8175     my $use_old_bios_files = undef;
8176
8177     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8178         $machine_type = $1;
8179         $use_old_bios_files = 1;
8180     } else {
8181         my $version = extract_version($machine_type, kvm_user_version());
8182         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8183         # load new efi bios files on migration. So this hack is required to allow
8184         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8185         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8186         $use_old_bios_files = !min_version($version, 2, 4);
8187     }
8188
8189     return ($use_old_bios_files, $machine_type);
8190 }
8191
8192 sub get_efivars_size {
8193     my ($conf, $efidisk) = @_;
8194
8195     my $arch = get_vm_arch($conf);
8196     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8197     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8198     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8199     return -s $ovmf_vars;
8200 }
8201
8202 sub update_efidisk_size {
8203     my ($conf) = @_;
8204
8205     return if !defined($conf->{efidisk0});
8206
8207     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8208     $disk->{size} = get_efivars_size($conf);
8209     $conf->{efidisk0} = print_drive($disk);
8210
8211     return;
8212 }
8213
8214 sub update_tpmstate_size {
8215     my ($conf) = @_;
8216
8217     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8218     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8219     $conf->{tpmstate0} = print_drive($disk);
8220 }
8221
8222 sub create_efidisk($$$$$$$) {
8223     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8224
8225     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8226
8227     my $vars_size_b = -s $ovmf_vars;
8228     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8229     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8230     PVE::Storage::activate_volumes($storecfg, [$volid]);
8231
8232     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8233     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8234
8235     return ($volid, $size/1024);
8236 }
8237
8238 sub vm_iothreads_list {
8239     my ($vmid) = @_;
8240
8241     my $res = mon_cmd($vmid, 'query-iothreads');
8242
8243     my $iothreads = {};
8244     foreach my $iothread (@$res) {
8245         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8246     }
8247
8248     return $iothreads;
8249 }
8250
8251 sub scsihw_infos {
8252     my ($conf, $drive) = @_;
8253
8254     my $maxdev = 0;
8255
8256     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8257         $maxdev = 7;
8258     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8259         $maxdev = 1;
8260     } else {
8261         $maxdev = 256;
8262     }
8263
8264     my $controller = int($drive->{index} / $maxdev);
8265     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8266         ? "virtioscsi"
8267         : "scsihw";
8268
8269     return ($maxdev, $controller, $controller_prefix);
8270 }
8271
8272 sub resolve_dst_disk_format {
8273         my ($storecfg, $storeid, $src_volname, $format) = @_;
8274         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8275
8276         if (!$format) {
8277             # if no target format is specified, use the source disk format as hint
8278             if ($src_volname) {
8279                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8280                 $format = qemu_img_format($scfg, $src_volname);
8281             } else {
8282                 return $defFormat;
8283             }
8284         }
8285
8286         # test if requested format is supported - else use default
8287         my $supported = grep { $_ eq $format } @$validFormats;
8288         $format = $defFormat if !$supported;
8289         return $format;
8290 }
8291
8292 # NOTE: if this logic changes, please update docs & possibly gui logic
8293 sub find_vmstate_storage {
8294     my ($conf, $storecfg) = @_;
8295
8296     # first, return storage from conf if set
8297     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8298
8299     my ($target, $shared, $local);
8300
8301     foreach_storage_used_by_vm($conf, sub {
8302         my ($sid) = @_;
8303         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8304         my $dst = $scfg->{shared} ? \$shared : \$local;
8305         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8306     });
8307
8308     # second, use shared storage where VM has at least one disk
8309     # third, use local storage where VM has at least one disk
8310     # fall back to local storage
8311     $target = $shared // $local // 'local';
8312
8313     return $target;
8314 }
8315
8316 sub generate_uuid {
8317     my ($uuid, $uuid_str);
8318     UUID::generate($uuid);
8319     UUID::unparse($uuid, $uuid_str);
8320     return $uuid_str;
8321 }
8322
8323 sub generate_smbios1_uuid {
8324     return "uuid=".generate_uuid();
8325 }
8326
8327 sub nbd_stop {
8328     my ($vmid) = @_;
8329
8330     mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8331 }
8332
8333 sub create_reboot_request {
8334     my ($vmid) = @_;
8335     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8336         or die "failed to create reboot trigger file: $!\n";
8337     close($fh);
8338 }
8339
8340 sub clear_reboot_request {
8341     my ($vmid) = @_;
8342     my $path = "/run/qemu-server/$vmid.reboot";
8343     my $res = 0;
8344
8345     $res = unlink($path);
8346     die "could not remove reboot request for $vmid: $!"
8347         if !$res && $! != POSIX::ENOENT;
8348
8349     return $res;
8350 }
8351
8352 sub bootorder_from_legacy {
8353     my ($conf, $bootcfg) = @_;
8354
8355     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8356     my $bootindex_hash = {};
8357     my $i = 1;
8358     foreach my $o (split(//, $boot)) {
8359         $bootindex_hash->{$o} = $i*100;
8360         $i++;
8361     }
8362
8363     my $bootorder = {};
8364
8365     PVE::QemuConfig->foreach_volume($conf, sub {
8366         my ($ds, $drive) = @_;
8367
8368         if (drive_is_cdrom ($drive, 1)) {
8369             if ($bootindex_hash->{d}) {
8370                 $bootorder->{$ds} = $bootindex_hash->{d};
8371                 $bootindex_hash->{d} += 1;
8372             }
8373         } elsif ($bootindex_hash->{c}) {
8374             $bootorder->{$ds} = $bootindex_hash->{c}
8375                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8376             $bootindex_hash->{c} += 1;
8377         }
8378     });
8379
8380     if ($bootindex_hash->{n}) {
8381         for (my $i = 0; $i < $MAX_NETS; $i++) {
8382             my $netname = "net$i";
8383             next if !$conf->{$netname};
8384             $bootorder->{$netname} = $bootindex_hash->{n};
8385             $bootindex_hash->{n} += 1;
8386         }
8387     }
8388
8389     return $bootorder;
8390 }
8391
8392 # Generate default device list for 'boot: order=' property. Matches legacy
8393 # default boot order, but with explicit device names. This is important, since
8394 # the fallback for when neither 'order' nor the old format is specified relies
8395 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8396 sub get_default_bootdevices {
8397     my ($conf) = @_;
8398
8399     my @ret = ();
8400
8401     # harddisk
8402     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8403     push @ret, $first if $first;
8404
8405     # cdrom
8406     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8407     push @ret, $first if $first;
8408
8409     # network
8410     for (my $i = 0; $i < $MAX_NETS; $i++) {
8411         my $netname = "net$i";
8412         next if !$conf->{$netname};
8413         push @ret, $netname;
8414         last;
8415     }
8416
8417     return \@ret;
8418 }
8419
8420 sub device_bootorder {
8421     my ($conf) = @_;
8422
8423     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8424
8425     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8426
8427     my $bootorder = {};
8428     if (!defined($boot) || $boot->{legacy}) {
8429         $bootorder = bootorder_from_legacy($conf, $boot);
8430     } elsif ($boot->{order}) {
8431         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8432         for my $dev (PVE::Tools::split_list($boot->{order})) {
8433             $bootorder->{$dev} = $i++;
8434         }
8435     }
8436
8437     return $bootorder;
8438 }
8439
8440 sub register_qmeventd_handle {
8441     my ($vmid) = @_;
8442
8443     my $fh;
8444     my $peer = "/var/run/qmeventd.sock";
8445     my $count = 0;
8446
8447     for (;;) {
8448         $count++;
8449         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8450         last if $fh;
8451         if ($! != EINTR && $! != EAGAIN) {
8452             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8453         }
8454         if ($count > 4) {
8455             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8456               . "after $count retries\n";
8457         }
8458         usleep(25000);
8459     }
8460
8461     # send handshake to mark VM as backing up
8462     print $fh to_json({vzdump => {vmid => "$vmid"}});
8463
8464     # return handle to be closed later when inhibit is no longer required
8465     return $fh;
8466 }
8467
8468 # bash completion helper
8469
8470 sub complete_backup_archives {
8471     my ($cmdname, $pname, $cvalue) = @_;
8472
8473     my $cfg = PVE::Storage::config();
8474
8475     my $storeid;
8476
8477     if ($cvalue =~ m/^([^:]+):/) {
8478         $storeid = $1;
8479     }
8480
8481     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8482
8483     my $res = [];
8484     foreach my $id (keys %$data) {
8485         foreach my $item (@{$data->{$id}}) {
8486             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8487             push @$res, $item->{volid} if defined($item->{volid});
8488         }
8489     }
8490
8491     return $res;
8492 }
8493
8494 my $complete_vmid_full = sub {
8495     my ($running) = @_;
8496
8497     my $idlist = vmstatus();
8498
8499     my $res = [];
8500
8501     foreach my $id (keys %$idlist) {
8502         my $d = $idlist->{$id};
8503         if (defined($running)) {
8504             next if $d->{template};
8505             next if $running && $d->{status} ne 'running';
8506             next if !$running && $d->{status} eq 'running';
8507         }
8508         push @$res, $id;
8509
8510     }
8511     return $res;
8512 };
8513
8514 sub complete_vmid {
8515     return &$complete_vmid_full();
8516 }
8517
8518 sub complete_vmid_stopped {
8519     return &$complete_vmid_full(0);
8520 }
8521
8522 sub complete_vmid_running {
8523     return &$complete_vmid_full(1);
8524 }
8525
8526 sub complete_storage {
8527
8528     my $cfg = PVE::Storage::config();
8529     my $ids = $cfg->{ids};
8530
8531     my $res = [];
8532     foreach my $sid (keys %$ids) {
8533         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8534         next if !$ids->{$sid}->{content}->{images};
8535         push @$res, $sid;
8536     }
8537
8538     return $res;
8539 }
8540
8541 sub complete_migration_storage {
8542     my ($cmd, $param, $current_value, $all_args) = @_;
8543
8544     my $targetnode = @$all_args[1];
8545
8546     my $cfg = PVE::Storage::config();
8547     my $ids = $cfg->{ids};
8548
8549     my $res = [];
8550     foreach my $sid (keys %$ids) {
8551         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8552         next if !$ids->{$sid}->{content}->{images};
8553         push @$res, $sid;
8554     }
8555
8556     return $res;
8557 }
8558
8559 sub vm_is_paused {
8560     my ($vmid, $include_suspended) = @_;
8561     my $qmpstatus = eval {
8562         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8563         mon_cmd($vmid, "query-status");
8564     };
8565     warn "$@\n" if $@;
8566     return $qmpstatus && (
8567         $qmpstatus->{status} eq "paused" ||
8568         $qmpstatus->{status} eq "prelaunch" ||
8569         ($include_suspended && $qmpstatus->{status} eq "suspended")
8570     );
8571 }
8572
8573 sub check_volume_storage_type {
8574     my ($storecfg, $vol) = @_;
8575
8576     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8577     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8578     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8579
8580     die "storage '$storeid' does not support content-type '$vtype'\n"
8581         if !$scfg->{content}->{$vtype};
8582
8583     return 1;
8584 }
8585
8586 sub add_nets_bridge_fdb {
8587     my ($conf, $vmid) = @_;
8588
8589     for my $opt (keys %$conf) {
8590         next if $opt !~ m/^net(\d+)$/;
8591         my $iface = "tap${vmid}i$1";
8592         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8593         my $net = parse_net($conf->{$opt}, 1) or next;
8594
8595         my $mac = $net->{macaddr};
8596         if (!$mac) {
8597             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8598                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8599             next;
8600         }
8601
8602         my $bridge = $net->{bridge};
8603         if (!$bridge) {
8604             log_warn("Interface '$iface' not attached to any bridge.");
8605             next;
8606         }
8607         if ($have_sdn) {
8608             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8609         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8610             PVE::Network::add_bridge_fdb($iface, $mac);
8611         }
8612     }
8613 }
8614
8615 sub del_nets_bridge_fdb {
8616     my ($conf, $vmid) = @_;
8617
8618     for my $opt (keys %$conf) {
8619         next if $opt !~ m/^net(\d+)$/;
8620         my $iface = "tap${vmid}i$1";
8621
8622         my $net = parse_net($conf->{$opt}) or next;
8623         my $mac = $net->{macaddr} or next;
8624
8625         my $bridge = $net->{bridge};
8626         if ($have_sdn) {
8627             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8628         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8629             PVE::Network::del_bridge_fdb($iface, $mac);
8630         }
8631     }
8632 }
8633
8634 sub create_ifaces_ipams_ips {
8635     my ($conf, $vmid) = @_;
8636
8637     return if !$have_sdn;
8638
8639     foreach my $opt (keys %$conf) {
8640         if ($opt =~ m/^net(\d+)$/) {
8641             my $value = $conf->{$opt};
8642             my $net = PVE::QemuServer::parse_net($value);
8643             eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8644             warn $@ if $@;
8645         }
8646     }
8647 }
8648
8649 sub delete_ifaces_ipams_ips {
8650     my ($conf, $vmid) = @_;
8651
8652     return if !$have_sdn;
8653
8654     foreach my $opt (keys %$conf) {
8655         if ($opt =~ m/^net(\d+)$/) {
8656             my $net = PVE::QemuServer::parse_net($conf->{$opt});
8657             eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8658             warn $@ if $@;
8659         }
8660     }
8661 }
8662
8663 1;