PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::Mapping::PCI;
  38 use PVE::Mapping::USB;
  39 use PVE::INotify;
  40 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  41 use PVE::ProcFSTools;
  42 use PVE::PBSClient;
  43 use PVE::RESTEnvironment qw(log_warn);
  44 use PVE::RPCEnvironment;
  45 use PVE::Storage;
  46 use PVE::SysFSTools;
  47 use PVE::Systemd;
  48 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  49
  50 use PVE::QMPClient;
  51 use PVE::QemuConfig;
  52 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  53 use PVE::QemuServer::Cloudinit;
  54 use PVE::QemuServer::CGroup;
  55 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  56 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  57 use PVE::QemuServer::Machine;
  58 use PVE::QemuServer::Memory;
  59 use PVE::QemuServer::Monitor qw(mon_cmd);
  60 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  61 use PVE::QemuServer::USB;
  62
  63 my $have_sdn;
  64 eval {
  65     require PVE::Network::SDN::Zones;
  66     $have_sdn = 1;
  67 };
  68
  69 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  70 my $OVMF = {
  71     x86_64 => {
  72         '4m-no-smm' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  75         ],
  76         '4m-no-smm-ms' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  79         ],
  80         '4m' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  83         ],
  84         '4m-ms' => [
  85             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  87         ],
  88         default => [
  89             "$EDK2_FW_BASE/OVMF_CODE.fd",
  90             "$EDK2_FW_BASE/OVMF_VARS.fd",
  91         ],
  92     },
  93     aarch64 => {
  94         default => [
  95             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  96             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  97         ],
  98     },
  99 };
 100
 101 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 102
 103 # Note about locking: we use flock on the config file protect against concurent actions.
 104 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 105 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 106 # But you can ignore this kind of lock with the --skiplock flag.
 107
 108 cfs_register_file(
 109     '/qemu-server/',
 110     \&parse_vm_config,
 111     \&write_vm_config
 112 );
 113
 114 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 115     description => "Some command save/restore state from this location.",
 116     type => 'string',
 117     maxLength => 128,
 118     optional => 1,
 119 });
 120
 121 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 122         description => "Specifies the QEMU machine type.",
 123         type => 'string',
 124         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 125         maxLength => 40,
 126         optional => 1,
 127 });
 128
 129 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 130 my $nodename_cache;
 131 sub nodename {
 132     $nodename_cache //= PVE::INotify::nodename();
 133     return $nodename_cache;
 134 }
 135
 136 my $watchdog_fmt = {
 137     model => {
 138         default_key => 1,
 139         type => 'string',
 140         enum => [qw(i6300esb ib700)],
 141         description => "Watchdog type to emulate.",
 142         default => 'i6300esb',
 143         optional => 1,
 144     },
 145     action => {
 146         type => 'string',
 147         enum => [qw(reset shutdown poweroff pause debug none)],
 148         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 149         optional => 1,
 150     },
 151 };
 152 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 153
 154 my $agent_fmt = {
 155     enabled => {
 156         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 157         type => 'boolean',
 158         default => 0,
 159         default_key => 1,
 160     },
 161     fstrim_cloned_disks => {
 162         description => "Run fstrim after moving a disk or migrating the VM.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0,
 166     },
 167     'freeze-fs-on-backup' => {
 168         description => "Freeze/thaw guest filesystems on backup for consistency.",
 169         type => 'boolean',
 170         optional => 1,
 171         default => 1,
 172     },
 173     type => {
 174         description => "Select the agent type",
 175         type => 'string',
 176         default => 'virtio',
 177         optional => 1,
 178         enum => [qw(virtio isa)],
 179     },
 180 };
 181
 182 my $vga_fmt = {
 183     type => {
 184         description => "Select the VGA type.",
 185         type => 'string',
 186         default => 'std',
 187         optional => 1,
 188         default_key => 1,
 189         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 190     },
 191     memory => {
 192         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 193         type => 'integer',
 194         optional => 1,
 195         minimum => 4,
 196         maximum => 512,
 197     },
 198 };
 199
 200 my $ivshmem_fmt = {
 201     size => {
 202         type => 'integer',
 203         minimum => 1,
 204         description => "The size of the file in MB.",
 205     },
 206     name => {
 207         type => 'string',
 208         pattern => '[a-zA-Z0-9\-]+',
 209         optional => 1,
 210         format_description => 'string',
 211         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 212     },
 213 };
 214
 215 my $audio_fmt = {
 216     device => {
 217         type => 'string',
 218         enum => [qw(ich9-intel-hda intel-hda AC97)],
 219         description =>  "Configure an audio device."
 220     },
 221     driver =>  {
 222         type => 'string',
 223         enum => ['spice', 'none'],
 224         default => 'spice',
 225         optional => 1,
 226         description => "Driver backend for the audio device."
 227     },
 228 };
 229
 230 my $spice_enhancements_fmt = {
 231     foldersharing => {
 232         type => 'boolean',
 233         optional => 1,
 234         default => '0',
 235         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 236     },
 237     videostreaming =>  {
 238         type => 'string',
 239         enum => ['off', 'all', 'filter'],
 240         default => 'off',
 241         optional => 1,
 242         description => "Enable video streaming. Uses compression for detected video streams."
 243     },
 244 };
 245
 246 my $rng_fmt = {
 247     source => {
 248         type => 'string',
 249         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 250         default_key => 1,
 251         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 252             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 253             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 254             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 255             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 256             ." a hardware RNG from the host.",
 257     },
 258     max_bytes => {
 259         type => 'integer',
 260         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 261             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 262             ." `0` to disable limiting (potentially dangerous!).",
 263         optional => 1,
 264
 265         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 266         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 267         # reading from /dev/urandom
 268         default => 1024,
 269     },
 270     period => {
 271         type => 'integer',
 272         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 273             ." the guest to retrieve another 'max_bytes' of entropy.",
 274         optional => 1,
 275         default => 1000,
 276     },
 277 };
 278
 279 my $meta_info_fmt = {
 280     'ctime' => {
 281         type => 'integer',
 282         description => "The guest creation timestamp as UNIX epoch time",
 283         minimum => 0,
 284         optional => 1,
 285     },
 286     'creation-qemu' => {
 287         type => 'string',
 288         description => "The QEMU (machine) version from the time this VM was created.",
 289         pattern => '\d+(\.\d+)+',
 290         optional => 1,
 291     },
 292 };
 293
 294 my $confdesc = {
 295     onboot => {
 296         optional => 1,
 297         type => 'boolean',
 298         description => "Specifies whether a VM will be started during system bootup.",
 299         default => 0,
 300     },
 301     autostart => {
 302         optional => 1,
 303         type => 'boolean',
 304         description => "Automatic restart after crash (currently ignored).",
 305         default => 0,
 306     },
 307     hotplug => {
 308         optional => 1,
 309         type => 'string', format => 'pve-hotplug-features',
 310         description => "Selectively enable hotplug features. This is a comma separated list of"
 311             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 312             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 313             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 314             ." windows > 7.",
 315         default => 'network,disk,usb',
 316     },
 317     reboot => {
 318         optional => 1,
 319         type => 'boolean',
 320         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 321         default => 1,
 322     },
 323     lock => {
 324         optional => 1,
 325         type => 'string',
 326         description => "Lock/unlock the VM.",
 327         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 328     },
 329     cpulimit => {
 330         optional => 1,
 331         type => 'number',
 332         description => "Limit of CPU usage.",
 333         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 334             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 335         minimum => 0,
 336         maximum => 128,
 337         default => 0,
 338     },
 339     cpuunits => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 343         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 344             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 345             ." weights of all the other running VMs.",
 346         minimum => 1,
 347         maximum => 262144,
 348         default => 'cgroup v1: 1024, cgroup v2: 100',
 349     },
 350     memory => {
 351         optional => 1,
 352         type => 'integer',
 353         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 354             ." you use the balloon device.",
 355         minimum => 16,
 356         default => 512,
 357     },
 358     balloon => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 362         minimum => 0,
 363     },
 364     shares => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 368             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 369             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 370         minimum => 0,
 371         maximum => 50000,
 372         default => 1000,
 373     },
 374     keyboard => {
 375         optional => 1,
 376         type => 'string',
 377         description => "Keyboard layout for VNC server. This option is generally not required and"
 378         ." is often better handled from within the guest OS.",
 379         enum => PVE::Tools::kvmkeymaplist(),
 380         default => undef,
 381     },
 382     name => {
 383         optional => 1,
 384         type => 'string', format => 'dns-name',
 385         description => "Set a name for the VM. Only used on the configuration web interface.",
 386     },
 387     scsihw => {
 388         optional => 1,
 389         type => 'string',
 390         description => "SCSI controller model",
 391         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 392         default => 'lsi',
 393     },
 394     description => {
 395         optional => 1,
 396         type => 'string',
 397         description => "Description for the VM. Shown in the web-interface VM's summary."
 398             ." This is saved as comment inside the configuration file.",
 399         maxLength => 1024 * 8,
 400     },
 401     ostype => {
 402         optional => 1,
 403         type => 'string',
 404         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 405         description => "Specify guest operating system.",
 406         verbose_description => <<EODESC,
 407 Specify guest operating system. This is used to enable special
 408 optimization/features for specific operating systems:
 409
 410 [horizontal]
 411 other;; unspecified OS
 412 wxp;; Microsoft Windows XP
 413 w2k;; Microsoft Windows 2000
 414 w2k3;; Microsoft Windows 2003
 415 w2k8;; Microsoft Windows 2008
 416 wvista;; Microsoft Windows Vista
 417 win7;; Microsoft Windows 7
 418 win8;; Microsoft Windows 8/2012/2012r2
 419 win10;; Microsoft Windows 10/2016/2019
 420 win11;; Microsoft Windows 11/2022
 421 l24;; Linux 2.4 Kernel
 422 l26;; Linux 2.6 - 6.X Kernel
 423 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 424 EODESC
 425     },
 426     boot => {
 427         optional => 1,
 428         type => 'string', format => 'pve-qm-boot',
 429         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 430             ." key or 'legacy=' is deprecated.",
 431     },
 432     bootdisk => {
 433         optional => 1,
 434         type => 'string', format => 'pve-qm-bootdisk',
 435         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 436         pattern => '(ide|sata|scsi|virtio)\d+',
 437     },
 438     smp => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPUs. Please use option -sockets instead.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     sockets => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of CPU sockets.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     cores => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of cores per socket.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     numa => {
 460         optional => 1,
 461         type => 'boolean',
 462         description => "Enable/disable NUMA.",
 463         default => 0,
 464     },
 465     hugepages => {
 466         optional => 1,
 467         type => 'string',
 468         description => "Enable/disable hugepages memory.",
 469         enum => [qw(any 2 1024)],
 470     },
 471     keephugepages => {
 472         optional => 1,
 473         type => 'boolean',
 474         default => 0,
 475         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 476             ." after VM shutdown and can be used for subsequent starts.",
 477     },
 478     vcpus => {
 479         optional => 1,
 480         type => 'integer',
 481         description => "Number of hotplugged vcpus.",
 482         minimum => 1,
 483         default => 0,
 484     },
 485     acpi => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable ACPI.",
 489         default => 1,
 490     },
 491     agent => {
 492         optional => 1,
 493         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 494         type => 'string',
 495         format => $agent_fmt,
 496     },
 497     kvm => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable KVM hardware virtualization.",
 501         default => 1,
 502     },
 503     tdf => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable time drift fix.",
 507         default => 0,
 508     },
 509     localtime => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 513             ." the `ostype` indicates a Microsoft Windows OS.",
 514     },
 515     freeze => {
 516         optional => 1,
 517         type => 'boolean',
 518         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 519     },
 520     vga => {
 521         optional => 1,
 522         type => 'string', format => $vga_fmt,
 523         description => "Configure the VGA hardware.",
 524         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 525             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 526             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 527             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 528             ." display server. For win* OS you can select how many independent displays you want,"
 529             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 530             ." using a serial device as terminal.",
 531     },
 532     watchdog => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-watchdog',
 535         description => "Create a virtual hardware watchdog device.",
 536         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 537             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 538             ." else the watchdog will reset the guest (or execute the respective action specified)",
 539     },
 540     startdate => {
 541         optional => 1,
 542         type => 'string',
 543         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 544         description => "Set the initial date of the real time clock. Valid format for date are:"
 545             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 546         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 547         default => 'now',
 548     },
 549     startup =>  get_standard_option('pve-startup-order'),
 550     template => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Enable/disable Template.",
 554         default => 0,
 555     },
 556     args => {
 557         optional => 1,
 558         type => 'string',
 559         description => "Arbitrary arguments passed to kvm.",
 560         verbose_description => <<EODESCR,
 561 Arbitrary arguments passed to kvm, for example:
 562
 563 args: -no-reboot -smbios 'type=0,vendor=FOO'
 564
 565 NOTE: this option is for experts only.
 566 EODESCR
 567     },
 568     tablet => {
 569         optional => 1,
 570         type => 'boolean',
 571         default => 1,
 572         description => "Enable/disable the USB tablet device.",
 573         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 574             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 575             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 576             ." may consider disabling this to save some context switches. This is turned off by"
 577             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 578     },
 579     migrate_speed => {
 580         optional => 1,
 581         type => 'integer',
 582         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 583         minimum => 0,
 584         default => 0,
 585     },
 586     migrate_downtime => {
 587         optional => 1,
 588         type => 'number',
 589         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 590         minimum => 0,
 591         default => 0.1,
 592     },
 593     cdrom => {
 594         optional => 1,
 595         type => 'string', format => 'pve-qm-ide',
 596         typetext => '<volume>',
 597         description => "This is an alias for option -ide2",
 598     },
 599     cpu => {
 600         optional => 1,
 601         description => "Emulated CPU type.",
 602         type => 'string',
 603         format => 'pve-vm-cpu-conf',
 604     },
 605     parent => get_standard_option('pve-snapshot-name', {
 606         optional => 1,
 607         description => "Parent snapshot name. This is used internally, and should not be modified.",
 608     }),
 609     snaptime => {
 610         optional => 1,
 611         description => "Timestamp for snapshots.",
 612         type => 'integer',
 613         minimum => 0,
 614     },
 615     vmstate => {
 616         optional => 1,
 617         type => 'string', format => 'pve-volume-id',
 618         description => "Reference to a volume which stores the VM state. This is used internally"
 619             ." for snapshots.",
 620     },
 621     vmstatestorage => get_standard_option('pve-storage-id', {
 622         description => "Default storage for VM state volumes/files.",
 623         optional => 1,
 624     }),
 625     runningmachine => get_standard_option('pve-qemu-machine', {
 626         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 627             ." for snapshots.",
 628     }),
 629     runningcpu => {
 630         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 631             ." internally for snapshots.",
 632         optional => 1,
 633         type => 'string',
 634         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 635         format_description => 'QEMU -cpu parameter'
 636     },
 637     machine => get_standard_option('pve-qemu-machine'),
 638     arch => {
 639         description => "Virtual processor architecture. Defaults to the host.",
 640         optional => 1,
 641         type => 'string',
 642         enum => [qw(x86_64 aarch64)],
 643     },
 644     smbios1 => {
 645         description => "Specify SMBIOS type 1 fields.",
 646         type => 'string', format => 'pve-qm-smbios1',
 647         maxLength => 512,
 648         optional => 1,
 649     },
 650     protection => {
 651         optional => 1,
 652         type => 'boolean',
 653         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 654             ." remove disk operations.",
 655         default => 0,
 656     },
 657     bios => {
 658         optional => 1,
 659         type => 'string',
 660         enum => [ qw(seabios ovmf) ],
 661         description => "Select BIOS implementation.",
 662         default => 'seabios',
 663     },
 664     vmgenid => {
 665         type => 'string',
 666         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 667         format_description => 'UUID',
 668         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 669             ." to disable explicitly.",
 670         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 671             ." value identifier to the guest OS. This allows to notify the guest operating system"
 672             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 673             ." execution or creation from a template). The guest operating system notices the"
 674             ." change, and is then able to react as appropriate by marking its copies of"
 675             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 676             ."Note that auto-creation only works when done through API/CLI create or update methods"
 677             .", but not when manually editing the config file.",
 678         default => "1 (autogenerated)",
 679         optional => 1,
 680     },
 681     hookscript => {
 682         type => 'string',
 683         format => 'pve-volume-id',
 684         optional => 1,
 685         description => "Script that will be executed during various steps in the vms lifetime.",
 686     },
 687     ivshmem => {
 688         type => 'string',
 689         format => $ivshmem_fmt,
 690         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 691             ." the host.",
 692         optional => 1,
 693     },
 694     audio0 => {
 695         type => 'string',
 696         format => $audio_fmt,
 697         description => "Configure a audio device, useful in combination with QXL/Spice.",
 698         optional => 1
 699     },
 700     spice_enhancements => {
 701         type => 'string',
 702         format => $spice_enhancements_fmt,
 703         description => "Configure additional enhancements for SPICE.",
 704         optional => 1
 705     },
 706     tags => {
 707         type => 'string', format => 'pve-tag-list',
 708         description => 'Tags of the VM. This is only meta information.',
 709         optional => 1,
 710     },
 711     rng0 => {
 712         type => 'string',
 713         format => $rng_fmt,
 714         description => "Configure a VirtIO-based Random Number Generator.",
 715         optional => 1,
 716     },
 717     meta => {
 718         type => 'string',
 719         format => $meta_info_fmt,
 720         description => "Some (read-only) meta-information about this guest.",
 721         optional => 1,
 722     },
 723     affinity => {
 724         type => 'string', format => 'pve-cpuset',
 725         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 726         optional => 1,
 727     },
 728 };
 729
 730 my $cicustom_fmt = {
 731     meta => {
 732         type => 'string',
 733         optional => 1,
 734         description => 'Specify a custom file containing all meta data passed to the VM via"
 735             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 736         format => 'pve-volume-id',
 737         format_description => 'volume',
 738     },
 739     network => {
 740         type => 'string',
 741         optional => 1,
 742         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     user => {
 747         type => 'string',
 748         optional => 1,
 749         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 750         format => 'pve-volume-id',
 751         format_description => 'volume',
 752     },
 753     vendor => {
 754         type => 'string',
 755         optional => 1,
 756         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 757         format => 'pve-volume-id',
 758         format_description => 'volume',
 759     },
 760 };
 761 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 762
 763 my $confdesc_cloudinit = {
 764     citype => {
 765         optional => 1,
 766         type => 'string',
 767         description => 'Specifies the cloud-init configuration format. The default depends on the'
 768             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 769             .' and `configdrive2` for windows.',
 770         enum => ['configdrive2', 'nocloud', 'opennebula'],
 771     },
 772     ciuser => {
 773         optional => 1,
 774         type => 'string',
 775         description => "cloud-init: User name to change ssh keys and password for instead of the"
 776             ." image's configured default user.",
 777     },
 778     cipassword => {
 779         optional => 1,
 780         type => 'string',
 781         description => 'cloud-init: Password to assign the user. Using this is generally not'
 782             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 783             .' support hashed passwords.',
 784     },
 785     ciupgrade => {
 786         optional => 1,
 787         type => 'boolean',
 788         description => 'cloud-init: do an automatic package upgrade after the first boot.',
 789         default => 1,
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.',
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.',
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_NETS = 32;
 842 my $MAX_SERIAL_PORTS = 4;
 843 my $MAX_PARALLEL_PORTS = 3;
 844 my $MAX_NUMA = 8;
 845
 846 my $numa_fmt = {
 847     cpus => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "CPUs accessing this NUMA node.",
 851         format_description => "id[-id];...",
 852     },
 853     memory => {
 854         type => "number",
 855         description => "Amount of memory this NUMA node provides.",
 856         optional => 1,
 857     },
 858     hostnodes => {
 859         type => "string",
 860         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 861         description => "Host NUMA nodes to use.",
 862         format_description => "id[-id];...",
 863         optional => 1,
 864     },
 865     policy => {
 866         type => 'string',
 867         enum => [qw(preferred bind interleave)],
 868         description => "NUMA allocation policy.",
 869         optional => 1,
 870     },
 871 };
 872 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 873 my $numadesc = {
 874     optional => 1,
 875     type => 'string', format => $numa_fmt,
 876     description => "NUMA topology.",
 877 };
 878 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 879
 880 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 881     $confdesc->{"numa$i"} = $numadesc;
 882 }
 883
 884 my $nic_model_list = [
 885     'e1000',
 886     'e1000-82540em',
 887     'e1000-82544gc',
 888     'e1000-82545em',
 889     'e1000e',
 890     'i82551',
 891     'i82557b',
 892     'i82559er',
 893     'ne2k_isa',
 894     'ne2k_pci',
 895     'pcnet',
 896     'rtl8139',
 897     'virtio',
 898     'vmxnet3',
 899 ];
 900 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 901
 902 my $net_fmt_bridge_descr = <<__EOD__;
 903 Bridge to attach the network device to. The Proxmox VE standard bridge
 904 is called 'vmbr0'.
 905
 906 If you do not specify a bridge, we create a kvm user (NATed) network
 907 device, which provides DHCP and DNS services. The following addresses
 908 are used:
 909
 910  10.0.2.2   Gateway
 911  10.0.2.3   DNS Server
 912  10.0.2.4   SMB Server
 913
 914 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 915 __EOD__
 916
 917 my $net_fmt = {
 918     macaddr  => get_standard_option('mac-addr', {
 919         description => "MAC address. That address must be unique withing your network. This is"
 920             ." automatically generated if not specified.",
 921     }),
 922     model => {
 923         type => 'string',
 924         description => "Network Card Model. The 'virtio' model provides the best performance with"
 925             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 926             ." best to use 'e1000'.",
 927         enum => $nic_model_list,
 928         default_key => 1,
 929     },
 930     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 931     bridge => get_standard_option('pve-bridge-id', {
 932         description => $net_fmt_bridge_descr,
 933         optional => 1,
 934     }),
 935     queues => {
 936         type => 'integer',
 937         minimum => 0, maximum => 64,
 938         description => 'Number of packet queues to be used on the device.',
 939         optional => 1,
 940     },
 941     rate => {
 942         type => 'number',
 943         minimum => 0,
 944         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 945         optional => 1,
 946     },
 947     tag => {
 948         type => 'integer',
 949         minimum => 1, maximum => 4094,
 950         description => 'VLAN tag to apply to packets on this interface.',
 951         optional => 1,
 952     },
 953     trunks => {
 954         type => 'string',
 955         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 956         description => 'VLAN trunks to pass through this interface.',
 957         format_description => 'vlanid[;vlanid...]',
 958         optional => 1,
 959     },
 960     firewall => {
 961         type => 'boolean',
 962         description => 'Whether this interface should be protected by the firewall.',
 963         optional => 1,
 964     },
 965     link_down => {
 966         type => 'boolean',
 967         description => 'Whether this interface should be disconnected (like pulling the plug).',
 968         optional => 1,
 969     },
 970     mtu => {
 971         type => 'integer',
 972         minimum => 1, maximum => 65520,
 973         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 974         optional => 1,
 975     },
 976 };
 977
 978 my $netdesc = {
 979     optional => 1,
 980     type => 'string', format => $net_fmt,
 981     description => "Specify network devices.",
 982 };
 983
 984 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 985
 986 my $ipconfig_fmt = {
 987     ip => {
 988         type => 'string',
 989         format => 'pve-ipv4-config',
 990         format_description => 'IPv4Format/CIDR',
 991         description => 'IPv4 address in CIDR format.',
 992         optional => 1,
 993         default => 'dhcp',
 994     },
 995     gw => {
 996         type => 'string',
 997         format => 'ipv4',
 998         format_description => 'GatewayIPv4',
 999         description => 'Default gateway for IPv4 traffic.',
1000         optional => 1,
1001         requires => 'ip',
1002     },
1003     ip6 => {
1004         type => 'string',
1005         format => 'pve-ipv6-config',
1006         format_description => 'IPv6Format/CIDR',
1007         description => 'IPv6 address in CIDR format.',
1008         optional => 1,
1009         default => 'dhcp',
1010     },
1011     gw6 => {
1012         type => 'string',
1013         format => 'ipv6',
1014         format_description => 'GatewayIPv6',
1015         description => 'Default gateway for IPv6 traffic.',
1016         optional => 1,
1017         requires => 'ip6',
1018     },
1019 };
1020 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1021 my $ipconfigdesc = {
1022     optional => 1,
1023     type => 'string', format => 'pve-qm-ipconfig',
1024     description => <<'EODESCR',
1025 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1026
1027 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1028
1029 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1030 gateway should be provided.
1031 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1032 cloud-init 19.4 or newer.
1033
1034 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1035 dhcp on IPv4.
1036 EODESCR
1037 };
1038 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1039
1040 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1041     $confdesc->{"net$i"} = $netdesc;
1042     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1043 }
1044
1045 foreach my $key (keys %$confdesc_cloudinit) {
1046     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1050 sub pve_verify_cpuset {
1051     my ($set_text, $noerr) = @_;
1052
1053     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1054
1055     if ($@) {
1056         return if $noerr;
1057         die "unable to parse cpuset option\n";
1058     }
1059
1060     return PVE::CpuSet->new($members)->short_string();
1061 }
1062
1063 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1064 sub verify_volume_id_or_qm_path {
1065     my ($volid, $noerr) = @_;
1066
1067     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1068
1069     return verify_volume_id_or_absolute_path($volid, $noerr);
1070 }
1071
1072 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1073 sub verify_volume_id_or_absolute_path {
1074     my ($volid, $noerr) = @_;
1075
1076     return $volid if $volid =~ m|^/|;
1077
1078     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1079     if ($@) {
1080         return if $noerr;
1081         die $@;
1082     }
1083     return $volid;
1084 }
1085
1086 my $serialdesc = {
1087         optional => 1,
1088         type => 'string',
1089         pattern => '(/dev/.+|socket)',
1090         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1091         verbose_description =>  <<EODESCR,
1092 Create a serial device inside the VM (n is 0 to 3), and pass through a
1093 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1094 host side (use 'qm terminal' to open a terminal connection).
1095
1096 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1097 use with special care.
1098
1099 CAUTION: Experimental! User reported problems with this option.
1100 EODESCR
1101 };
1102
1103 my $paralleldesc= {
1104         optional => 1,
1105         type => 'string',
1106         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1107         description =>  "Map host parallel devices (n is 0 to 2).",
1108         verbose_description =>  <<EODESCR,
1109 Map host parallel devices (n is 0 to 2).
1110
1111 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1112 machines - use with special care.
1113
1114 CAUTION: Experimental! User reported problems with this option.
1115 EODESCR
1116 };
1117
1118 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1119     $confdesc->{"parallel$i"} = $paralleldesc;
1120 }
1121
1122 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1123     $confdesc->{"serial$i"} = $serialdesc;
1124 }
1125
1126 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1127     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1128 }
1129
1130 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1131     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1132 }
1133
1134 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1135     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1136 }
1137
1138 my $boot_fmt = {
1139     legacy => {
1140         optional => 1,
1141         default_key => 1,
1142         type => 'string',
1143         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1144                      . " Deprecated, use 'order=' instead.",
1145         pattern => '[acdn]{1,4}',
1146         format_description => "[acdn]{1,4}",
1147
1148         # note: this is also the fallback if boot: is not given at all
1149         default => 'cdn',
1150     },
1151     order => {
1152         optional => 1,
1153         type => 'string',
1154         format => 'pve-qm-bootdev-list',
1155         format_description => "device[;device...]",
1156         description => <<EODESC,
1157 The guest will attempt to boot from devices in the order they appear here.
1158
1159 Disks, optical drives and passed-through storage USB devices will be directly
1160 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1161 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1162
1163 Note that only devices in this list will be marked as bootable and thus loaded
1164 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1165 (e.g. software-raid), you need to specify all of them here.
1166
1167 Overrides the deprecated 'legacy=[acdn]*' value when given.
1168 EODESC
1169     },
1170 };
1171 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1172
1173 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1174 sub verify_bootdev {
1175     my ($dev, $noerr) = @_;
1176
1177     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1178     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1179
1180     my $check = sub {
1181         my ($base) = @_;
1182         return 0 if $dev !~ m/^$base\d+$/;
1183         return 0 if !$confdesc->{$dev};
1184         return 1;
1185     };
1186
1187     return $dev if $check->("net");
1188     return $dev if $check->("usb");
1189     return $dev if $check->("hostpci");
1190
1191     return if $noerr;
1192     die "invalid boot device '$dev'\n";
1193 }
1194
1195 sub print_bootorder {
1196     my ($devs) = @_;
1197     return "" if !@$devs;
1198     my $data = { order => join(';', @$devs) };
1199     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1200 }
1201
1202 my $kvm_api_version = 0;
1203
1204 sub kvm_version {
1205     return $kvm_api_version if $kvm_api_version;
1206
1207     open my $fh, '<', '/dev/kvm' or return;
1208
1209     # 0xae00 => KVM_GET_API_VERSION
1210     $kvm_api_version = ioctl($fh, 0xae00, 0);
1211     close($fh);
1212
1213     return $kvm_api_version;
1214 }
1215
1216 my $kvm_user_version = {};
1217 my $kvm_mtime = {};
1218
1219 sub kvm_user_version {
1220     my ($binary) = @_;
1221
1222     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1223     my $st = stat($binary);
1224
1225     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1226     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1227         $cachedmtime == $st->mtime;
1228
1229     $kvm_user_version->{$binary} = 'unknown';
1230     $kvm_mtime->{$binary} = $st->mtime;
1231
1232     my $code = sub {
1233         my $line = shift;
1234         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1235             $kvm_user_version->{$binary} = $2;
1236         }
1237     };
1238
1239     eval { run_command([$binary, '--version'], outfunc => $code); };
1240     warn $@ if $@;
1241
1242     return $kvm_user_version->{$binary};
1243
1244 }
1245 my sub extract_version {
1246     my ($machine_type, $version) = @_;
1247     $version = kvm_user_version() if !defined($version);
1248     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1249 }
1250
1251 sub kernel_has_vhost_net {
1252     return -c '/dev/vhost-net';
1253 }
1254
1255 sub option_exists {
1256     my $key = shift;
1257     return defined($confdesc->{$key});
1258 }
1259
1260 my $cdrom_path;
1261 sub get_cdrom_path {
1262
1263     return $cdrom_path if defined($cdrom_path);
1264
1265     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1266
1267     if (!defined($cdrom_path)) {
1268         log_warn("no physical CD-ROM available, ignoring");
1269         $cdrom_path = '';
1270     }
1271
1272     return $cdrom_path;
1273 }
1274
1275 sub get_iso_path {
1276     my ($storecfg, $vmid, $cdrom) = @_;
1277
1278     if ($cdrom eq 'cdrom') {
1279         return get_cdrom_path();
1280     } elsif ($cdrom eq 'none') {
1281         return '';
1282     } elsif ($cdrom =~ m|^/|) {
1283         return $cdrom;
1284     } else {
1285         return PVE::Storage::path($storecfg, $cdrom);
1286     }
1287 }
1288
1289 # try to convert old style file names to volume IDs
1290 sub filename_to_volume_id {
1291     my ($vmid, $file, $media) = @_;
1292
1293      if (!($file eq 'none' || $file eq 'cdrom' ||
1294           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1295
1296         return if $file =~ m|/|;
1297
1298         if ($media && $media eq 'cdrom') {
1299             $file = "local:iso/$file";
1300         } else {
1301             $file = "local:$vmid/$file";
1302         }
1303     }
1304
1305     return $file;
1306 }
1307
1308 sub verify_media_type {
1309     my ($opt, $vtype, $media) = @_;
1310
1311     return if !$media;
1312
1313     my $etype;
1314     if ($media eq 'disk') {
1315         $etype = 'images';
1316     } elsif ($media eq 'cdrom') {
1317         $etype = 'iso';
1318     } else {
1319         die "internal error";
1320     }
1321
1322     return if ($vtype eq $etype);
1323
1324     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1325 }
1326
1327 sub cleanup_drive_path {
1328     my ($opt, $storecfg, $drive) = @_;
1329
1330     # try to convert filesystem paths to volume IDs
1331
1332     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1333         ($drive->{file} !~ m|^/dev/.+|) &&
1334         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1335         ($drive->{file} !~ m/^\d+$/)) {
1336         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1337         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1338             if !$vtype;
1339         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1340         verify_media_type($opt, $vtype, $drive->{media});
1341         $drive->{file} = $volid;
1342     }
1343
1344     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1345 }
1346
1347 sub parse_hotplug_features {
1348     my ($data) = @_;
1349
1350     my $res = {};
1351
1352     return $res if $data eq '0';
1353
1354     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1355
1356     foreach my $feature (PVE::Tools::split_list($data)) {
1357         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1358             $res->{$1} = 1;
1359         } else {
1360             die "invalid hotplug feature '$feature'\n";
1361         }
1362     }
1363     return $res;
1364 }
1365
1366 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1367 sub pve_verify_hotplug_features {
1368     my ($value, $noerr) = @_;
1369
1370     return $value if parse_hotplug_features($value);
1371
1372     return if $noerr;
1373
1374     die "unable to parse hotplug option\n";
1375 }
1376
1377 sub scsi_inquiry {
1378     my($fh, $noerr) = @_;
1379
1380     my $SG_IO = 0x2285;
1381     my $SG_GET_VERSION_NUM = 0x2282;
1382
1383     my $versionbuf = "\x00" x 8;
1384     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1385     if (!$ret) {
1386         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1387         return;
1388     }
1389     my $version = unpack("I", $versionbuf);
1390     if ($version < 30000) {
1391         die "scsi generic interface too old\n"  if !$noerr;
1392         return;
1393     }
1394
1395     my $buf = "\x00" x 36;
1396     my $sensebuf = "\x00" x 8;
1397     my $cmd = pack("C x3 C x1", 0x12, 36);
1398
1399     # see /usr/include/scsi/sg.h
1400     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1401
1402     my $packet = pack(
1403         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1404     );
1405
1406     $ret = ioctl($fh, $SG_IO, $packet);
1407     if (!$ret) {
1408         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1409         return;
1410     }
1411
1412     my @res = unpack($sg_io_hdr_t, $packet);
1413     if ($res[17] || $res[18]) {
1414         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1415         return;
1416     }
1417
1418     my $res = {};
1419     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1420
1421     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1422     $res->{type} &= 0x1F;
1423
1424     return $res;
1425 }
1426
1427 sub path_is_scsi {
1428     my ($path) = @_;
1429
1430     my $fh = IO::File->new("+<$path") || return;
1431     my $res = scsi_inquiry($fh, 1);
1432     close($fh);
1433
1434     return $res;
1435 }
1436
1437 sub print_tabletdevice_full {
1438     my ($conf, $arch) = @_;
1439
1440     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1441
1442     # we use uhci for old VMs because tablet driver was buggy in older qemu
1443     my $usbbus;
1444     if ($q35 || $arch eq 'aarch64') {
1445         $usbbus = 'ehci';
1446     } else {
1447         $usbbus = 'uhci';
1448     }
1449
1450     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1451 }
1452
1453 sub print_keyboarddevice_full {
1454     my ($conf, $arch) = @_;
1455
1456     return if $arch ne 'aarch64';
1457
1458     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1459 }
1460
1461 my sub get_drive_id {
1462     my ($drive) = @_;
1463     return "$drive->{interface}$drive->{index}";
1464 }
1465
1466 sub print_drivedevice_full {
1467     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1468
1469     my $device = '';
1470     my $maxdev = 0;
1471
1472     my $drive_id = get_drive_id($drive);
1473     if ($drive->{interface} eq 'virtio') {
1474         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1475         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1476         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1477     } elsif ($drive->{interface} eq 'scsi') {
1478
1479         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1480         my $unit = $drive->{index} % $maxdev;
1481         my $devicetype = 'hd';
1482         my $path = '';
1483         if (drive_is_cdrom($drive)) {
1484             $devicetype = 'cd';
1485         } else {
1486             if ($drive->{file} =~ m|^/|) {
1487                 $path = $drive->{file};
1488                 if (my $info = path_is_scsi($path)) {
1489                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1490                         $devicetype = 'block';
1491                     } elsif ($info->{type} == 1) { # tape
1492                         $devicetype = 'generic';
1493                     }
1494                 }
1495             } else {
1496                  $path = PVE::Storage::path($storecfg, $drive->{file});
1497             }
1498
1499             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1500             my $version = extract_version($machine_type, kvm_user_version());
1501             if ($path =~ m/^iscsi\:\/\// &&
1502                !min_version($version, 4, 1)) {
1503                 $devicetype = 'generic';
1504             }
1505         }
1506
1507         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1508             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1509         } else {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1511                 .",lun=$drive->{index}";
1512         }
1513         $device .= ",drive=drive-$drive_id,id=$drive_id";
1514
1515         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1516             $device .= ",rotation_rate=1";
1517         }
1518         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1519
1520     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1521         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1522         my $controller = int($drive->{index} / $maxdev);
1523         my $unit = $drive->{index} % $maxdev;
1524         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1525
1526         $device = "ide-$devicetype";
1527         if ($drive->{interface} eq 'ide') {
1528             $device .= ",bus=ide.$controller,unit=$unit";
1529         } else {
1530             $device .= ",bus=ahci$controller.$unit";
1531         }
1532         $device .= ",drive=drive-$drive_id,id=$drive_id";
1533
1534         if ($devicetype eq 'hd') {
1535             if (my $model = $drive->{model}) {
1536                 $model = URI::Escape::uri_unescape($model);
1537                 $device .= ",model=$model";
1538             }
1539             if ($drive->{ssd}) {
1540                 $device .= ",rotation_rate=1";
1541             }
1542         }
1543         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1544     } elsif ($drive->{interface} eq 'usb') {
1545         die "implement me";
1546         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1547     } else {
1548         die "unsupported interface type";
1549     }
1550
1551     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1552
1553     if (my $serial = $drive->{serial}) {
1554         $serial = URI::Escape::uri_unescape($serial);
1555         $device .= ",serial=$serial";
1556     }
1557
1558
1559     return $device;
1560 }
1561
1562 sub get_initiator_name {
1563     my $initiator;
1564
1565     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1566     while (defined(my $line = <$fh>)) {
1567         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1568         $initiator = $1;
1569         last;
1570     }
1571     $fh->close();
1572
1573     return $initiator;
1574 }
1575
1576 my sub storage_allows_io_uring_default {
1577     my ($scfg, $cache_direct) = @_;
1578
1579     # io_uring with cache mode writeback or writethrough on krbd will hang...
1580     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1581
1582     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1583     # sometimes, just plain disable...
1584     return if $scfg && $scfg->{type} eq 'lvm';
1585
1586     # io_uring causes problems when used with CIFS since kernel 5.15
1587     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1588     return if $scfg && $scfg->{type} eq 'cifs';
1589
1590     return 1;
1591 }
1592
1593 my sub drive_uses_cache_direct {
1594     my ($drive, $scfg) = @_;
1595
1596     my $cache_direct = 0;
1597
1598     if (my $cache = $drive->{cache}) {
1599         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1600     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1601         $cache_direct = 1;
1602     }
1603
1604     return $cache_direct;
1605 }
1606
1607 sub print_drive_commandline_full {
1608     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1609
1610     my $path;
1611     my $volid = $drive->{file};
1612     my $format = $drive->{format};
1613     my $drive_id = get_drive_id($drive);
1614
1615     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1616     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1617
1618     if (drive_is_cdrom($drive)) {
1619         $path = get_iso_path($storecfg, $vmid, $volid);
1620         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1621     } else {
1622         if ($storeid) {
1623             $path = PVE::Storage::path($storecfg, $volid);
1624             $format //= qemu_img_format($scfg, $volname);
1625         } else {
1626             $path = $volid;
1627             $format //= "raw";
1628         }
1629    }
1630
1631    my $is_rbd = $path =~ m/^rbd:/;
1632
1633     my $opts = '';
1634     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1635     foreach my $o (@qemu_drive_options) {
1636         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1637     }
1638
1639     # snapshot only accepts on|off
1640     if (defined($drive->{snapshot})) {
1641         my $v = $drive->{snapshot} ? 'on' : 'off';
1642         $opts .= ",snapshot=$v";
1643     }
1644
1645     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1646         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1647     }
1648
1649     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1650         my ($dir, $qmpname) = @$type;
1651         if (my $v = $drive->{"mbps$dir"}) {
1652             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1653         }
1654         if (my $v = $drive->{"mbps${dir}_max"}) {
1655             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1656         }
1657         if (my $v = $drive->{"bps${dir}_max_length"}) {
1658             $opts .= ",throttling.bps$qmpname-max-length=$v";
1659         }
1660         if (my $v = $drive->{"iops${dir}"}) {
1661             $opts .= ",throttling.iops$qmpname=$v";
1662         }
1663         if (my $v = $drive->{"iops${dir}_max"}) {
1664             $opts .= ",throttling.iops$qmpname-max=$v";
1665         }
1666         if (my $v = $drive->{"iops${dir}_max_length"}) {
1667             $opts .= ",throttling.iops$qmpname-max-length=$v";
1668         }
1669     }
1670
1671     if ($pbs_name) {
1672         $format = "rbd" if $is_rbd;
1673         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1674             if !$format;
1675         $opts .= ",format=alloc-track,file.driver=$format";
1676     } elsif ($format) {
1677         $opts .= ",format=$format";
1678     }
1679
1680     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1681
1682     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1683
1684     if (!$drive->{aio}) {
1685         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1686             # io_uring supports all cache modes
1687             $opts .= ",aio=io_uring";
1688         } else {
1689             # aio native works only with O_DIRECT
1690             if($cache_direct) {
1691                 $opts .= ",aio=native";
1692             } else {
1693                 $opts .= ",aio=threads";
1694             }
1695         }
1696     }
1697
1698     if (!drive_is_cdrom($drive)) {
1699         my $detectzeroes;
1700         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1701             $detectzeroes = 'off';
1702         } elsif ($drive->{discard}) {
1703             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1704         } else {
1705             # This used to be our default with discard not being specified:
1706             $detectzeroes = 'on';
1707         }
1708
1709         # note: 'detect-zeroes' works per blockdev and we want it to persist
1710         # after the alloc-track is removed, so put it on 'file' directly
1711         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1712         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1713     }
1714
1715     if ($pbs_name) {
1716         $opts .= ",backing=$pbs_name";
1717         $opts .= ",auto-remove=on";
1718     }
1719
1720     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1721     my $file_param = "file";
1722     if ($pbs_name) {
1723         # non-rbd drivers require the underlying file to be a seperate block
1724         # node, so add a second .file indirection
1725         $file_param .= ".file" if !$is_rbd;
1726         $file_param .= ".filename";
1727     }
1728     my $pathinfo = $path ? "$file_param=$path," : '';
1729
1730     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1731 }
1732
1733 sub print_pbs_blockdev {
1734     my ($pbs_conf, $pbs_name) = @_;
1735     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1736     $blockdev .= ",repository=$pbs_conf->{repository}";
1737     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1738     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1739     $blockdev .= ",archive=$pbs_conf->{archive}";
1740     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1741     return $blockdev;
1742 }
1743
1744 sub print_netdevice_full {
1745     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1746
1747     my $device = $net->{model};
1748     if ($net->{model} eq 'virtio') {
1749          $device = 'virtio-net-pci';
1750      };
1751
1752     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1753     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1754     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1755         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1756         # and out of each queue plus one config interrupt and control vector queue
1757         my $vectors = $net->{queues} * 2 + 2;
1758         $tmpstr .= ",vectors=$vectors,mq=on";
1759         if (min_version($machine_version, 7, 1)) {
1760             $tmpstr .= ",packed=on";
1761         }
1762     }
1763
1764     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1765         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1766     }
1767
1768     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1769
1770     if (my $mtu = $net->{mtu}) {
1771         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1772             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1773             if ($mtu == 1) {
1774                  $mtu = $bridge_mtu;
1775             } elsif ($mtu < 576) {
1776                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1777             } elsif ($mtu > $bridge_mtu) {
1778                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1779             }
1780             $tmpstr .= ",host_mtu=$mtu";
1781         } else {
1782             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1783         }
1784     }
1785
1786     if ($use_old_bios_files) {
1787         my $romfile;
1788         if ($device eq 'virtio-net-pci') {
1789             $romfile = 'pxe-virtio.rom';
1790         } elsif ($device eq 'e1000') {
1791             $romfile = 'pxe-e1000.rom';
1792         } elsif ($device eq 'e1000e') {
1793             $romfile = 'pxe-e1000e.rom';
1794         } elsif ($device eq 'ne2k') {
1795             $romfile = 'pxe-ne2k_pci.rom';
1796         } elsif ($device eq 'pcnet') {
1797             $romfile = 'pxe-pcnet.rom';
1798         } elsif ($device eq 'rtl8139') {
1799             $romfile = 'pxe-rtl8139.rom';
1800         }
1801         $tmpstr .= ",romfile=$romfile" if $romfile;
1802     }
1803
1804     return $tmpstr;
1805 }
1806
1807 sub print_netdev_full {
1808     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1809
1810     my $i = '';
1811     if ($netid =~ m/^net(\d+)$/) {
1812         $i = int($1);
1813     }
1814
1815     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1816
1817     my $ifname = "tap${vmid}i$i";
1818
1819     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1820     die "interface name '$ifname' is too long (max 15 character)\n"
1821         if length($ifname) >= 16;
1822
1823     my $vhostparam = '';
1824     if (is_native($arch)) {
1825         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1826     }
1827
1828     my $vmname = $conf->{name} || "vm$vmid";
1829
1830     my $netdev = "";
1831     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1832
1833     if ($net->{bridge}) {
1834         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1835             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1836     } else {
1837         $netdev = "type=user,id=$netid,hostname=$vmname";
1838     }
1839
1840     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1841
1842     return $netdev;
1843 }
1844
1845 my $vga_map = {
1846     'cirrus' => 'cirrus-vga',
1847     'std' => 'VGA',
1848     'vmware' => 'vmware-svga',
1849     'virtio' => 'virtio-vga',
1850     'virtio-gl' => 'virtio-vga-gl',
1851 };
1852
1853 sub print_vga_device {
1854     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1855
1856     my $type = $vga_map->{$vga->{type}};
1857     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1858         $type = 'virtio-gpu';
1859     }
1860     my $vgamem_mb = $vga->{memory};
1861
1862     my $max_outputs = '';
1863     if ($qxlnum) {
1864         $type = $id ? 'qxl' : 'qxl-vga';
1865
1866         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1867             # set max outputs so linux can have up to 4 qxl displays with one device
1868             if (min_version($machine_version, 4, 1)) {
1869                 $max_outputs = ",max_outputs=4";
1870             }
1871         }
1872     }
1873
1874     die "no devicetype for $vga->{type}\n" if !$type;
1875
1876     my $memory = "";
1877     if ($vgamem_mb) {
1878         if ($vga->{type} =~ /^virtio/) {
1879             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1880             $memory = ",max_hostmem=$bytes";
1881         } elsif ($qxlnum) {
1882             # from https://www.spice-space.org/multiple-monitors.html
1883             $memory = ",vgamem_mb=$vga->{memory}";
1884             my $ram = $vgamem_mb * 4;
1885             my $vram = $vgamem_mb * 2;
1886             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1887         } else {
1888             $memory = ",vgamem_mb=$vga->{memory}";
1889         }
1890     } elsif ($qxlnum && $id) {
1891         $memory = ",ram_size=67108864,vram_size=33554432";
1892     }
1893
1894     my $edidoff = "";
1895     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1896         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1897     }
1898
1899     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1900     my $vgaid = "vga" . ($id // '');
1901     my $pciaddr;
1902     if ($q35 && $vgaid eq 'vga') {
1903         # the first display uses pcie.0 bus on q35 machines
1904         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1905     } else {
1906         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1907     }
1908
1909     if ($vga->{type} eq 'virtio-gl') {
1910         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1911         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1912             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1913
1914         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1915             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1916     }
1917
1918     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1919 }
1920
1921 sub parse_number_sets {
1922     my ($set) = @_;
1923     my $res = [];
1924     foreach my $part (split(/;/, $set)) {
1925         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1926             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1927             push @$res, [ $1, $2 ];
1928         } else {
1929             die "invalid range: $part\n";
1930         }
1931     }
1932     return $res;
1933 }
1934
1935 sub parse_numa {
1936     my ($data) = @_;
1937
1938     my $res = parse_property_string($numa_fmt, $data);
1939     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1940     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1941     return $res;
1942 }
1943
1944 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1945 sub parse_net {
1946     my ($data, $disable_mac_autogen) = @_;
1947
1948     my $res = eval { parse_property_string($net_fmt, $data) };
1949     if ($@) {
1950         warn $@;
1951         return;
1952     }
1953     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1954         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1955         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1956     }
1957     return $res;
1958 }
1959
1960 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1961 sub parse_ipconfig {
1962     my ($data) = @_;
1963
1964     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1965     if ($@) {
1966         warn $@;
1967         return;
1968     }
1969
1970     if ($res->{gw} && !$res->{ip}) {
1971         warn 'gateway specified without specifying an IP address';
1972         return;
1973     }
1974     if ($res->{gw6} && !$res->{ip6}) {
1975         warn 'IPv6 gateway specified without specifying an IPv6 address';
1976         return;
1977     }
1978     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1979         warn 'gateway specified together with DHCP';
1980         return;
1981     }
1982     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1983         # gw6 + auto/dhcp
1984         warn "IPv6 gateway specified together with $res->{ip6} address";
1985         return;
1986     }
1987
1988     if (!$res->{ip} && !$res->{ip6}) {
1989         return { ip => 'dhcp', ip6 => 'dhcp' };
1990     }
1991
1992     return $res;
1993 }
1994
1995 sub print_net {
1996     my $net = shift;
1997
1998     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1999 }
2000
2001 sub add_random_macs {
2002     my ($settings) = @_;
2003
2004     foreach my $opt (keys %$settings) {
2005         next if $opt !~ m/^net(\d+)$/;
2006         my $net = parse_net($settings->{$opt});
2007         next if !$net;
2008         $settings->{$opt} = print_net($net);
2009     }
2010 }
2011
2012 sub vm_is_volid_owner {
2013     my ($storecfg, $vmid, $volid) = @_;
2014
2015     if ($volid !~  m|^/|) {
2016         my ($path, $owner);
2017         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2018         if ($owner && ($owner == $vmid)) {
2019             return 1;
2020         }
2021     }
2022
2023     return;
2024 }
2025
2026 sub vmconfig_register_unused_drive {
2027     my ($storecfg, $vmid, $conf, $drive) = @_;
2028
2029     if (drive_is_cloudinit($drive)) {
2030         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2031         warn $@ if $@;
2032         delete $conf->{cloudinit};
2033     } elsif (!drive_is_cdrom($drive)) {
2034         my $volid = $drive->{file};
2035         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2036             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2037         }
2038     }
2039 }
2040
2041 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2042 my $smbios1_fmt = {
2043     uuid => {
2044         type => 'string',
2045         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2046         format_description => 'UUID',
2047         description => "Set SMBIOS1 UUID.",
2048         optional => 1,
2049     },
2050     version => {
2051         type => 'string',
2052         pattern => '[A-Za-z0-9+\/]+={0,2}',
2053         format_description => 'Base64 encoded string',
2054         description => "Set SMBIOS1 version.",
2055         optional => 1,
2056     },
2057     serial => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 serial number.",
2062         optional => 1,
2063     },
2064     manufacturer => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 manufacturer.",
2069         optional => 1,
2070     },
2071     product => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 product ID.",
2076         optional => 1,
2077     },
2078     sku => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 SKU string.",
2083         optional => 1,
2084     },
2085     family => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 family string.",
2090         optional => 1,
2091     },
2092     base64 => {
2093         type => 'boolean',
2094         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2095         optional => 1,
2096     },
2097 };
2098
2099 sub parse_smbios1 {
2100     my ($data) = @_;
2101
2102     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2103     warn $@ if $@;
2104     return $res;
2105 }
2106
2107 sub print_smbios1 {
2108     my ($smbios1) = @_;
2109     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2110 }
2111
2112 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2113
2114 sub parse_watchdog {
2115     my ($value) = @_;
2116
2117     return if !$value;
2118
2119     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2120     warn $@ if $@;
2121     return $res;
2122 }
2123
2124 sub parse_guest_agent {
2125     my ($conf) = @_;
2126
2127     return {} if !defined($conf->{agent});
2128
2129     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2130     warn $@ if $@;
2131
2132     # if the agent is disabled ignore the other potentially set properties
2133     return {} if !$res->{enabled};
2134     return $res;
2135 }
2136
2137 sub get_qga_key {
2138     my ($conf, $key) = @_;
2139     return undef if !defined($conf->{agent});
2140
2141     my $agent = parse_guest_agent($conf);
2142     return $agent->{$key};
2143 }
2144
2145 sub parse_vga {
2146     my ($value) = @_;
2147
2148     return {} if !$value;
2149     my $res = eval { parse_property_string($vga_fmt, $value) };
2150     warn $@ if $@;
2151     return $res;
2152 }
2153
2154 sub parse_rng {
2155     my ($value) = @_;
2156
2157     return if !$value;
2158
2159     my $res = eval { parse_property_string($rng_fmt, $value) };
2160     warn $@ if $@;
2161     return $res;
2162 }
2163
2164 sub parse_meta_info {
2165     my ($value) = @_;
2166
2167     return if !$value;
2168
2169     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2170     warn $@ if $@;
2171     return $res;
2172 }
2173
2174 sub new_meta_info_string {
2175     my () = @_; # for now do not allow to override any value
2176
2177     return PVE::JSONSchema::print_property_string(
2178         {
2179             'creation-qemu' => kvm_user_version(),
2180             ctime => "". int(time()),
2181         },
2182         $meta_info_fmt
2183     );
2184 }
2185
2186 sub qemu_created_version_fixups {
2187     my ($conf, $forcemachine, $kvmver) = @_;
2188
2189     my $meta = parse_meta_info($conf->{meta}) // {};
2190     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2191
2192     # check if we need to apply some handling for VMs that always use the latest machine version but
2193     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2194     # would be required (we do not want to pin machine version for non-windows OS type)
2195     if (
2196         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2197         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2198         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2199         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2200     ) {
2201         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2202         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2203             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2204             # and thus with the predictable interface naming of systemd
2205             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2206         }
2207     }
2208     return;
2209 }
2210
2211 # add JSON properties for create and set function
2212 sub json_config_properties {
2213     my ($prop, $with_disk_alloc) = @_;
2214
2215     my $skip_json_config_opts = {
2216         parent => 1,
2217         snaptime => 1,
2218         vmstate => 1,
2219         runningmachine => 1,
2220         runningcpu => 1,
2221         meta => 1,
2222     };
2223
2224     foreach my $opt (keys %$confdesc) {
2225         next if $skip_json_config_opts->{$opt};
2226
2227         if ($with_disk_alloc && is_valid_drivename($opt)) {
2228             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2229         } else {
2230             $prop->{$opt} = $confdesc->{$opt};
2231         }
2232     }
2233
2234     return $prop;
2235 }
2236
2237 # Properties that we can read from an OVF file
2238 sub json_ovf_properties {
2239     my $prop = {};
2240
2241     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2242         $prop->{$device} = {
2243             type => 'string',
2244             format => 'pve-volume-id-or-absolute-path',
2245             description => "Disk image that gets imported to $device",
2246             optional => 1,
2247         };
2248     }
2249
2250     $prop->{cores} = {
2251         type => 'integer',
2252         description => "The number of CPU cores.",
2253         optional => 1,
2254     };
2255     $prop->{memory} = {
2256         type => 'integer',
2257         description => "Amount of RAM for the VM in MB.",
2258         optional => 1,
2259     };
2260     $prop->{name} = {
2261         type => 'string',
2262         description => "Name of the VM.",
2263         optional => 1,
2264     };
2265
2266     return $prop;
2267 }
2268
2269 # return copy of $confdesc_cloudinit to generate documentation
2270 sub cloudinit_config_properties {
2271
2272     return dclone($confdesc_cloudinit);
2273 }
2274
2275 sub cloudinit_pending_properties {
2276     my $p = {
2277         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2278         name => 1,
2279     };
2280     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2281     return $p;
2282 }
2283
2284 sub check_type {
2285     my ($key, $value) = @_;
2286
2287     die "unknown setting '$key'\n" if !$confdesc->{$key};
2288
2289     my $type = $confdesc->{$key}->{type};
2290
2291     if (!defined($value)) {
2292         die "got undefined value\n";
2293     }
2294
2295     if ($value =~ m/[\n\r]/) {
2296         die "property contains a line feed\n";
2297     }
2298
2299     if ($type eq 'boolean') {
2300         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2301         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2302         die "type check ('boolean') failed - got '$value'\n";
2303     } elsif ($type eq 'integer') {
2304         return int($1) if $value =~ m/^(\d+)$/;
2305         die "type check ('integer') failed - got '$value'\n";
2306     } elsif ($type eq 'number') {
2307         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2308         die "type check ('number') failed - got '$value'\n";
2309     } elsif ($type eq 'string') {
2310         if (my $fmt = $confdesc->{$key}->{format}) {
2311             PVE::JSONSchema::check_format($fmt, $value);
2312             return $value;
2313         }
2314         $value =~ s/^\"(.*)\"$/$1/;
2315         return $value;
2316     } else {
2317         die "internal error"
2318     }
2319 }
2320
2321 sub destroy_vm {
2322     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2323
2324     my $conf = PVE::QemuConfig->load_config($vmid);
2325
2326     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2327         PVE::QemuConfig->check_lock($conf);
2328     }
2329
2330     if ($conf->{template}) {
2331         # check if any base image is still used by a linked clone
2332         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2333                 my ($ds, $drive) = @_;
2334                 return if drive_is_cdrom($drive);
2335
2336                 my $volid = $drive->{file};
2337                 return if !$volid || $volid =~ m|^/|;
2338
2339                 die "base volume '$volid' is still in use by linked cloned\n"
2340                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2341
2342         });
2343     }
2344
2345     my $volids = {};
2346     my $remove_owned_drive = sub {
2347         my ($ds, $drive) = @_;
2348         return if drive_is_cdrom($drive, 1);
2349
2350         my $volid = $drive->{file};
2351         return if !$volid || $volid =~ m|^/|;
2352         return if $volids->{$volid};
2353
2354         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2355         return if !$path || !$owner || ($owner != $vmid);
2356
2357         $volids->{$volid} = 1;
2358         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2359         warn "Could not remove disk '$volid', check manually: $@" if $@;
2360     };
2361
2362     # only remove disks owned by this VM (referenced in the config)
2363     my $include_opts = {
2364         include_unused => 1,
2365         extra_keys => ['vmstate'],
2366     };
2367     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2368
2369     for my $snap (values %{$conf->{snapshots}}) {
2370         next if !defined($snap->{vmstate});
2371         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2372         next if !defined($drive);
2373         $remove_owned_drive->('vmstate', $drive);
2374     }
2375
2376     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2377
2378     if ($purge_unreferenced) { # also remove unreferenced disk
2379         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2380         PVE::Storage::foreach_volid($vmdisks, sub {
2381             my ($volid, $sid, $volname, $d) = @_;
2382             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2383             warn $@ if $@;
2384         });
2385     }
2386
2387     if (defined $replacement_conf) {
2388         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2389     } else {
2390         PVE::QemuConfig->destroy_config($vmid);
2391     }
2392 }
2393
2394 sub parse_vm_config {
2395     my ($filename, $raw, $strict) = @_;
2396
2397     return if !defined($raw);
2398
2399     my $res = {
2400         digest => Digest::SHA::sha1_hex($raw),
2401         snapshots => {},
2402         pending => {},
2403         cloudinit => {},
2404     };
2405
2406     my $handle_error = sub {
2407         my ($msg) = @_;
2408
2409         if ($strict) {
2410             die $msg;
2411         } else {
2412             warn $msg;
2413         }
2414     };
2415
2416     $filename =~ m|/qemu-server/(\d+)\.conf$|
2417         || die "got strange filename '$filename'";
2418
2419     my $vmid = $1;
2420
2421     my $conf = $res;
2422     my $descr;
2423     my $finish_description = sub {
2424         if (defined($descr)) {
2425             $descr =~ s/\s+$//;
2426             $conf->{description} = $descr;
2427         }
2428         $descr = undef;
2429     };
2430     my $section = '';
2431
2432     my @lines = split(/\n/, $raw);
2433     foreach my $line (@lines) {
2434         next if $line =~ m/^\s*$/;
2435
2436         if ($line =~ m/^\[PENDING\]\s*$/i) {
2437             $section = 'pending';
2438             $finish_description->();
2439             $conf = $res->{$section} = {};
2440             next;
2441         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2442             $section = 'cloudinit';
2443             $finish_description->();
2444             $conf = $res->{$section} = {};
2445             next;
2446
2447         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2448             $section = $1;
2449             $finish_description->();
2450             $conf = $res->{snapshots}->{$section} = {};
2451             next;
2452         }
2453
2454         if ($line =~ m/^\#(.*)$/) {
2455             $descr = '' if !defined($descr);
2456             $descr .= PVE::Tools::decode_text($1) . "\n";
2457             next;
2458         }
2459
2460         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2461             $descr = '' if !defined($descr);
2462             $descr .= PVE::Tools::decode_text($2);
2463         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2464             $conf->{snapstate} = $1;
2465         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2466             my $key = $1;
2467             my $value = $2;
2468             $conf->{$key} = $value;
2469         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2470             my $value = $1;
2471             if ($section eq 'pending') {
2472                 $conf->{delete} = $value; # we parse this later
2473             } else {
2474                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2475             }
2476         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2477             my $key = $1;
2478             my $value = $2;
2479             if ($section eq 'cloudinit') {
2480                 # ignore validation only used for informative purpose
2481                 $conf->{$key} = $value;
2482                 next;
2483             }
2484             eval { $value = check_type($key, $value); };
2485             if ($@) {
2486                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2487             } else {
2488                 $key = 'ide2' if $key eq 'cdrom';
2489                 my $fmt = $confdesc->{$key}->{format};
2490                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2491                     my $v = parse_drive($key, $value);
2492                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2493                         $v->{file} = $volid;
2494                         $value = print_drive($v);
2495                     } else {
2496                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2497                         next;
2498                     }
2499                 }
2500
2501                 $conf->{$key} = $value;
2502             }
2503         } else {
2504             $handle_error->("vm $vmid - unable to parse config: $line\n");
2505         }
2506     }
2507
2508     $finish_description->();
2509     delete $res->{snapstate}; # just to be sure
2510
2511     return $res;
2512 }
2513
2514 sub write_vm_config {
2515     my ($filename, $conf) = @_;
2516
2517     delete $conf->{snapstate}; # just to be sure
2518
2519     if ($conf->{cdrom}) {
2520         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2521         $conf->{ide2} = $conf->{cdrom};
2522         delete $conf->{cdrom};
2523     }
2524
2525     # we do not use 'smp' any longer
2526     if ($conf->{sockets}) {
2527         delete $conf->{smp};
2528     } elsif ($conf->{smp}) {
2529         $conf->{sockets} = $conf->{smp};
2530         delete $conf->{cores};
2531         delete $conf->{smp};
2532     }
2533
2534     my $used_volids = {};
2535
2536     my $cleanup_config = sub {
2537         my ($cref, $pending, $snapname) = @_;
2538
2539         foreach my $key (keys %$cref) {
2540             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2541                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2542             my $value = $cref->{$key};
2543             if ($key eq 'delete') {
2544                 die "propertry 'delete' is only allowed in [PENDING]\n"
2545                     if !$pending;
2546                 # fixme: check syntax?
2547                 next;
2548             }
2549             eval { $value = check_type($key, $value); };
2550             die "unable to parse value of '$key' - $@" if $@;
2551
2552             $cref->{$key} = $value;
2553
2554             if (!$snapname && is_valid_drivename($key)) {
2555                 my $drive = parse_drive($key, $value);
2556                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2557             }
2558         }
2559     };
2560
2561     &$cleanup_config($conf);
2562
2563     &$cleanup_config($conf->{pending}, 1);
2564
2565     foreach my $snapname (keys %{$conf->{snapshots}}) {
2566         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2567         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2568     }
2569
2570     # remove 'unusedX' settings if we re-add a volume
2571     foreach my $key (keys %$conf) {
2572         my $value = $conf->{$key};
2573         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2574             delete $conf->{$key};
2575         }
2576     }
2577
2578     my $generate_raw_config = sub {
2579         my ($conf, $pending) = @_;
2580
2581         my $raw = '';
2582
2583         # add description as comment to top of file
2584         if (defined(my $descr = $conf->{description})) {
2585             if ($descr) {
2586                 foreach my $cl (split(/\n/, $descr)) {
2587                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2588                 }
2589             } else {
2590                 $raw .= "#\n" if $pending;
2591             }
2592         }
2593
2594         foreach my $key (sort keys %$conf) {
2595             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2596             $raw .= "$key: $conf->{$key}\n";
2597         }
2598         return $raw;
2599     };
2600
2601     my $raw = &$generate_raw_config($conf);
2602
2603     if (scalar(keys %{$conf->{pending}})){
2604         $raw .= "\n[PENDING]\n";
2605         $raw .= &$generate_raw_config($conf->{pending}, 1);
2606     }
2607
2608     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2609         $raw .= "\n[special:cloudinit]\n";
2610         $raw .= &$generate_raw_config($conf->{cloudinit});
2611     }
2612
2613     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2614         $raw .= "\n[$snapname]\n";
2615         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2616     }
2617
2618     return $raw;
2619 }
2620
2621 sub load_defaults {
2622
2623     my $res = {};
2624
2625     # we use static defaults from our JSON schema configuration
2626     foreach my $key (keys %$confdesc) {
2627         if (defined(my $default = $confdesc->{$key}->{default})) {
2628             $res->{$key} = $default;
2629         }
2630     }
2631
2632     return $res;
2633 }
2634
2635 sub config_list {
2636     my $vmlist = PVE::Cluster::get_vmlist();
2637     my $res = {};
2638     return $res if !$vmlist || !$vmlist->{ids};
2639     my $ids = $vmlist->{ids};
2640     my $nodename = nodename();
2641
2642     foreach my $vmid (keys %$ids) {
2643         my $d = $ids->{$vmid};
2644         next if !$d->{node} || $d->{node} ne $nodename;
2645         next if !$d->{type} || $d->{type} ne 'qemu';
2646         $res->{$vmid}->{exists} = 1;
2647     }
2648     return $res;
2649 }
2650
2651 # test if VM uses local resources (to prevent migration)
2652 sub check_local_resources {
2653     my ($conf, $noerr) = @_;
2654
2655     my @loc_res = ();
2656     my $mapped_res = [];
2657
2658     my $nodelist = PVE::Cluster::get_nodelist();
2659     my $pci_map = PVE::Mapping::PCI::config();
2660     my $usb_map = PVE::Mapping::USB::config();
2661
2662     my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2663
2664     my $add_missing_mapping = sub {
2665         my ($type, $key, $id) = @_;
2666         for my $node (@$nodelist) {
2667             my $entry;
2668             if ($type eq 'pci') {
2669                 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2670             } elsif ($type eq 'usb') {
2671                 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2672             }
2673             if (!scalar($entry->@*)) {
2674                 push @{$missing_mappings_by_node->{$node}}, $key;
2675             }
2676         }
2677     };
2678
2679     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2680     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2681
2682     push @loc_res, "ivshmem" if $conf->{ivshmem};
2683
2684     foreach my $k (keys %$conf) {
2685         if ($k =~ m/^usb/) {
2686             my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2687             next if $entry->{host} =~ m/^spice$/i;
2688             if ($entry->{mapping}) {
2689                 $add_missing_mapping->('usb', $k, $entry->{mapping});
2690                 push @$mapped_res, $k;
2691             }
2692         }
2693         if ($k =~ m/^hostpci/) {
2694             my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2695             if ($entry->{mapping}) {
2696                 $add_missing_mapping->('pci', $k, $entry->{mapping});
2697                 push @$mapped_res, $k;
2698             }
2699         }
2700         # sockets are safe: they will recreated be on the target side post-migrate
2701         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2702         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2703     }
2704
2705     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2706
2707     return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2708 }
2709
2710 # check if used storages are available on all nodes (use by migrate)
2711 sub check_storage_availability {
2712     my ($storecfg, $conf, $node) = @_;
2713
2714     PVE::QemuConfig->foreach_volume($conf, sub {
2715         my ($ds, $drive) = @_;
2716
2717         my $volid = $drive->{file};
2718         return if !$volid;
2719
2720         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2721         return if !$sid;
2722
2723         # check if storage is available on both nodes
2724         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2725         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2726
2727         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2728
2729         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2730             if !$scfg->{content}->{$vtype};
2731    });
2732 }
2733
2734 # list nodes where all VM images are available (used by has_feature API)
2735 sub shared_nodes {
2736     my ($conf, $storecfg) = @_;
2737
2738     my $nodelist = PVE::Cluster::get_nodelist();
2739     my $nodehash = { map { $_ => 1 } @$nodelist };
2740     my $nodename = nodename();
2741
2742     PVE::QemuConfig->foreach_volume($conf, sub {
2743         my ($ds, $drive) = @_;
2744
2745         my $volid = $drive->{file};
2746         return if !$volid;
2747
2748         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2749         if ($storeid) {
2750             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2751             if ($scfg->{disable}) {
2752                 $nodehash = {};
2753             } elsif (my $avail = $scfg->{nodes}) {
2754                 foreach my $node (keys %$nodehash) {
2755                     delete $nodehash->{$node} if !$avail->{$node};
2756                 }
2757             } elsif (!$scfg->{shared}) {
2758                 foreach my $node (keys %$nodehash) {
2759                     delete $nodehash->{$node} if $node ne $nodename
2760                 }
2761             }
2762         }
2763     });
2764
2765     return $nodehash
2766 }
2767
2768 sub check_local_storage_availability {
2769     my ($conf, $storecfg) = @_;
2770
2771     my $nodelist = PVE::Cluster::get_nodelist();
2772     my $nodehash = { map { $_ => {} } @$nodelist };
2773
2774     PVE::QemuConfig->foreach_volume($conf, sub {
2775         my ($ds, $drive) = @_;
2776
2777         my $volid = $drive->{file};
2778         return if !$volid;
2779
2780         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2781         if ($storeid) {
2782             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2783
2784             if ($scfg->{disable}) {
2785                 foreach my $node (keys %$nodehash) {
2786                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2787                 }
2788             } elsif (my $avail = $scfg->{nodes}) {
2789                 foreach my $node (keys %$nodehash) {
2790                     if (!$avail->{$node}) {
2791                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2792                     }
2793                 }
2794             }
2795         }
2796     });
2797
2798     foreach my $node (values %$nodehash) {
2799         if (my $unavail = $node->{unavailable_storages}) {
2800             $node->{unavailable_storages} = [ sort keys %$unavail ];
2801         }
2802     }
2803
2804     return $nodehash
2805 }
2806
2807 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2808 sub check_running {
2809     my ($vmid, $nocheck, $node) = @_;
2810
2811     # $nocheck is set when called during a migration, in which case the config
2812     # file might still or already reside on the *other* node
2813     # - because rename has already happened, and current node is source
2814     # - because rename hasn't happened yet, and current node is target
2815     # - because rename has happened, current node is target, but hasn't yet
2816     # processed it yet
2817     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2818     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2819 }
2820
2821 sub vzlist {
2822
2823     my $vzlist = config_list();
2824
2825     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2826
2827     while (defined(my $de = $fd->read)) {
2828         next if $de !~ m/^(\d+)\.pid$/;
2829         my $vmid = $1;
2830         next if !defined($vzlist->{$vmid});
2831         if (my $pid = check_running($vmid)) {
2832             $vzlist->{$vmid}->{pid} = $pid;
2833         }
2834     }
2835
2836     return $vzlist;
2837 }
2838
2839 our $vmstatus_return_properties = {
2840     vmid => get_standard_option('pve-vmid'),
2841     status => {
2842         description => "QEMU process status.",
2843         type => 'string',
2844         enum => ['stopped', 'running'],
2845     },
2846     maxmem => {
2847         description => "Maximum memory in bytes.",
2848         type => 'integer',
2849         optional => 1,
2850         renderer => 'bytes',
2851     },
2852     maxdisk => {
2853         description => "Root disk size in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     name => {
2859         description => "VM name.",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     qmpstatus => {
2864         description => "VM run state from the 'query-status' QMP monitor command.",
2865         type => 'string',
2866         optional => 1,
2867     },
2868     pid => {
2869         description => "PID of running qemu process.",
2870         type => 'integer',
2871         optional => 1,
2872     },
2873     uptime => {
2874         description => "Uptime.",
2875         type => 'integer',
2876         optional => 1,
2877         renderer => 'duration',
2878     },
2879     cpus => {
2880         description => "Maximum usable CPUs.",
2881         type => 'number',
2882         optional => 1,
2883     },
2884     lock => {
2885         description => "The current config lock, if any.",
2886         type => 'string',
2887         optional => 1,
2888     },
2889     tags => {
2890         description  => "The current configured tags, if any",
2891         type => 'string',
2892         optional => 1,
2893     },
2894     'running-machine' => {
2895         description => "The currently running machine type (if running).",
2896         type => 'string',
2897         optional => 1,
2898     },
2899     'running-qemu' => {
2900         description => "The currently running QEMU version (if running).",
2901         type => 'string',
2902         optional => 1,
2903     },
2904 };
2905
2906 my $last_proc_pid_stat;
2907
2908 # get VM status information
2909 # This must be fast and should not block ($full == false)
2910 # We only query KVM using QMP if $full == true (this can be slow)
2911 sub vmstatus {
2912     my ($opt_vmid, $full) = @_;
2913
2914     my $res = {};
2915
2916     my $storecfg = PVE::Storage::config();
2917
2918     my $list = vzlist();
2919     my $defaults = load_defaults();
2920
2921     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2922
2923     my $cpucount = $cpuinfo->{cpus} || 1;
2924
2925     foreach my $vmid (keys %$list) {
2926         next if $opt_vmid && ($vmid ne $opt_vmid);
2927
2928         my $conf = PVE::QemuConfig->load_config($vmid);
2929
2930         my $d = { vmid => int($vmid) };
2931         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2932
2933         # fixme: better status?
2934         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2935
2936         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2937         if (defined($size)) {
2938             $d->{disk} = 0; # no info available
2939             $d->{maxdisk} = $size;
2940         } else {
2941             $d->{disk} = 0;
2942             $d->{maxdisk} = 0;
2943         }
2944
2945         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2946             * ($conf->{cores} || $defaults->{cores});
2947         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2948         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2949
2950         $d->{name} = $conf->{name} || "VM $vmid";
2951         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2952             : $defaults->{memory}*(1024*1024);
2953
2954         if ($conf->{balloon}) {
2955             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2956             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2957                 : $defaults->{shares};
2958         }
2959
2960         $d->{uptime} = 0;
2961         $d->{cpu} = 0;
2962         $d->{mem} = 0;
2963
2964         $d->{netout} = 0;
2965         $d->{netin} = 0;
2966
2967         $d->{diskread} = 0;
2968         $d->{diskwrite} = 0;
2969
2970         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2971
2972         $d->{serial} = 1 if conf_has_serial($conf);
2973         $d->{lock} = $conf->{lock} if $conf->{lock};
2974         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2975
2976         $res->{$vmid} = $d;
2977     }
2978
2979     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2980     foreach my $dev (keys %$netdev) {
2981         next if $dev !~ m/^tap([1-9]\d*)i/;
2982         my $vmid = $1;
2983         my $d = $res->{$vmid};
2984         next if !$d;
2985
2986         $d->{netout} += $netdev->{$dev}->{receive};
2987         $d->{netin} += $netdev->{$dev}->{transmit};
2988
2989         if ($full) {
2990             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2991             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2992         }
2993
2994     }
2995
2996     my $ctime = gettimeofday;
2997
2998     foreach my $vmid (keys %$list) {
2999
3000         my $d = $res->{$vmid};
3001         my $pid = $d->{pid};
3002         next if !$pid;
3003
3004         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3005         next if !$pstat; # not running
3006
3007         my $used = $pstat->{utime} + $pstat->{stime};
3008
3009         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3010
3011         if ($pstat->{vsize}) {
3012             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3013         }
3014
3015         my $old = $last_proc_pid_stat->{$pid};
3016         if (!$old) {
3017             $last_proc_pid_stat->{$pid} = {
3018                 time => $ctime,
3019                 used => $used,
3020                 cpu => 0,
3021             };
3022             next;
3023         }
3024
3025         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3026
3027         if ($dtime > 1000) {
3028             my $dutime = $used -  $old->{used};
3029
3030             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3031             $last_proc_pid_stat->{$pid} = {
3032                 time => $ctime,
3033                 used => $used,
3034                 cpu => $d->{cpu},
3035             };
3036         } else {
3037             $d->{cpu} = $old->{cpu};
3038         }
3039     }
3040
3041     return $res if !$full;
3042
3043     my $qmpclient = PVE::QMPClient->new();
3044
3045     my $ballooncb = sub {
3046         my ($vmid, $resp) = @_;
3047
3048         my $info = $resp->{'return'};
3049         return if !$info->{max_mem};
3050
3051         my $d = $res->{$vmid};
3052
3053         # use memory assigned to VM
3054         $d->{maxmem} = $info->{max_mem};
3055         $d->{balloon} = $info->{actual};
3056
3057         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3058             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3059             $d->{freemem} = $info->{free_mem};
3060         }
3061
3062         $d->{ballooninfo} = $info;
3063     };
3064
3065     my $blockstatscb = sub {
3066         my ($vmid, $resp) = @_;
3067         my $data = $resp->{'return'} || [];
3068         my $totalrdbytes = 0;
3069         my $totalwrbytes = 0;
3070
3071         for my $blockstat (@$data) {
3072             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3073             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3074
3075             $blockstat->{device} =~ s/drive-//;
3076             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3077         }
3078         $res->{$vmid}->{diskread} = $totalrdbytes;
3079         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3080     };
3081
3082     my $machinecb = sub {
3083         my ($vmid, $resp) = @_;
3084         my $data = $resp->{'return'} || [];
3085
3086         $res->{$vmid}->{'running-machine'} =
3087             PVE::QemuServer::Machine::current_from_query_machines($data);
3088     };
3089
3090     my $versioncb = sub {
3091         my ($vmid, $resp) = @_;
3092         my $data = $resp->{'return'} // {};
3093         my $version = 'unknown';
3094
3095         if (my $v = $data->{qemu}) {
3096             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3097         }
3098
3099         $res->{$vmid}->{'running-qemu'} = $version;
3100     };
3101
3102     my $statuscb = sub {
3103         my ($vmid, $resp) = @_;
3104
3105         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3106         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3107         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3108         # this fails if ballon driver is not loaded, so this must be
3109         # the last commnand (following command are aborted if this fails).
3110         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3111
3112         my $status = 'unknown';
3113         if (!defined($status = $resp->{'return'}->{status})) {
3114             warn "unable to get VM status\n";
3115             return;
3116         }
3117
3118         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3119     };
3120
3121     foreach my $vmid (keys %$list) {
3122         next if $opt_vmid && ($vmid ne $opt_vmid);
3123         next if !$res->{$vmid}->{pid}; # not running
3124         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3125     }
3126
3127     $qmpclient->queue_execute(undef, 2);
3128
3129     foreach my $vmid (keys %$list) {
3130         next if $opt_vmid && ($vmid ne $opt_vmid);
3131         next if !$res->{$vmid}->{pid}; #not running
3132
3133         # we can't use the $qmpclient since it might have already aborted on
3134         # 'query-balloon', but this might also fail for older versions...
3135         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3136         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3137     }
3138
3139     foreach my $vmid (keys %$list) {
3140         next if $opt_vmid && ($vmid ne $opt_vmid);
3141         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3142     }
3143
3144     return $res;
3145 }
3146
3147 sub conf_has_serial {
3148     my ($conf) = @_;
3149
3150     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3151         if ($conf->{"serial$i"}) {
3152             return 1;
3153         }
3154     }
3155
3156     return 0;
3157 }
3158
3159 sub conf_has_audio {
3160     my ($conf, $id) = @_;
3161
3162     $id //= 0;
3163     my $audio = $conf->{"audio$id"};
3164     return if !defined($audio);
3165
3166     my $audioproperties = parse_property_string($audio_fmt, $audio);
3167     my $audiodriver = $audioproperties->{driver} // 'spice';
3168
3169     return {
3170         dev => $audioproperties->{device},
3171         dev_id => "audiodev$id",
3172         backend => $audiodriver,
3173         backend_id => "$audiodriver-backend${id}",
3174     };
3175 }
3176
3177 sub audio_devs {
3178     my ($audio, $audiopciaddr, $machine_version) = @_;
3179
3180     my $devs = [];
3181
3182     my $id = $audio->{dev_id};
3183     my $audiodev = "";
3184     if (min_version($machine_version, 4, 2)) {
3185         $audiodev = ",audiodev=$audio->{backend_id}";
3186     }
3187
3188     if ($audio->{dev} eq 'AC97') {
3189         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3190     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3191         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3192         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3193         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3194     } else {
3195         die "unkown audio device '$audio->{dev}', implement me!";
3196     }
3197
3198     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3199
3200     return $devs;
3201 }
3202
3203 sub get_tpm_paths {
3204     my ($vmid) = @_;
3205     return {
3206         socket => "/var/run/qemu-server/$vmid.swtpm",
3207         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3208     };
3209 }
3210
3211 sub add_tpm_device {
3212     my ($vmid, $devices, $conf) = @_;
3213
3214     return if !$conf->{tpmstate0};
3215
3216     my $paths = get_tpm_paths($vmid);
3217
3218     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3219     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3220     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3221 }
3222
3223 sub start_swtpm {
3224     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3225
3226     return if !$tpmdrive;
3227
3228     my $state;
3229     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3230     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3231     if ($storeid) {
3232         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3233     } else {
3234         $state = $tpm->{file};
3235     }
3236
3237     my $paths = get_tpm_paths($vmid);
3238
3239     # during migration, we will get state from remote
3240     #
3241     if (!$migration) {
3242         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3243         my $setup_cmd = [
3244             "swtpm_setup",
3245             "--tpmstate",
3246             "file://$state",
3247             "--createek",
3248             "--create-ek-cert",
3249             "--create-platform-cert",
3250             "--lock-nvram",
3251             "--config",
3252             "/etc/swtpm_setup.conf", # do not use XDG configs
3253             "--runas",
3254             "0", # force creation as root, error if not possible
3255             "--not-overwrite", # ignore existing state, do not modify
3256         ];
3257
3258         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3259         # TPM 2.0 supports ECC crypto, use if possible
3260         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3261
3262         run_command($setup_cmd, outfunc => sub {
3263             print "swtpm_setup: $1\n";
3264         });
3265     }
3266
3267     # Used to distinguish different invocations in the log.
3268     my $log_prefix = "[id=" . int(time()) . "] ";
3269
3270     my $emulator_cmd = [
3271         "swtpm",
3272         "socket",
3273         "--tpmstate",
3274         "backend-uri=file://$state,mode=0600",
3275         "--ctrl",
3276         "type=unixio,path=$paths->{socket},mode=0600",
3277         "--pid",
3278         "file=$paths->{pid}",
3279         "--terminate", # terminate on QEMU disconnect
3280         "--daemon",
3281         "--log",
3282         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3283     ];
3284     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3285     run_command($emulator_cmd, outfunc => sub { print $1; });
3286
3287     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3288     while (! -e $paths->{pid}) {
3289         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3290         usleep(50_000);
3291     }
3292
3293     # return untainted PID of swtpm daemon so it can be killed on error
3294     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3295     return $1;
3296 }
3297
3298 sub vga_conf_has_spice {
3299     my ($vga) = @_;
3300
3301     my $vgaconf = parse_vga($vga);
3302     my $vgatype = $vgaconf->{type};
3303     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3304
3305     return $1 || 1;
3306 }
3307
3308 sub is_native($) {
3309     my ($arch) = @_;
3310     return get_host_arch() eq $arch;
3311 }
3312
3313 sub get_vm_arch {
3314     my ($conf) = @_;
3315     return $conf->{arch} // get_host_arch();
3316 }
3317
3318 my $default_machines = {
3319     x86_64 => 'pc',
3320     aarch64 => 'virt',
3321 };
3322
3323 sub get_installed_machine_version {
3324     my ($kvmversion) = @_;
3325     $kvmversion = kvm_user_version() if !defined($kvmversion);
3326     $kvmversion =~ m/^(\d+\.\d+)/;
3327     return $1;
3328 }
3329
3330 sub windows_get_pinned_machine_version {
3331     my ($machine, $base_version, $kvmversion) = @_;
3332
3333     my $pin_version = $base_version;
3334     if (!defined($base_version) ||
3335         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3336     ) {
3337         $pin_version = get_installed_machine_version($kvmversion);
3338     }
3339     if (!$machine || $machine eq 'pc') {
3340         $machine = "pc-i440fx-$pin_version";
3341     } elsif ($machine eq 'q35') {
3342         $machine = "pc-q35-$pin_version";
3343     } elsif ($machine eq 'virt') {
3344         $machine = "virt-$pin_version";
3345     } else {
3346         warn "unknown machine type '$machine', not touching that!\n";
3347     }
3348
3349     return $machine;
3350 }
3351
3352 sub get_vm_machine {
3353     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3354
3355     my $machine = $forcemachine || $conf->{machine};
3356
3357     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3358         $kvmversion //= kvm_user_version();
3359         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3360         # layout which confuses windows quite a bit and may result in various regressions..
3361         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3362         if (windows_version($conf->{ostype})) {
3363             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3364         }
3365         $arch //= 'x86_64';
3366         $machine ||= $default_machines->{$arch};
3367         if ($add_pve_version) {
3368             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3369             $machine .= "+pve$pvever";
3370         }
3371     }
3372
3373     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3374         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3375         $machine = $1 if $is_pxe;
3376
3377         # for version-pinned machines that do not include a pve-version (e.g.
3378         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3379         $machine .= '+pve0';
3380
3381         $machine .= '.pxe' if $is_pxe;
3382     }
3383
3384     return $machine;
3385 }
3386
3387 sub get_ovmf_files($$$) {
3388     my ($arch, $efidisk, $smm) = @_;
3389
3390     my $types = $OVMF->{$arch}
3391         or die "no OVMF images known for architecture '$arch'\n";
3392
3393     my $type = 'default';
3394     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3395         $type = $smm ? "4m" : "4m-no-smm";
3396         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3397     }
3398
3399     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3400     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3401     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3402
3403     return ($ovmf_code, $ovmf_vars);
3404 }
3405
3406 my $Arch2Qemu = {
3407     aarch64 => '/usr/bin/qemu-system-aarch64',
3408     x86_64 => '/usr/bin/qemu-system-x86_64',
3409 };
3410 sub get_command_for_arch($) {
3411     my ($arch) = @_;
3412     return '/usr/bin/kvm' if is_native($arch);
3413
3414     my $cmd = $Arch2Qemu->{$arch}
3415         or die "don't know how to emulate architecture '$arch'\n";
3416     return $cmd;
3417 }
3418
3419 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3420 # to use in a QEMU command line (-cpu element), first array_intersect the result
3421 # of query_supported_ with query_understood_. This is necessary because:
3422 #
3423 # a) query_understood_ returns flags the host cannot use and
3424 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3425 #    flags, but CPU settings - with most of them being flags. Those settings
3426 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3427 #
3428 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3429 # expensive. If you need the value returned from this, you can get it much
3430 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3431 # $accel being 'kvm' or 'tcg'.
3432 #
3433 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3434 # changes, automatically populating pmxcfs.
3435 #
3436 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3437 # since kvm and tcg machines support different flags
3438 #
3439 sub query_supported_cpu_flags {
3440     my ($arch) = @_;
3441
3442     $arch //= get_host_arch();
3443     my $default_machine = $default_machines->{$arch};
3444
3445     my $flags = {};
3446
3447     # FIXME: Once this is merged, the code below should work for ARM as well:
3448     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3449     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3450         $arch eq "aarch64";
3451
3452     my $kvm_supported = defined(kvm_version());
3453     my $qemu_cmd = get_command_for_arch($arch);
3454     my $fakevmid = -1;
3455     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3456
3457     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3458     my $query_supported_run_qemu = sub {
3459         my ($kvm) = @_;
3460
3461         my $flags = {};
3462         my $cmd = [
3463             $qemu_cmd,
3464             '-machine', $default_machine,
3465             '-display', 'none',
3466             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3467             '-mon', 'chardev=qmp,mode=control',
3468             '-pidfile', $pidfile,
3469             '-S', '-daemonize'
3470         ];
3471
3472         if (!$kvm) {
3473             push @$cmd, '-accel', 'tcg';
3474         }
3475
3476         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3477         die "QEMU flag querying VM exited with code " . $rc if $rc;
3478
3479         eval {
3480             my $cmd_result = mon_cmd(
3481                 $fakevmid,
3482                 'query-cpu-model-expansion',
3483                 type => 'full',
3484                 model => { name => 'host' }
3485             );
3486
3487             my $props = $cmd_result->{model}->{props};
3488             foreach my $prop (keys %$props) {
3489                 next if $props->{$prop} ne '1';
3490                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3491                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3492                 # We only keep those with underscores, to match /proc/cpuinfo
3493                 $prop =~ s/\.|-/_/g;
3494                 $flags->{$prop} = 1;
3495             }
3496         };
3497         my $err = $@;
3498
3499         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3500         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3501
3502         die $err if $err;
3503
3504         return [ sort keys %$flags ];
3505     };
3506
3507     # We need to query QEMU twice, since KVM and TCG have different supported flags
3508     PVE::QemuConfig->lock_config($fakevmid, sub {
3509         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3510         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3511
3512         if ($kvm_supported) {
3513             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3514             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3515         }
3516     });
3517
3518     return $flags;
3519 }
3520
3521 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3522 my $understood_cpu_flag_dir = "/usr/share/kvm";
3523 sub query_understood_cpu_flags {
3524     my $arch = get_host_arch();
3525     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3526
3527     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3528         if ! -e $filepath;
3529
3530     my $raw = file_get_contents($filepath);
3531     $raw =~ s/^\s+|\s+$//g;
3532     my @flags = split(/\s+/, $raw);
3533
3534     return \@flags;
3535 }
3536
3537 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3538 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3539 my sub should_disable_smm {
3540     my ($conf, $vga, $machine) = @_;
3541
3542     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3543
3544     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3545         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3546 }
3547
3548 my sub print_ovmf_drive_commandlines {
3549     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3550
3551     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3552
3553     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3554
3555     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3556     if ($d) {
3557         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3558         my ($path, $format) = $d->@{'file', 'format'};
3559         if ($storeid) {
3560             $path = PVE::Storage::path($storecfg, $d->{file});
3561             if (!defined($format)) {
3562                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3563                 $format = qemu_img_format($scfg, $volname);
3564             }
3565         } elsif (!defined($format)) {
3566             die "efidisk format must be specified\n";
3567         }
3568         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3569         if ($path =~ m/^rbd:/) {
3570             $var_drive_str .= ',cache=writeback';
3571             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3572         }
3573         $var_drive_str .= ",format=$format,file=$path";
3574
3575         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3576         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3577     } else {
3578         log_warn("no efidisk configured! Using temporary efivars disk.");
3579         my $path = "/tmp/$vmid-ovmf.fd";
3580         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3581         $var_drive_str .= ",format=raw,file=$path";
3582         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3583     }
3584
3585     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3586 }
3587
3588 sub config_to_command {
3589     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3590         $pbs_backing) = @_;
3591
3592     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3593     my $devices = [];
3594     my $bridges = {};
3595     my $ostype = $conf->{ostype};
3596     my $winversion = windows_version($ostype);
3597     my $kvm = $conf->{kvm};
3598     my $nodename = nodename();
3599
3600     my $arch = get_vm_arch($conf);
3601     my $kvm_binary = get_command_for_arch($arch);
3602     my $kvmver = kvm_user_version($kvm_binary);
3603
3604     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3605         $kvmver //= "undefined";
3606         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3607     }
3608
3609     my $add_pve_version = min_version($kvmver, 4, 1);
3610
3611     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3612     my $machine_version = extract_version($machine_type, $kvmver);
3613     $kvm //= 1 if is_native($arch);
3614
3615     $machine_version =~ m/(\d+)\.(\d+)/;
3616     my ($machine_major, $machine_minor) = ($1, $2);
3617
3618     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3619         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3620     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3621         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3622             ." please upgrade node '$nodename'\n"
3623     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3624         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3625         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3626             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3627             ." node '$nodename'\n";
3628     }
3629
3630     # if a specific +pve version is required for a feature, use $version_guard
3631     # instead of min_version to allow machines to be run with the minimum
3632     # required version
3633     my $required_pve_version = 0;
3634     my $version_guard = sub {
3635         my ($major, $minor, $pve) = @_;
3636         return 0 if !min_version($machine_version, $major, $minor, $pve);
3637         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3638         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3639         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3640         return 1;
3641     };
3642
3643     if ($kvm && !defined kvm_version()) {
3644         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3645             ." or enable in BIOS.\n";
3646     }
3647
3648     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3649     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3650     my $use_old_bios_files = undef;
3651     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3652
3653     my $cmd = [];
3654     if ($conf->{affinity}) {
3655         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3656     }
3657
3658     push @$cmd, $kvm_binary;
3659
3660     push @$cmd, '-id', $vmid;
3661
3662     my $vmname = $conf->{name} || "vm$vmid";
3663
3664     push @$cmd, '-name', "$vmname,debug-threads=on";
3665
3666     push @$cmd, '-no-shutdown';
3667
3668     my $use_virtio = 0;
3669
3670     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3671     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3672     push @$cmd, '-mon', "chardev=qmp,mode=control";
3673
3674     if (min_version($machine_version, 2, 12)) {
3675         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3676         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3677     }
3678
3679     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3680
3681     push @$cmd, '-daemonize';
3682
3683     if ($conf->{smbios1}) {
3684         my $smbios_conf = parse_smbios1($conf->{smbios1});
3685         if ($smbios_conf->{base64}) {
3686             # Do not pass base64 flag to qemu
3687             delete $smbios_conf->{base64};
3688             my $smbios_string = "";
3689             foreach my $key (keys %$smbios_conf) {
3690                 my $value;
3691                 if ($key eq "uuid") {
3692                     $value = $smbios_conf->{uuid}
3693                 } else {
3694                     $value = decode_base64($smbios_conf->{$key});
3695                 }
3696                 # qemu accepts any binary data, only commas need escaping by double comma
3697                 $value =~ s/,/,,/g;
3698                 $smbios_string .= "," . $key . "=" . $value if $value;
3699             }
3700             push @$cmd, '-smbios', "type=1" . $smbios_string;
3701         } else {
3702             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3703         }
3704     }
3705
3706     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3707         my ($code_drive_str, $var_drive_str) =
3708             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3709         push $cmd->@*, '-drive', $code_drive_str;
3710         push $cmd->@*, '-drive', $var_drive_str;
3711     }
3712
3713     if ($q35) { # tell QEMU to load q35 config early
3714         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3715         if (min_version($machine_version, 4, 0)) {
3716             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3717         } else {
3718             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3719         }
3720     }
3721
3722     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3723         push @$cmd, $fixups->@*;
3724     }
3725
3726     if ($conf->{vmgenid}) {
3727         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3728     }
3729
3730     # add usb controllers
3731     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3732         $conf, $bridges, $arch, $machine_type, $machine_version);
3733     push @$devices, @usbcontrollers if @usbcontrollers;
3734     my $vga = parse_vga($conf->{vga});
3735
3736     my $qxlnum = vga_conf_has_spice($conf->{vga});
3737     $vga->{type} = 'qxl' if $qxlnum;
3738
3739     if (!$vga->{type}) {
3740         if ($arch eq 'aarch64') {
3741             $vga->{type} = 'virtio';
3742         } elsif (min_version($machine_version, 2, 9)) {
3743             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3744         } else {
3745             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3746         }
3747     }
3748
3749     # enable absolute mouse coordinates (needed by vnc)
3750     my $tablet = $conf->{tablet};
3751     if (!defined($tablet)) {
3752         $tablet = $defaults->{tablet};
3753         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3754         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3755     }
3756
3757     if ($tablet) {
3758         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3759         my $kbd = print_keyboarddevice_full($conf, $arch);
3760         push @$devices, '-device', $kbd if defined($kbd);
3761     }
3762
3763     my $bootorder = device_bootorder($conf);
3764
3765     # host pci device passthrough
3766     my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3767         $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3768
3769     # usb devices
3770     my $usb_dev_features = {};
3771     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3772
3773     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3774         $conf, $usb_dev_features, $bootorder, $machine_version);
3775     push @$devices, @usbdevices if @usbdevices;
3776
3777     # serial devices
3778     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3779         my $path = $conf->{"serial$i"} or next;
3780         if ($path eq 'socket') {
3781             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3782             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3783             # On aarch64, serial0 is the UART device. QEMU only allows
3784             # connecting UART devices via the '-serial' command line, as
3785             # the device has a fixed slot on the hardware...
3786             if ($arch eq 'aarch64' && $i == 0) {
3787                 push @$devices, '-serial', "chardev:serial$i";
3788             } else {
3789                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3790             }
3791         } else {
3792             die "no such serial device\n" if ! -c $path;
3793             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3794             push @$devices, '-device', "isa-serial,chardev=serial$i";
3795         }
3796     }
3797
3798     # parallel devices
3799     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3800         if (my $path = $conf->{"parallel$i"}) {
3801             die "no such parallel device\n" if ! -c $path;
3802             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3803             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3804             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3805         }
3806     }
3807
3808     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3809         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3810         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3811         push @$devices, @$audio_devs;
3812     }
3813
3814     add_tpm_device($vmid, $devices, $conf);
3815
3816     my $sockets = 1;
3817     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3818     $sockets = $conf->{sockets} if  $conf->{sockets};
3819
3820     my $cores = $conf->{cores} || 1;
3821
3822     my $maxcpus = $sockets * $cores;
3823
3824     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3825
3826     my $allowed_vcpus = $cpuinfo->{cpus};
3827
3828     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3829
3830     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3831         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3832         for (my $i = 2; $i <= $vcpus; $i++)  {
3833             my $cpustr = print_cpu_device($conf,$i);
3834             push @$cmd, '-device', $cpustr;
3835         }
3836
3837     } else {
3838
3839         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3840     }
3841     push @$cmd, '-nodefaults';
3842
3843     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3844
3845     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3846
3847     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3848
3849     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3850         push @$devices, '-device', print_vga_device(
3851             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3852
3853         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3854
3855         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3856         push @$cmd,  '-vnc', "unix:$socket,password=on";
3857     } else {
3858         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3859         push @$cmd, '-nographic';
3860     }
3861
3862     # time drift fix
3863     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3864     my $useLocaltime = $conf->{localtime};
3865
3866     if ($winversion >= 5) { # windows
3867         $useLocaltime = 1 if !defined($conf->{localtime});
3868
3869         # use time drift fix when acpi is enabled
3870         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3871             $tdf = 1 if !defined($conf->{tdf});
3872         }
3873     }
3874
3875     if ($winversion >= 6) {
3876         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3877         push @$machineFlags, 'hpet=off';
3878     }
3879
3880     push @$rtcFlags, 'driftfix=slew' if $tdf;
3881
3882     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3883         push @$rtcFlags, "base=$conf->{startdate}";
3884     } elsif ($useLocaltime) {
3885         push @$rtcFlags, 'base=localtime';
3886     }
3887
3888     if ($forcecpu) {
3889         push @$cmd, '-cpu', $forcecpu;
3890     } else {
3891         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3892     }
3893
3894     PVE::QemuServer::Memory::config(
3895         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3896
3897     push @$cmd, '-S' if $conf->{freeze};
3898
3899     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3900
3901     my $guest_agent = parse_guest_agent($conf);
3902
3903     if ($guest_agent->{enabled}) {
3904         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3905         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3906
3907         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3908             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3909             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3910             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3911         } elsif ($guest_agent->{type} eq 'isa') {
3912             push @$devices, '-device', "isa-serial,chardev=qga0";
3913         }
3914     }
3915
3916     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3917     if ($rng && $version_guard->(4, 1, 2)) {
3918         check_rng_source($rng->{source});
3919
3920         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3921         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3922         my $limiter_str = "";
3923         if ($max_bytes) {
3924             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3925         }
3926
3927         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3928         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3929         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3930     }
3931
3932     my $spice_port;
3933
3934     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3935         if ($qxlnum > 1) {
3936             if ($winversion){
3937                 for (my $i = 1; $i < $qxlnum; $i++){
3938                     push @$devices, '-device', print_vga_device(
3939                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3940                 }
3941             } else {
3942                 # assume other OS works like Linux
3943                 my ($ram, $vram) = ("134217728", "67108864");
3944                 if ($vga->{memory}) {
3945                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3946                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3947                 }
3948                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3949                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3950             }
3951         }
3952
3953         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3954
3955         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3956         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3957         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3958
3959         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3960         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3961         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3962
3963         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3964         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3965
3966         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3967         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3968         if ($spice_enhancement->{foldersharing}) {
3969             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3970             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3971         }
3972
3973         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3974         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3975             if $spice_enhancement->{videostreaming};
3976
3977         push @$devices, '-spice', "$spice_opts";
3978     }
3979
3980     # enable balloon by default, unless explicitly disabled
3981     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3982         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3983         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3984         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3985         push @$devices, '-device', $ballooncmd;
3986     }
3987
3988     if ($conf->{watchdog}) {
3989         my $wdopts = parse_watchdog($conf->{watchdog});
3990         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3991         my $watchdog = $wdopts->{model} || 'i6300esb';
3992         push @$devices, '-device', "$watchdog$pciaddr";
3993         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3994     }
3995
3996     my $vollist = [];
3997     my $scsicontroller = {};
3998     my $ahcicontroller = {};
3999     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4000
4001     # Add iscsi initiator name if available
4002     if (my $initiator = get_initiator_name()) {
4003         push @$devices, '-iscsi', "initiator-name=$initiator";
4004     }
4005
4006     PVE::QemuConfig->foreach_volume($conf, sub {
4007         my ($ds, $drive) = @_;
4008
4009         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4010             check_volume_storage_type($storecfg, $drive->{file});
4011             push @$vollist, $drive->{file};
4012         }
4013
4014         # ignore efidisk here, already added in bios/fw handling code above
4015         return if $drive->{interface} eq 'efidisk';
4016         # similar for TPM
4017         return if $drive->{interface} eq 'tpmstate';
4018
4019         $use_virtio = 1 if $ds =~ m/^virtio/;
4020
4021         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4022
4023         if ($drive->{interface} eq 'virtio'){
4024            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4025         }
4026
4027         if ($drive->{interface} eq 'scsi') {
4028
4029             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4030
4031             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4032                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4033
4034             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4035             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4036
4037             my $iothread = '';
4038             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4039                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4040                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4041             } elsif ($drive->{iothread}) {
4042                 log_warn(
4043                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4044                 );
4045             }
4046
4047             my $queues = '';
4048             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4049                 $queues = ",num_queues=$drive->{queues}";
4050             }
4051
4052             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4053                 if !$scsicontroller->{$controller};
4054             $scsicontroller->{$controller}=1;
4055         }
4056
4057         if ($drive->{interface} eq 'sata') {
4058             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4059             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4060             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4061                 if !$ahcicontroller->{$controller};
4062             $ahcicontroller->{$controller}=1;
4063         }
4064
4065         my $pbs_conf = $pbs_backing->{$ds};
4066         my $pbs_name = undef;
4067         if ($pbs_conf) {
4068             $pbs_name = "drive-$ds-pbs";
4069             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4070         }
4071
4072         my $drive_cmd = print_drive_commandline_full(
4073             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4074
4075         # extra protection for templates, but SATA and IDE don't support it..
4076         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4077
4078         push @$devices, '-drive',$drive_cmd;
4079         push @$devices, '-device', print_drivedevice_full(
4080             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4081     });
4082
4083     for (my $i = 0; $i < $MAX_NETS; $i++) {
4084         my $netname = "net$i";
4085
4086         next if !$conf->{$netname};
4087         my $d = parse_net($conf->{$netname});
4088         next if !$d;
4089         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4090
4091         $use_virtio = 1 if $d->{model} eq 'virtio';
4092
4093         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4094
4095         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4096         push @$devices, '-netdev', $netdevfull;
4097
4098         my $netdevicefull = print_netdevice_full(
4099             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4100
4101         push @$devices, '-device', $netdevicefull;
4102     }
4103
4104     if ($conf->{ivshmem}) {
4105         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4106
4107         my $bus;
4108         if ($q35) {
4109             $bus = print_pcie_addr("ivshmem");
4110         } else {
4111             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4112         }
4113
4114         my $ivshmem_name = $ivshmem->{name} // $vmid;
4115         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4116
4117         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4118         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4119             .",size=$ivshmem->{size}M";
4120     }
4121
4122     # pci.4 is nested in pci.1
4123     $bridges->{1} = 1 if $bridges->{4};
4124
4125     if (!$q35) { # add pci bridges
4126         if (min_version($machine_version, 2, 3)) {
4127            $bridges->{1} = 1;
4128            $bridges->{2} = 1;
4129         }
4130         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4131     }
4132
4133     for my $k (sort {$b cmp $a} keys %$bridges) {
4134         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4135
4136         my $k_name = $k;
4137         if ($k == 2 && $legacy_igd) {
4138             $k_name = "$k-igd";
4139         }
4140         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4141         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4142
4143         if ($q35) { # add after -readconfig pve-q35.cfg
4144             splice @$devices, 2, 0, '-device', $devstr;
4145         } else {
4146             unshift @$devices, '-device', $devstr if $k > 0;
4147         }
4148     }
4149
4150     if (!$kvm) {
4151         push @$machineFlags, 'accel=tcg';
4152     }
4153
4154     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4155
4156     my $machine_type_min = $machine_type;
4157     if ($add_pve_version) {
4158         $machine_type_min =~ s/\+pve\d+$//;
4159         $machine_type_min .= "+pve$required_pve_version";
4160     }
4161     push @$machineFlags, "type=${machine_type_min}";
4162
4163     push @$cmd, @$devices;
4164     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4165     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4166     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4167
4168     if (my $vmstate = $conf->{vmstate}) {
4169         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4170         push @$vollist, $vmstate;
4171         push @$cmd, '-loadstate', $statepath;
4172         print "activating and using '$vmstate' as vmstate\n";
4173     }
4174
4175     if (PVE::QemuConfig->is_template($conf)) {
4176         # needed to workaround base volumes being read-only
4177         push @$cmd, '-snapshot';
4178     }
4179
4180     # add custom args
4181     if ($conf->{args}) {
4182         my $aa = PVE::Tools::split_args($conf->{args});
4183         push @$cmd, @$aa;
4184     }
4185
4186     return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4187 }
4188
4189 sub check_rng_source {
4190     my ($source) = @_;
4191
4192     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4193     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4194         if ! -e $source;
4195
4196     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4197     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4198         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4199         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4200         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4201             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4202             ." to the host.\n";
4203     }
4204 }
4205
4206 sub spice_port {
4207     my ($vmid) = @_;
4208
4209     my $res = mon_cmd($vmid, 'query-spice');
4210
4211     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4212 }
4213
4214 sub vm_devices_list {
4215     my ($vmid) = @_;
4216
4217     my $res = mon_cmd($vmid, 'query-pci');
4218     my $devices_to_check = [];
4219     my $devices = {};
4220     foreach my $pcibus (@$res) {
4221         push @$devices_to_check, @{$pcibus->{devices}},
4222     }
4223
4224     while (@$devices_to_check) {
4225         my $to_check = [];
4226         for my $d (@$devices_to_check) {
4227             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4228             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4229
4230             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4231             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4232         }
4233         $devices_to_check = $to_check;
4234     }
4235
4236     my $resblock = mon_cmd($vmid, 'query-block');
4237     foreach my $block (@$resblock) {
4238         if($block->{device} =~ m/^drive-(\S+)/){
4239                 $devices->{$1} = 1;
4240         }
4241     }
4242
4243     my $resmice = mon_cmd($vmid, 'query-mice');
4244     foreach my $mice (@$resmice) {
4245         if ($mice->{name} eq 'QEMU HID Tablet') {
4246             $devices->{tablet} = 1;
4247             last;
4248         }
4249     }
4250
4251     # for usb devices there is no query-usb
4252     # but we can iterate over the entries in
4253     # qom-list path=/machine/peripheral
4254     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4255     foreach my $per (@$resperipheral) {
4256         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4257             $devices->{$per->{name}} = 1;
4258         }
4259     }
4260
4261     return $devices;
4262 }
4263
4264 sub vm_deviceplug {
4265     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4266
4267     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4268
4269     my $devices_list = vm_devices_list($vmid);
4270     return 1 if defined($devices_list->{$deviceid});
4271
4272     # add PCI bridge if we need it for the device
4273     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4274
4275     if ($deviceid eq 'tablet') {
4276         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4277     } elsif ($deviceid eq 'keyboard') {
4278         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4279     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4280         my $id = $1;
4281         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4282         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4283     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4284         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4285     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4286         qemu_iothread_add($vmid, $deviceid, $device);
4287
4288         qemu_driveadd($storecfg, $vmid, $device);
4289         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4290
4291         qemu_deviceadd($vmid, $devicefull);
4292         eval { qemu_deviceaddverify($vmid, $deviceid); };
4293         if (my $err = $@) {
4294             eval { qemu_drivedel($vmid, $deviceid); };
4295             warn $@ if $@;
4296             die $err;
4297         }
4298     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4299         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4300         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4301         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4302
4303         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4304
4305         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4306             qemu_iothread_add($vmid, $deviceid, $device);
4307             $devicefull .= ",iothread=iothread-$deviceid";
4308         }
4309
4310         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4311             $devicefull .= ",num_queues=$device->{queues}";
4312         }
4313
4314         qemu_deviceadd($vmid, $devicefull);
4315         qemu_deviceaddverify($vmid, $deviceid);
4316     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4317         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4318         qemu_driveadd($storecfg, $vmid, $device);
4319
4320         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4321         eval { qemu_deviceadd($vmid, $devicefull); };
4322         if (my $err = $@) {
4323             eval { qemu_drivedel($vmid, $deviceid); };
4324             warn $@ if $@;
4325             die $err;
4326         }
4327     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4328         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4329
4330         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4331         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4332         my $use_old_bios_files = undef;
4333         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4334
4335         my $netdevicefull = print_netdevice_full(
4336             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4337         qemu_deviceadd($vmid, $netdevicefull);
4338         eval {
4339             qemu_deviceaddverify($vmid, $deviceid);
4340             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4341         };
4342         if (my $err = $@) {
4343             eval { qemu_netdevdel($vmid, $deviceid); };
4344             warn $@ if $@;
4345             die $err;
4346         }
4347     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4348         my $bridgeid = $2;
4349         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4350         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4351
4352         qemu_deviceadd($vmid, $devicefull);
4353         qemu_deviceaddverify($vmid, $deviceid);
4354     } else {
4355         die "can't hotplug device '$deviceid'\n";
4356     }
4357
4358     return 1;
4359 }
4360
4361 # fixme: this should raise exceptions on error!
4362 sub vm_deviceunplug {
4363     my ($vmid, $conf, $deviceid) = @_;
4364
4365     my $devices_list = vm_devices_list($vmid);
4366     return 1 if !defined($devices_list->{$deviceid});
4367
4368     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4369     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4370
4371     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4372         qemu_devicedel($vmid, $deviceid);
4373     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4374         qemu_devicedel($vmid, $deviceid);
4375         qemu_devicedelverify($vmid, $deviceid);
4376     } elsif ($deviceid =~ m/^usb\d+$/) {
4377         qemu_devicedel($vmid, $deviceid);
4378         qemu_devicedelverify($vmid, $deviceid);
4379     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4380         my $device = parse_drive($deviceid, $conf->{$deviceid});
4381
4382         qemu_devicedel($vmid, $deviceid);
4383         qemu_devicedelverify($vmid, $deviceid);
4384         qemu_drivedel($vmid, $deviceid);
4385         qemu_iothread_del($vmid, $deviceid, $device);
4386     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4387         qemu_devicedel($vmid, $deviceid);
4388         qemu_devicedelverify($vmid, $deviceid);
4389     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4390         my $device = parse_drive($deviceid, $conf->{$deviceid});
4391
4392         qemu_devicedel($vmid, $deviceid);
4393         qemu_devicedelverify($vmid, $deviceid);
4394         qemu_drivedel($vmid, $deviceid);
4395         qemu_deletescsihw($conf, $vmid, $deviceid);
4396
4397         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4398             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4399     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4400         qemu_devicedel($vmid, $deviceid);
4401         qemu_devicedelverify($vmid, $deviceid);
4402         qemu_netdevdel($vmid, $deviceid);
4403     } else {
4404         die "can't unplug device '$deviceid'\n";
4405     }
4406
4407     return 1;
4408 }
4409
4410 sub qemu_spice_usbredir_chardev_add {
4411     my ($vmid, $id) = @_;
4412
4413     mon_cmd($vmid, "chardev-add" , (
4414         id => $id,
4415         backend => {
4416             type => 'spicevmc',
4417             data => {
4418                 type => "usbredir",
4419             },
4420         },
4421     ));
4422 }
4423
4424 sub qemu_deviceadd {
4425     my ($vmid, $devicefull) = @_;
4426
4427     $devicefull = "driver=".$devicefull;
4428     my %options =  split(/[=,]/, $devicefull);
4429
4430     mon_cmd($vmid, "device_add" , %options);
4431 }
4432
4433 sub qemu_devicedel {
4434     my ($vmid, $deviceid) = @_;
4435
4436     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4437 }
4438
4439 sub qemu_iothread_add {
4440     my ($vmid, $deviceid, $device) = @_;
4441
4442     if ($device->{iothread}) {
4443         my $iothreads = vm_iothreads_list($vmid);
4444         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4445     }
4446 }
4447
4448 sub qemu_iothread_del {
4449     my ($vmid, $deviceid, $device) = @_;
4450
4451     if ($device->{iothread}) {
4452         my $iothreads = vm_iothreads_list($vmid);
4453         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4454     }
4455 }
4456
4457 sub qemu_objectadd {
4458     my ($vmid, $objectid, $qomtype) = @_;
4459
4460     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4461
4462     return 1;
4463 }
4464
4465 sub qemu_objectdel {
4466     my ($vmid, $objectid) = @_;
4467
4468     mon_cmd($vmid, "object-del", id => $objectid);
4469
4470     return 1;
4471 }
4472
4473 sub qemu_driveadd {
4474     my ($storecfg, $vmid, $device) = @_;
4475
4476     my $kvmver = get_running_qemu_version($vmid);
4477     my $io_uring = min_version($kvmver, 6, 0);
4478     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4479     $drive =~ s/\\/\\\\/g;
4480     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4481
4482     # If the command succeeds qemu prints: "OK"
4483     return 1 if $ret =~ m/OK/s;
4484
4485     die "adding drive failed: $ret\n";
4486 }
4487
4488 sub qemu_drivedel {
4489     my ($vmid, $deviceid) = @_;
4490
4491     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4492     $ret =~ s/^\s+//;
4493
4494     return 1 if $ret eq "";
4495
4496     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4497     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4498
4499     die "deleting drive $deviceid failed : $ret\n";
4500 }
4501
4502 sub qemu_deviceaddverify {
4503     my ($vmid, $deviceid) = @_;
4504
4505     for (my $i = 0; $i <= 5; $i++) {
4506          my $devices_list = vm_devices_list($vmid);
4507          return 1 if defined($devices_list->{$deviceid});
4508          sleep 1;
4509     }
4510
4511     die "error on hotplug device '$deviceid'\n";
4512 }
4513
4514
4515 sub qemu_devicedelverify {
4516     my ($vmid, $deviceid) = @_;
4517
4518     # need to verify that the device is correctly removed as device_del
4519     # is async and empty return is not reliable
4520
4521     for (my $i = 0; $i <= 5; $i++) {
4522          my $devices_list = vm_devices_list($vmid);
4523          return 1 if !defined($devices_list->{$deviceid});
4524          sleep 1;
4525     }
4526
4527     die "error on hot-unplugging device '$deviceid'\n";
4528 }
4529
4530 sub qemu_findorcreatescsihw {
4531     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4532
4533     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4534
4535     my $scsihwid="$controller_prefix$controller";
4536     my $devices_list = vm_devices_list($vmid);
4537
4538     if (!defined($devices_list->{$scsihwid})) {
4539         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4540     }
4541
4542     return 1;
4543 }
4544
4545 sub qemu_deletescsihw {
4546     my ($conf, $vmid, $opt) = @_;
4547
4548     my $device = parse_drive($opt, $conf->{$opt});
4549
4550     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4551         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4552         return 1;
4553     }
4554
4555     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4556
4557     my $devices_list = vm_devices_list($vmid);
4558     foreach my $opt (keys %{$devices_list}) {
4559         if (is_valid_drivename($opt)) {
4560             my $drive = parse_drive($opt, $conf->{$opt});
4561             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4562                 return 1;
4563             }
4564         }
4565     }
4566
4567     my $scsihwid="scsihw$controller";
4568
4569     vm_deviceunplug($vmid, $conf, $scsihwid);
4570
4571     return 1;
4572 }
4573
4574 sub qemu_add_pci_bridge {
4575     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4576
4577     my $bridges = {};
4578
4579     my $bridgeid;
4580
4581     print_pci_addr($device, $bridges, $arch, $machine_type);
4582
4583     while (my ($k, $v) = each %$bridges) {
4584         $bridgeid = $k;
4585     }
4586     return 1 if !defined($bridgeid) || $bridgeid < 1;
4587
4588     my $bridge = "pci.$bridgeid";
4589     my $devices_list = vm_devices_list($vmid);
4590
4591     if (!defined($devices_list->{$bridge})) {
4592         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4593     }
4594
4595     return 1;
4596 }
4597
4598 sub qemu_set_link_status {
4599     my ($vmid, $device, $up) = @_;
4600
4601     mon_cmd($vmid, "set_link", name => $device,
4602                up => $up ? JSON::true : JSON::false);
4603 }
4604
4605 sub qemu_netdevadd {
4606     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4607
4608     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4609     my %options =  split(/[=,]/, $netdev);
4610
4611     if (defined(my $vhost = $options{vhost})) {
4612         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4613     }
4614
4615     if (defined(my $queues = $options{queues})) {
4616         $options{queues} = $queues + 0;
4617     }
4618
4619     mon_cmd($vmid, "netdev_add",  %options);
4620     return 1;
4621 }
4622
4623 sub qemu_netdevdel {
4624     my ($vmid, $deviceid) = @_;
4625
4626     mon_cmd($vmid, "netdev_del", id => $deviceid);
4627 }
4628
4629 sub qemu_usb_hotplug {
4630     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4631
4632     return if !$device;
4633
4634     # remove the old one first
4635     vm_deviceunplug($vmid, $conf, $deviceid);
4636
4637     # check if xhci controller is necessary and available
4638     my $devicelist = vm_devices_list($vmid);
4639
4640     if (!$devicelist->{xhci}) {
4641         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4642         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4643     }
4644
4645     # add the new one
4646     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4647 }
4648
4649 sub qemu_cpu_hotplug {
4650     my ($vmid, $conf, $vcpus) = @_;
4651
4652     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4653
4654     my $sockets = 1;
4655     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4656     $sockets = $conf->{sockets} if  $conf->{sockets};
4657     my $cores = $conf->{cores} || 1;
4658     my $maxcpus = $sockets * $cores;
4659
4660     $vcpus = $maxcpus if !$vcpus;
4661
4662     die "you can't add more vcpus than maxcpus\n"
4663         if $vcpus > $maxcpus;
4664
4665     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4666
4667     if ($vcpus < $currentvcpus) {
4668
4669         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4670
4671             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4672                 qemu_devicedel($vmid, "cpu$i");
4673                 my $retry = 0;
4674                 my $currentrunningvcpus = undef;
4675                 while (1) {
4676                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4677                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4678                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4679                     $retry++;
4680                     sleep 1;
4681                 }
4682                 #update conf after each succesfull cpu unplug
4683                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4684                 PVE::QemuConfig->write_config($vmid, $conf);
4685             }
4686         } else {
4687             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4688         }
4689
4690         return;
4691     }
4692
4693     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4694     die "vcpus in running vm does not match its configuration\n"
4695         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4696
4697     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4698
4699         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4700             my $cpustr = print_cpu_device($conf, $i);
4701             qemu_deviceadd($vmid, $cpustr);
4702
4703             my $retry = 0;
4704             my $currentrunningvcpus = undef;
4705             while (1) {
4706                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4707                 last if scalar(@{$currentrunningvcpus}) == $i;
4708                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4709                 sleep 1;
4710                 $retry++;
4711             }
4712             #update conf after each succesfull cpu hotplug
4713             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4714             PVE::QemuConfig->write_config($vmid, $conf);
4715         }
4716     } else {
4717
4718         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4719             mon_cmd($vmid, "cpu-add", id => int($i));
4720         }
4721     }
4722 }
4723
4724 sub qemu_block_set_io_throttle {
4725     my ($vmid, $deviceid,
4726         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4727         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4728         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4729         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4730
4731     return if !check_running($vmid) ;
4732
4733     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4734         bps => int($bps),
4735         bps_rd => int($bps_rd),
4736         bps_wr => int($bps_wr),
4737         iops => int($iops),
4738         iops_rd => int($iops_rd),
4739         iops_wr => int($iops_wr),
4740         bps_max => int($bps_max),
4741         bps_rd_max => int($bps_rd_max),
4742         bps_wr_max => int($bps_wr_max),
4743         iops_max => int($iops_max),
4744         iops_rd_max => int($iops_rd_max),
4745         iops_wr_max => int($iops_wr_max),
4746         bps_max_length => int($bps_max_length),
4747         bps_rd_max_length => int($bps_rd_max_length),
4748         bps_wr_max_length => int($bps_wr_max_length),
4749         iops_max_length => int($iops_max_length),
4750         iops_rd_max_length => int($iops_rd_max_length),
4751         iops_wr_max_length => int($iops_wr_max_length),
4752     );
4753
4754 }
4755
4756 sub qemu_block_resize {
4757     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4758
4759     my $running = check_running($vmid);
4760
4761     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4762
4763     return if !$running;
4764
4765     my $padding = (1024 - $size % 1024) % 1024;
4766     $size = $size + $padding;
4767
4768     mon_cmd(
4769         $vmid,
4770         "block_resize",
4771         device => $deviceid,
4772         size => int($size),
4773         timeout => 60,
4774     );
4775 }
4776
4777 sub qemu_volume_snapshot {
4778     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4779
4780     my $running = check_running($vmid);
4781
4782     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4783         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4784     } else {
4785         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4786     }
4787 }
4788
4789 sub qemu_volume_snapshot_delete {
4790     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4791
4792     my $running = check_running($vmid);
4793
4794     if($running) {
4795
4796         $running = undef;
4797         my $conf = PVE::QemuConfig->load_config($vmid);
4798         PVE::QemuConfig->foreach_volume($conf, sub {
4799             my ($ds, $drive) = @_;
4800             $running = 1 if $drive->{file} eq $volid;
4801         });
4802     }
4803
4804     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4805         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4806     } else {
4807         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4808     }
4809 }
4810
4811 sub set_migration_caps {
4812     my ($vmid, $savevm) = @_;
4813
4814     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4815
4816     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4817     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4818
4819     my $cap_ref = [];
4820
4821     my $enabled_cap = {
4822         "auto-converge" => 1,
4823         "xbzrle" => 1,
4824         "x-rdma-pin-all" => 0,
4825         "zero-blocks" => 0,
4826         "compress" => 0,
4827         "dirty-bitmaps" => $dirty_bitmaps,
4828     };
4829
4830     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4831
4832     for my $supported_capability (@$supported_capabilities) {
4833         push @$cap_ref, {
4834             capability => $supported_capability->{capability},
4835             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4836         };
4837     }
4838
4839     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4840 }
4841
4842 sub foreach_volid {
4843     my ($conf, $func, @param) = @_;
4844
4845     my $volhash = {};
4846
4847     my $test_volid = sub {
4848         my ($key, $drive, $snapname, $pending) = @_;
4849
4850         my $volid = $drive->{file};
4851         return if !$volid;
4852
4853         $volhash->{$volid}->{cdrom} //= 1;
4854         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4855
4856         my $replicate = $drive->{replicate} // 1;
4857         $volhash->{$volid}->{replicate} //= 0;
4858         $volhash->{$volid}->{replicate} = 1 if $replicate;
4859
4860         $volhash->{$volid}->{shared} //= 0;
4861         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4862
4863         $volhash->{$volid}->{referenced_in_config} //= 0;
4864         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname) && !defined($pending);
4865
4866         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4867             if defined($snapname);
4868
4869         $volhash->{$volid}->{referenced_in_pending} = 1 if defined($pending);
4870
4871         my $size = $drive->{size};
4872         $volhash->{$volid}->{size} //= $size if $size;
4873
4874         $volhash->{$volid}->{is_vmstate} //= 0;
4875         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4876
4877         $volhash->{$volid}->{is_tpmstate} //= 0;
4878         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4879
4880         $volhash->{$volid}->{is_unused} //= 0;
4881         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4882
4883         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4884     };
4885
4886     my $include_opts = {
4887         extra_keys => ['vmstate'],
4888         include_unused => 1,
4889     };
4890
4891     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4892
4893     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4894         if defined($conf->{pending}) && $conf->{pending}->%*;
4895
4896     foreach my $snapname (keys %{$conf->{snapshots}}) {
4897         my $snap = $conf->{snapshots}->{$snapname};
4898         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4899     }
4900
4901     foreach my $volid (keys %$volhash) {
4902         &$func($volid, $volhash->{$volid}, @param);
4903     }
4904 }
4905
4906 my $fast_plug_option = {
4907     'description' => 1,
4908     'hookscript' => 1,
4909     'lock' => 1,
4910     'migrate_downtime' => 1,
4911     'migrate_speed' => 1,
4912     'name' => 1,
4913     'onboot' => 1,
4914     'protection' => 1,
4915     'shares' => 1,
4916     'startup' => 1,
4917     'tags' => 1,
4918     'vmstatestorage' => 1,
4919 };
4920
4921 for my $opt (keys %$confdesc_cloudinit) {
4922     $fast_plug_option->{$opt} = 1;
4923 };
4924
4925 # hotplug changes in [PENDING]
4926 # $selection hash can be used to only apply specified options, for
4927 # example: { cores => 1 } (only apply changed 'cores')
4928 # $errors ref is used to return error messages
4929 sub vmconfig_hotplug_pending {
4930     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4931
4932     my $defaults = load_defaults();
4933     my $arch = get_vm_arch($conf);
4934     my $machine_type = get_vm_machine($conf, undef, $arch);
4935
4936     # commit values which do not have any impact on running VM first
4937     # Note: those option cannot raise errors, we we do not care about
4938     # $selection and always apply them.
4939
4940     my $add_error = sub {
4941         my ($opt, $msg) = @_;
4942         $errors->{$opt} = "hotplug problem - $msg";
4943     };
4944
4945     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4946
4947     my $cloudinit_record_changed = sub {
4948         my ($conf, $opt, $old, $new) = @_;
4949         return if !$cloudinit_pending_properties->{$opt};
4950
4951         my $ci = ($conf->{cloudinit} //= {});
4952
4953         my $recorded = $ci->{$opt};
4954         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4955
4956         if (defined($new)) {
4957             if (defined($old)) {
4958                 # an existing value is being modified
4959                 if (defined($recorded)) {
4960                     # the value was already not in sync
4961                     if ($new eq $recorded) {
4962                         # a value is being reverted to the cloud-init state:
4963                         delete $ci->{$opt};
4964                         delete $added{$opt};
4965                     } else {
4966                         # the value was changed multiple times, do nothing
4967                     }
4968                 } elsif ($added{$opt}) {
4969                     # the value had been marked as added and is being changed, do nothing
4970                 } else {
4971                     # the value is new, record it:
4972                     $ci->{$opt} = $old;
4973                 }
4974             } else {
4975                 # a new value is being added
4976                 if (defined($recorded)) {
4977                     # it was already not in sync
4978                     if ($new eq $recorded) {
4979                         # a value is being reverted to the cloud-init state:
4980                         delete $ci->{$opt};
4981                         delete $added{$opt};
4982                     } else {
4983                         # the value had temporarily been removed, do nothing
4984                     }
4985                 } elsif ($added{$opt}) {
4986                     # the value had been marked as added already, do nothing
4987                 } else {
4988                     # the value is new, add it
4989                     $added{$opt} = 1;
4990                 }
4991             }
4992         } elsif (!defined($old)) {
4993             # a non-existent value is being removed? ignore...
4994         } else {
4995             # a value is being deleted
4996             if (defined($recorded)) {
4997                 # a value was already recorded, just keep it
4998             } elsif ($added{$opt}) {
4999                 # the value was marked as added, remove it
5000                 delete $added{$opt};
5001             } else {
5002                 # a previously unrecorded value is being removed, record the old value:
5003                 $ci->{$opt} = $old;
5004             }
5005         }
5006
5007         my $added = join(',', sort keys %added);
5008         $ci->{added} = $added if length($added);
5009     };
5010
5011     my $changes = 0;
5012     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5013         if ($fast_plug_option->{$opt}) {
5014             my $new = delete $conf->{pending}->{$opt};
5015             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5016             $conf->{$opt} = $new;
5017             $changes = 1;
5018         }
5019     }
5020
5021     if ($changes) {
5022         PVE::QemuConfig->write_config($vmid, $conf);
5023     }
5024
5025     my $ostype = $conf->{ostype};
5026     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5027     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5028     my $usb_hotplug = $hotplug_features->{usb}
5029         && min_version($version, 7, 1)
5030         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5031
5032     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5033     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5034
5035     foreach my $opt (sort keys %$pending_delete_hash) {
5036         next if $selection && !$selection->{$opt};
5037         my $force = $pending_delete_hash->{$opt}->{force};
5038         eval {
5039             if ($opt eq 'hotplug') {
5040                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5041             } elsif ($opt eq 'tablet') {
5042                 die "skip\n" if !$hotplug_features->{usb};
5043                 if ($defaults->{tablet}) {
5044                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5045                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5046                         if $arch eq 'aarch64';
5047                 } else {
5048                     vm_deviceunplug($vmid, $conf, 'tablet');
5049                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5050                 }
5051             } elsif ($opt =~ m/^usb(\d+)$/) {
5052                 my $index = $1;
5053                 die "skip\n" if !$usb_hotplug;
5054                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5055                 vm_deviceunplug($vmid, $conf, $opt);
5056             } elsif ($opt eq 'vcpus') {
5057                 die "skip\n" if !$hotplug_features->{cpu};
5058                 qemu_cpu_hotplug($vmid, $conf, undef);
5059             } elsif ($opt eq 'balloon') {
5060                 # enable balloon device is not hotpluggable
5061                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5062                 # here we reset the ballooning value to memory
5063                 my $balloon = $conf->{memory} || $defaults->{memory};
5064                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5065             } elsif ($fast_plug_option->{$opt}) {
5066                 # do nothing
5067             } elsif ($opt =~ m/^net(\d+)$/) {
5068                 die "skip\n" if !$hotplug_features->{network};
5069                 vm_deviceunplug($vmid, $conf, $opt);
5070             } elsif (is_valid_drivename($opt)) {
5071                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5072                 vm_deviceunplug($vmid, $conf, $opt);
5073                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5074             } elsif ($opt =~ m/^memory$/) {
5075                 die "skip\n" if !$hotplug_features->{memory};
5076                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5077             } elsif ($opt eq 'cpuunits') {
5078                 $cgroup->change_cpu_shares(undef);
5079             } elsif ($opt eq 'cpulimit') {
5080                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5081             } else {
5082                 die "skip\n";
5083             }
5084         };
5085         if (my $err = $@) {
5086             &$add_error($opt, $err) if $err ne "skip\n";
5087         } else {
5088             my $old = delete $conf->{$opt};
5089             $cloudinit_record_changed->($conf, $opt, $old, undef);
5090             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5091         }
5092     }
5093
5094     my $cloudinit_opt;
5095     foreach my $opt (keys %{$conf->{pending}}) {
5096         next if $selection && !$selection->{$opt};
5097         my $value = $conf->{pending}->{$opt};
5098         eval {
5099             if ($opt eq 'hotplug') {
5100                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5101             } elsif ($opt eq 'tablet') {
5102                 die "skip\n" if !$hotplug_features->{usb};
5103                 if ($value == 1) {
5104                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5105                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5106                         if $arch eq 'aarch64';
5107                 } elsif ($value == 0) {
5108                     vm_deviceunplug($vmid, $conf, 'tablet');
5109                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5110                 }
5111             } elsif ($opt =~ m/^usb(\d+)$/) {
5112                 my $index = $1;
5113                 die "skip\n" if !$usb_hotplug;
5114                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5115                 my $id = $opt;
5116                 if ($d->{host} =~ m/^spice$/i)  {
5117                     $id = "usbredirdev$index";
5118                 }
5119                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5120             } elsif ($opt eq 'vcpus') {
5121                 die "skip\n" if !$hotplug_features->{cpu};
5122                 qemu_cpu_hotplug($vmid, $conf, $value);
5123             } elsif ($opt eq 'balloon') {
5124                 # enable/disable balloning device is not hotpluggable
5125                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5126                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5127                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5128
5129                 # allow manual ballooning if shares is set to zero
5130                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5131                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5132                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5133                 }
5134             } elsif ($opt =~ m/^net(\d+)$/) {
5135                 # some changes can be done without hotplug
5136                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5137                                     $vmid, $opt, $value, $arch, $machine_type);
5138             } elsif (is_valid_drivename($opt)) {
5139                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5140                 # some changes can be done without hotplug
5141                 my $drive = parse_drive($opt, $value);
5142                 if (drive_is_cloudinit($drive)) {
5143                     $cloudinit_opt = [$opt, $drive];
5144                     # apply all the other changes first, then generate the cloudinit disk
5145                     die "skip\n";
5146                 }
5147                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5148                                      $vmid, $opt, $value, $arch, $machine_type);
5149             } elsif ($opt =~ m/^memory$/) { #dimms
5150                 die "skip\n" if !$hotplug_features->{memory};
5151                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5152             } elsif ($opt eq 'cpuunits') {
5153                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5154                 $cgroup->change_cpu_shares($new_cpuunits);
5155             } elsif ($opt eq 'cpulimit') {
5156                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5157                 $cgroup->change_cpu_quota($cpulimit, 100000);
5158             } elsif ($opt eq 'agent') {
5159                 vmconfig_update_agent($conf, $opt, $value);
5160             } else {
5161                 die "skip\n";  # skip non-hot-pluggable options
5162             }
5163         };
5164         if (my $err = $@) {
5165             &$add_error($opt, $err) if $err ne "skip\n";
5166         } else {
5167             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5168             $conf->{$opt} = $value;
5169             delete $conf->{pending}->{$opt};
5170         }
5171     }
5172
5173     if (defined($cloudinit_opt)) {
5174         my ($opt, $drive) = @$cloudinit_opt;
5175         my $value = $conf->{pending}->{$opt};
5176         eval {
5177             my $temp = {%$conf, $opt => $value};
5178             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5179             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5180                                  $vmid, $opt, $value, $arch, $machine_type);
5181         };
5182         if (my $err = $@) {
5183             &$add_error($opt, $err) if $err ne "skip\n";
5184         } else {
5185             $conf->{$opt} = $value;
5186             delete $conf->{pending}->{$opt};
5187         }
5188     }
5189
5190     # unplug xhci controller if no usb device is left
5191     if ($usb_hotplug) {
5192         my $has_usb = 0;
5193         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5194             next if !defined($conf->{"usb$i"});
5195             $has_usb = 1;
5196             last;
5197         }
5198         if (!$has_usb) {
5199             vm_deviceunplug($vmid, $conf, 'xhci');
5200         }
5201     }
5202
5203     PVE::QemuConfig->write_config($vmid, $conf);
5204
5205     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5206         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5207     }
5208 }
5209
5210 sub try_deallocate_drive {
5211     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5212
5213     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5214         my $volid = $drive->{file};
5215         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5216             my $sid = PVE::Storage::parse_volume_id($volid);
5217             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5218
5219             # check if the disk is really unused
5220             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5221                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5222             PVE::Storage::vdisk_free($storecfg, $volid);
5223             return 1;
5224         } else {
5225             # If vm is not owner of this disk remove from config
5226             return 1;
5227         }
5228     }
5229
5230     return;
5231 }
5232
5233 sub vmconfig_delete_or_detach_drive {
5234     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5235
5236     my $drive = parse_drive($opt, $conf->{$opt});
5237
5238     my $rpcenv = PVE::RPCEnvironment::get();
5239     my $authuser = $rpcenv->get_user();
5240
5241     if ($force) {
5242         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5243         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5244     } else {
5245         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5246     }
5247 }
5248
5249
5250
5251 sub vmconfig_apply_pending {
5252     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5253
5254     return if !scalar(keys %{$conf->{pending}});
5255
5256     my $add_apply_error = sub {
5257         my ($opt, $msg) = @_;
5258         my $err_msg = "unable to apply pending change $opt : $msg";
5259         $errors->{$opt} = $err_msg;
5260         warn $err_msg;
5261     };
5262
5263     # cold plug
5264
5265     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5266     foreach my $opt (sort keys %$pending_delete_hash) {
5267         my $force = $pending_delete_hash->{$opt}->{force};
5268         eval {
5269             if ($opt =~ m/^unused/) {
5270                 die "internal error";
5271             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5272                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5273             }
5274         };
5275         if (my $err = $@) {
5276             $add_apply_error->($opt, $err);
5277         } else {
5278             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5279             delete $conf->{$opt};
5280         }
5281     }
5282
5283     PVE::QemuConfig->cleanup_pending($conf);
5284
5285     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5286
5287     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5288         next if $opt eq 'delete'; # just to be sure
5289         eval {
5290             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5291                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5292             }
5293         };
5294         if (my $err = $@) {
5295             $add_apply_error->($opt, $err);
5296         } else {
5297
5298             if (is_valid_drivename($opt)) {
5299                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5300                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5301             }
5302
5303             $conf->{$opt} = delete $conf->{pending}->{$opt};
5304         }
5305     }
5306
5307     # write all changes at once to avoid unnecessary i/o
5308     PVE::QemuConfig->write_config($vmid, $conf);
5309     if ($generate_cloudinit) {
5310         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5311             # After successful generation and if there were changes to be applied, update the
5312             # config to drop the {cloudinit} entry.
5313             PVE::QemuConfig->write_config($vmid, $conf);
5314         }
5315     }
5316 }
5317
5318 sub vmconfig_update_net {
5319     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5320
5321     my $newnet = parse_net($value);
5322
5323     if ($conf->{$opt}) {
5324         my $oldnet = parse_net($conf->{$opt});
5325
5326         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5327             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5328             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5329             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5330             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5331
5332             # for non online change, we try to hot-unplug
5333             die "skip\n" if !$hotplug;
5334             vm_deviceunplug($vmid, $conf, $opt);
5335         } else {
5336
5337             die "internal error" if $opt !~ m/net(\d+)/;
5338             my $iface = "tap${vmid}i$1";
5339
5340             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5341                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5342                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5343                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5344                 PVE::Network::tap_unplug($iface);
5345
5346                 if ($have_sdn) {
5347                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5348                 } else {
5349                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5350                 }
5351             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5352                 # Rate can be applied on its own but any change above needs to
5353                 # include the rate in tap_plug since OVS resets everything.
5354                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5355             }
5356
5357             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5358                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5359             }
5360
5361             return 1;
5362         }
5363     }
5364
5365     if ($hotplug) {
5366         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5367     } else {
5368         die "skip\n";
5369     }
5370 }
5371
5372 sub vmconfig_update_agent {
5373     my ($conf, $opt, $value) = @_;
5374
5375     die "skip\n" if !$conf->{$opt};
5376
5377     my $hotplug_options = { fstrim_cloned_disks => 1 };
5378
5379     my $old_agent = parse_guest_agent($conf);
5380     my $agent = parse_guest_agent({$opt => $value});
5381
5382     for my $option (keys %$agent) { # added/changed options
5383         next if defined($hotplug_options->{$option});
5384         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5385     }
5386
5387     for my $option (keys %$old_agent) { # removed options
5388         next if defined($hotplug_options->{$option});
5389         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5390     }
5391
5392     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5393 }
5394
5395 sub vmconfig_update_disk {
5396     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5397
5398     my $drive = parse_drive($opt, $value);
5399
5400     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5401         my $media = $drive->{media} || 'disk';
5402         my $oldmedia = $old_drive->{media} || 'disk';
5403         die "unable to change media type\n" if $media ne $oldmedia;
5404
5405         if (!drive_is_cdrom($old_drive)) {
5406
5407             if ($drive->{file} ne $old_drive->{file}) {
5408
5409                 die "skip\n" if !$hotplug;
5410
5411                 # unplug and register as unused
5412                 vm_deviceunplug($vmid, $conf, $opt);
5413                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5414
5415             } else {
5416                 # update existing disk
5417
5418                 # skip non hotpluggable value
5419                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5420                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5421                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5422                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5423                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5424                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5425                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5426                     die "skip\n";
5427                 }
5428
5429                 # apply throttle
5430                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5431                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5432                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5433                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5434                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5435                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5436                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5437                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5438                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5439                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5440                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5441                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5442                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5443                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5444                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5445                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5446                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5447                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5448
5449                     qemu_block_set_io_throttle(
5450                         $vmid,"drive-$opt",
5451                         ($drive->{mbps} || 0)*1024*1024,
5452                         ($drive->{mbps_rd} || 0)*1024*1024,
5453                         ($drive->{mbps_wr} || 0)*1024*1024,
5454                          $drive->{iops} || 0,
5455                          $drive->{iops_rd} || 0,
5456                          $drive->{iops_wr} || 0,
5457                         ($drive->{mbps_max} || 0)*1024*1024,
5458                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5459                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5460                          $drive->{iops_max} || 0,
5461                          $drive->{iops_rd_max} || 0,
5462                          $drive->{iops_wr_max} || 0,
5463                          $drive->{bps_max_length} || 1,
5464                          $drive->{bps_rd_max_length} || 1,
5465                          $drive->{bps_wr_max_length} || 1,
5466                          $drive->{iops_max_length} || 1,
5467                          $drive->{iops_rd_max_length} || 1,
5468                          $drive->{iops_wr_max_length} || 1,
5469                     );
5470
5471                 }
5472
5473                 return 1;
5474             }
5475
5476         } else { # cdrom
5477
5478             if ($drive->{file} eq 'none') {
5479                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5480                 if (drive_is_cloudinit($old_drive)) {
5481                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5482                 }
5483             } else {
5484                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5485
5486                 # force eject if locked
5487                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5488
5489                 if ($path) {
5490                     mon_cmd($vmid, "blockdev-change-medium",
5491                         id => "$opt", filename => "$path");
5492                 }
5493             }
5494
5495             return 1;
5496         }
5497     }
5498
5499     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5500     # hotplug new disks
5501     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5502     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5503 }
5504
5505 sub vmconfig_update_cloudinit_drive {
5506     my ($storecfg, $conf, $vmid) = @_;
5507
5508     my $cloudinit_ds = undef;
5509     my $cloudinit_drive = undef;
5510
5511     PVE::QemuConfig->foreach_volume($conf, sub {
5512         my ($ds, $drive) = @_;
5513         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5514             $cloudinit_ds = $ds;
5515             $cloudinit_drive = $drive;
5516         }
5517     });
5518
5519     return if !$cloudinit_drive;
5520
5521     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5522         PVE::QemuConfig->write_config($vmid, $conf);
5523     }
5524
5525     my $running = PVE::QemuServer::check_running($vmid);
5526
5527     if ($running) {
5528         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5529         if ($path) {
5530             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5531             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5532         }
5533     }
5534 }
5535
5536 # called in locked context by incoming migration
5537 sub vm_migrate_get_nbd_disks {
5538     my ($storecfg, $conf, $replicated_volumes) = @_;
5539
5540     my $local_volumes = {};
5541     PVE::QemuConfig->foreach_volume($conf, sub {
5542         my ($ds, $drive) = @_;
5543
5544         return if drive_is_cdrom($drive);
5545         return if $ds eq 'tpmstate0';
5546
5547         my $volid = $drive->{file};
5548
5549         return if !$volid;
5550
5551         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5552
5553         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5554         return if $scfg->{shared};
5555
5556         # replicated disks re-use existing state via bitmap
5557         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5558         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5559     });
5560     return $local_volumes;
5561 }
5562
5563 # called in locked context by incoming migration
5564 sub vm_migrate_alloc_nbd_disks {
5565     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5566
5567     my $nbd = {};
5568     foreach my $opt (sort keys %$source_volumes) {
5569         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5570
5571         if ($use_existing) {
5572             $nbd->{$opt}->{drivestr} = print_drive($drive);
5573             $nbd->{$opt}->{volid} = $volid;
5574             $nbd->{$opt}->{replicated} = 1;
5575             next;
5576         }
5577
5578         # storage mapping + volname = regular migration
5579         # storage mapping + format = remote migration
5580         # order of precedence, filtered by whether storage supports it:
5581         # 1. explicit requested format
5582         # 2. format of current volume
5583         # 3. default format of storage
5584         if (!$storagemap->{identity}) {
5585             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5586             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5587             if (!$format || !grep { $format eq $_ } @$validFormats) {
5588                 if ($volname) {
5589                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5590                     my $fileFormat = qemu_img_format($scfg, $volname);
5591                     $format = $fileFormat
5592                         if grep { $fileFormat eq $_ } @$validFormats;
5593                 }
5594                 $format //= $defFormat;
5595             }
5596         } else {
5597             # can't happen for remote migration, so $volname is always defined
5598             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5599             $format = qemu_img_format($scfg, $volname);
5600         }
5601
5602         my $size = $drive->{size} / 1024;
5603         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5604         my $newdrive = $drive;
5605         $newdrive->{format} = $format;
5606         $newdrive->{file} = $newvolid;
5607         my $drivestr = print_drive($newdrive);
5608         $nbd->{$opt}->{drivestr} = $drivestr;
5609         $nbd->{$opt}->{volid} = $newvolid;
5610     }
5611
5612     return $nbd;
5613 }
5614
5615 # see vm_start_nolock for parameters, additionally:
5616 # migrate_opts:
5617 #   storagemap = parsed storage map for allocating NBD disks
5618 sub vm_start {
5619     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5620
5621     return PVE::QemuConfig->lock_config($vmid, sub {
5622         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5623
5624         die "you can't start a vm if it's a template\n"
5625             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5626
5627         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5628         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5629
5630         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5631
5632         if ($has_backup_lock && $running) {
5633             # a backup is currently running, attempt to start the guest in the
5634             # existing QEMU instance
5635             return vm_resume($vmid);
5636         }
5637
5638         PVE::QemuConfig->check_lock($conf)
5639             if !($params->{skiplock} || $has_suspended_lock);
5640
5641         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5642
5643         die "VM $vmid already running\n" if $running;
5644
5645         if (my $storagemap = $migrate_opts->{storagemap}) {
5646             my $replicated = $migrate_opts->{replicated_volumes};
5647             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5648             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5649
5650             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5651                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5652             }
5653         }
5654
5655         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5656     });
5657 }
5658
5659
5660 # params:
5661 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5662 #   skiplock => 0/1, skip checking for config lock
5663 #   skiptemplate => 0/1, skip checking whether VM is template
5664 #   forcemachine => to force QEMU machine (rollback/migration)
5665 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5666 #   timeout => in seconds
5667 #   paused => start VM in paused state (backup)
5668 #   resume => resume from hibernation
5669 #   pbs-backing => {
5670 #      sata0 => {
5671 #         repository
5672 #         snapshot
5673 #         keyfile
5674 #         archive
5675 #      },
5676 #      virtio2 => ...
5677 #   }
5678 # migrate_opts:
5679 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5680 #   migratedfrom => source node
5681 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5682 #   network => CIDR of migration network
5683 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5684 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5685 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5686 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5687 #       contained in config
5688 sub vm_start_nolock {
5689     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5690
5691     my $statefile = $params->{statefile};
5692     my $resume = $params->{resume};
5693
5694     my $migratedfrom = $migrate_opts->{migratedfrom};
5695     my $migration_type = $migrate_opts->{type};
5696
5697     my $res = {};
5698
5699     # clean up leftover reboot request files
5700     eval { clear_reboot_request($vmid); };
5701     warn $@ if $@;
5702
5703     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5704         vmconfig_apply_pending($vmid, $conf, $storecfg);
5705         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5706     }
5707
5708     # don't regenerate the ISO if the VM is started as part of a live migration
5709     # this way we can reuse the old ISO with the correct config
5710     if (!$migratedfrom) {
5711         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5712             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5713             # $conf->{cloudinit}, so we could just not do this?
5714             # But we do it above, so for now let's be consistent.
5715             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5716         }
5717     }
5718
5719     # override offline migrated volumes, conf is out of date still
5720     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5721         for my $key (sort keys $offline_volumes->%*) {
5722             my $parsed = parse_drive($key, $conf->{$key});
5723             $parsed->{file} = $offline_volumes->{$key};
5724             $conf->{$key} = print_drive($parsed);
5725         }
5726     }
5727
5728     my $defaults = load_defaults();
5729
5730     # set environment variable useful inside network script
5731     # for remote migration the config is available on the target node!
5732     if (!$migrate_opts->{remote_node}) {
5733         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5734     }
5735
5736     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5737
5738     my $forcemachine = $params->{forcemachine};
5739     my $forcecpu = $params->{forcecpu};
5740     if ($resume) {
5741         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5742         $forcemachine = $conf->{runningmachine};
5743         $forcecpu = $conf->{runningcpu};
5744         print "Resuming suspended VM\n";
5745     }
5746
5747     my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5748         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5749
5750     my $migration_ip;
5751     my $get_migration_ip = sub {
5752         my ($nodename) = @_;
5753
5754         return $migration_ip if defined($migration_ip);
5755
5756         my $cidr = $migrate_opts->{network};
5757
5758         if (!defined($cidr)) {
5759             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5760             $cidr = $dc_conf->{migration}->{network};
5761         }
5762
5763         if (defined($cidr)) {
5764             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5765
5766             die "could not get IP: no address configured on local " .
5767                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5768
5769             die "could not get IP: multiple addresses configured on local " .
5770                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5771
5772             $migration_ip = @$ips[0];
5773         }
5774
5775         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5776             if !defined($migration_ip);
5777
5778         return $migration_ip;
5779     };
5780
5781     if ($statefile) {
5782         if ($statefile eq 'tcp') {
5783             my $migrate = $res->{migrate} = { proto => 'tcp' };
5784             $migrate->{addr} = "localhost";
5785             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5786             my $nodename = nodename();
5787
5788             if (!defined($migration_type)) {
5789                 if (defined($datacenterconf->{migration}->{type})) {
5790                     $migration_type = $datacenterconf->{migration}->{type};
5791                 } else {
5792                     $migration_type = 'secure';
5793                 }
5794             }
5795
5796             if ($migration_type eq 'insecure') {
5797                 $migrate->{addr} = $get_migration_ip->($nodename);
5798                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5799             }
5800
5801             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5802             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5803             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5804             push @$cmd, '-incoming', $migrate->{uri};
5805             push @$cmd, '-S';
5806
5807         } elsif ($statefile eq 'unix') {
5808             # should be default for secure migrations as a ssh TCP forward
5809             # tunnel is not deterministic reliable ready and fails regurarly
5810             # to set up in time, so use UNIX socket forwards
5811             my $migrate = $res->{migrate} = { proto => 'unix' };
5812             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5813             unlink $migrate->{addr};
5814
5815             $migrate->{uri} = "unix:$migrate->{addr}";
5816             push @$cmd, '-incoming', $migrate->{uri};
5817             push @$cmd, '-S';
5818
5819         } elsif (-e $statefile) {
5820             push @$cmd, '-loadstate', $statefile;
5821         } else {
5822             my $statepath = PVE::Storage::path($storecfg, $statefile);
5823             push @$vollist, $statefile;
5824             push @$cmd, '-loadstate', $statepath;
5825         }
5826     } elsif ($params->{paused}) {
5827         push @$cmd, '-S';
5828     }
5829
5830     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5831
5832     my $pci_reserve_list = [];
5833     for my $device (values $pci_devices->%*) {
5834         next if $device->{mdev}; # we don't reserve for mdev devices
5835         push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5836     }
5837
5838     # reserve all PCI IDs before actually doing anything with them
5839     PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5840
5841     eval {
5842         my $uuid;
5843         for my $id (sort keys %$pci_devices) {
5844             my $d = $pci_devices->{$id};
5845             my ($index) = ($id =~ m/^hostpci(\d+)$/);
5846
5847             my $chosen_mdev;
5848             for my $dev ($d->{ids}->@*) {
5849                 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5850                 if ($d->{mdev}) {
5851                     warn $@ if $@;
5852                     $chosen_mdev = $info;
5853                     last if $chosen_mdev; # if successful, we're done
5854                 } else {
5855                     die $@ if $@;
5856                 }
5857             }
5858
5859             next if !$d->{mdev};
5860             die "could not create mediated device\n" if !defined($chosen_mdev);
5861
5862             # nvidia grid needs the uuid of the mdev as qemu parameter
5863             if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5864                 if (defined($conf->{smbios1})) {
5865                     my $smbios_conf = parse_smbios1($conf->{smbios1});
5866                     $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5867                 }
5868                 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5869             }
5870         }
5871         push @$cmd, '-uuid', $uuid if defined($uuid);
5872     };
5873     if (my $err = $@) {
5874         eval { cleanup_pci_devices($vmid, $conf) };
5875         warn $@ if $@;
5876         die $err;
5877     }
5878
5879     PVE::Storage::activate_volumes($storecfg, $vollist);
5880
5881
5882     my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5883     eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5884     eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5885     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5886     # timeout should be more than enough here...
5887     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5888
5889     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5890
5891     my %run_params = (
5892         timeout => $statefile ? undef : $start_timeout,
5893         umask => 0077,
5894         noerr => 1,
5895     );
5896
5897     # when migrating, prefix QEMU output so other side can pick up any
5898     # errors that might occur and show the user
5899     if ($migratedfrom) {
5900         $run_params{quiet} = 1;
5901         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5902     }
5903
5904     my %systemd_properties = (
5905         Slice => 'qemu.slice',
5906         KillMode => 'process',
5907         SendSIGKILL => 0,
5908         TimeoutStopUSec => ULONG_MAX, # infinity
5909     );
5910
5911     if (PVE::CGroup::cgroup_mode() == 2) {
5912         $systemd_properties{CPUWeight} = $cpuunits;
5913     } else {
5914         $systemd_properties{CPUShares} = $cpuunits;
5915     }
5916
5917     if (my $cpulimit = $conf->{cpulimit}) {
5918         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5919     }
5920     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5921
5922     my $run_qemu = sub {
5923         PVE::Tools::run_fork sub {
5924             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5925
5926             my $tpmpid;
5927             if (my $tpm = $conf->{tpmstate0}) {
5928                 # start the TPM emulator so QEMU can connect on start
5929                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5930             }
5931
5932             my $exitcode = run_command($cmd, %run_params);
5933             if ($exitcode) {
5934                 if ($tpmpid) {
5935                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5936                     kill 'TERM', $tpmpid;
5937                 }
5938                 die "QEMU exited with code $exitcode\n";
5939             }
5940         };
5941     };
5942
5943     if ($conf->{hugepages}) {
5944
5945         my $code = sub {
5946             my $hotplug_features =
5947                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5948             my $hugepages_topology =
5949                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5950
5951             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5952
5953             PVE::QemuServer::Memory::hugepages_mount();
5954             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5955
5956             eval { $run_qemu->() };
5957             if (my $err = $@) {
5958                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5959                     if !$conf->{keephugepages};
5960                 die $err;
5961             }
5962
5963             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5964                 if !$conf->{keephugepages};
5965         };
5966         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5967
5968     } else {
5969         eval { $run_qemu->() };
5970     }
5971
5972     if (my $err = $@) {
5973         # deactivate volumes if start fails
5974         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5975         warn $@ if $@;
5976         eval { cleanup_pci_devices($vmid, $conf) };
5977         warn $@ if $@;
5978
5979         die "start failed: $err";
5980     }
5981
5982     # re-reserve all PCI IDs now that we can know the actual VM PID
5983     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5984     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5985     warn $@ if $@;
5986
5987     if (defined($res->{migrate})) {
5988         print "migration listens on $res->{migrate}->{uri}\n";
5989     } elsif ($statefile) {
5990         eval { mon_cmd($vmid, "cont"); };
5991         warn $@ if $@;
5992     }
5993
5994     #start nbd server for storage migration
5995     if (my $nbd = $migrate_opts->{nbd}) {
5996         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5997
5998         my $migrate_storage_uri;
5999         # nbd_protocol_version > 0 for unix socket support
6000         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6001             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6002             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
6003             $migrate_storage_uri = "nbd:unix:$socket_path";
6004             $res->{migrate}->{unix_sockets} = [$socket_path];
6005         } else {
6006             my $nodename = nodename();
6007             my $localip = $get_migration_ip->($nodename);
6008             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6009             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6010
6011             mon_cmd($vmid, "nbd-server-start", addr => {
6012                 type => 'inet',
6013                 data => {
6014                     host => "${localip}",
6015                     port => "${storage_migrate_port}",
6016                 },
6017             });
6018             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6019             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6020         }
6021
6022         my $block_info = mon_cmd($vmid, "query-block");
6023         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6024
6025         foreach my $opt (sort keys %$nbd) {
6026             my $drivestr = $nbd->{$opt}->{drivestr};
6027             my $volid = $nbd->{$opt}->{volid};
6028
6029             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6030
6031             mon_cmd(
6032                 $vmid,
6033                 "block-export-add",
6034                 id => "drive-$opt",
6035                 'node-name' => $block_node,
6036                 writable => JSON::true,
6037                 type => "nbd",
6038                 name => "drive-$opt", # NBD export name
6039             );
6040
6041             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6042             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6043             print "re-using replicated volume: $opt - $volid\n"
6044                 if $nbd->{$opt}->{replicated};
6045
6046             $res->{drives}->{$opt} = $nbd->{$opt};
6047             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6048         }
6049     }
6050
6051     if ($migratedfrom) {
6052         eval {
6053             set_migration_caps($vmid);
6054         };
6055         warn $@ if $@;
6056
6057         if ($spice_port) {
6058             print "spice listens on port $spice_port\n";
6059             $res->{spice_port} = $spice_port;
6060             if ($migrate_opts->{spice_ticket}) {
6061                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6062                     $migrate_opts->{spice_ticket});
6063                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6064             }
6065         }
6066
6067     } else {
6068         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6069             if !$statefile && $conf->{balloon};
6070
6071         foreach my $opt (keys %$conf) {
6072             next if $opt !~  m/^net\d+$/;
6073             my $nicconf = parse_net($conf->{$opt});
6074             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6075         }
6076         add_nets_bridge_fdb($conf, $vmid);
6077     }
6078
6079    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6080         eval {
6081             mon_cmd(
6082                 $vmid,
6083                 'qom-set',
6084                 path => "machine/peripheral/balloon0",
6085                 property => "guest-stats-polling-interval",
6086                 value => 2
6087             );
6088         };
6089         log_warn("could not set polling interval for ballooning - $@") if $@;
6090     }
6091
6092     if ($resume) {
6093         print "Resumed VM, removing state\n";
6094         if (my $vmstate = $conf->{vmstate}) {
6095             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6096             PVE::Storage::vdisk_free($storecfg, $vmstate);
6097         }
6098         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6099         PVE::QemuConfig->write_config($vmid, $conf);
6100     }
6101
6102     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6103
6104     return $res;
6105 }
6106
6107 sub vm_commandline {
6108     my ($storecfg, $vmid, $snapname) = @_;
6109
6110     my $conf = PVE::QemuConfig->load_config($vmid);
6111
6112     my ($forcemachine, $forcecpu);
6113     if ($snapname) {
6114         my $snapshot = $conf->{snapshots}->{$snapname};
6115         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6116
6117         # check for machine or CPU overrides in snapshot
6118         $forcemachine = $snapshot->{runningmachine};
6119         $forcecpu = $snapshot->{runningcpu};
6120
6121         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6122
6123         $conf = $snapshot;
6124     }
6125
6126     my $defaults = load_defaults();
6127
6128     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6129
6130     return PVE::Tools::cmd2string($cmd);
6131 }
6132
6133 sub vm_reset {
6134     my ($vmid, $skiplock) = @_;
6135
6136     PVE::QemuConfig->lock_config($vmid, sub {
6137
6138         my $conf = PVE::QemuConfig->load_config($vmid);
6139
6140         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6141
6142         mon_cmd($vmid, "system_reset");
6143     });
6144 }
6145
6146 sub get_vm_volumes {
6147     my ($conf) = @_;
6148
6149     my $vollist = [];
6150     foreach_volid($conf, sub {
6151         my ($volid, $attr) = @_;
6152
6153         return if $volid =~ m|^/|;
6154
6155         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6156         return if !$sid;
6157
6158         push @$vollist, $volid;
6159     });
6160
6161     return $vollist;
6162 }
6163
6164 sub cleanup_pci_devices {
6165     my ($vmid, $conf) = @_;
6166
6167     foreach my $key (keys %$conf) {
6168         next if $key !~ m/^hostpci(\d+)$/;
6169         my $hostpciindex = $1;
6170         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6171         my $d = parse_hostpci($conf->{$key});
6172         if ($d->{mdev}) {
6173             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6174             # don't want to break ABI just for this two liner
6175             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6176
6177             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6178             # out when we do it first. so wait for 10 seconds and then try it
6179             if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/) {
6180                 sleep 10;
6181             }
6182
6183             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6184         }
6185     }
6186     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6187 }
6188
6189 sub vm_stop_cleanup {
6190     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6191
6192     eval {
6193
6194         if (!$keepActive) {
6195             my $vollist = get_vm_volumes($conf);
6196             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6197
6198             if (my $tpmdrive = $conf->{tpmstate0}) {
6199                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6200                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6201                 if ($storeid) {
6202                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6203                 }
6204             }
6205         }
6206
6207         foreach my $ext (qw(mon qmp pid vnc qga)) {
6208             unlink "/var/run/qemu-server/${vmid}.$ext";
6209         }
6210
6211         if ($conf->{ivshmem}) {
6212             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6213             # just delete it for now, VMs which have this already open do not
6214             # are affected, but new VMs will get a separated one. If this
6215             # becomes an issue we either add some sort of ref-counting or just
6216             # add a "don't delete on stop" flag to the ivshmem format.
6217             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6218         }
6219
6220         cleanup_pci_devices($vmid, $conf);
6221
6222         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6223     };
6224     warn $@ if $@; # avoid errors - just warn
6225 }
6226
6227 # call only in locked context
6228 sub _do_vm_stop {
6229     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6230
6231     my $pid = check_running($vmid, $nocheck);
6232     return if !$pid;
6233
6234     my $conf;
6235     if (!$nocheck) {
6236         $conf = PVE::QemuConfig->load_config($vmid);
6237         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6238         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6239             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6240             $timeout = $opts->{down} if $opts->{down};
6241         }
6242         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6243     }
6244
6245     eval {
6246         if ($shutdown) {
6247             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6248                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6249             } else {
6250                 mon_cmd($vmid, "system_powerdown");
6251             }
6252         } else {
6253             mon_cmd($vmid, "quit");
6254         }
6255     };
6256     my $err = $@;
6257
6258     if (!$err) {
6259         $timeout = 60 if !defined($timeout);
6260
6261         my $count = 0;
6262         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6263             $count++;
6264             sleep 1;
6265         }
6266
6267         if ($count >= $timeout) {
6268             if ($force) {
6269                 warn "VM still running - terminating now with SIGTERM\n";
6270                 kill 15, $pid;
6271             } else {
6272                 die "VM quit/powerdown failed - got timeout\n";
6273             }
6274         } else {
6275             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6276             return;
6277         }
6278     } else {
6279        if (!check_running($vmid, $nocheck)) {
6280             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6281             return;
6282        }
6283        if ($force) {
6284             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6285             kill 15, $pid;
6286         } else {
6287             die "VM quit/powerdown failed\n";
6288         }
6289     }
6290
6291     # wait again
6292     $timeout = 10;
6293
6294     my $count = 0;
6295     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6296         $count++;
6297         sleep 1;
6298     }
6299
6300     if ($count >= $timeout) {
6301         warn "VM still running - terminating now with SIGKILL\n";
6302         kill 9, $pid;
6303         sleep 1;
6304     }
6305
6306     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6307 }
6308
6309 # Note: use $nocheck to skip tests if VM configuration file exists.
6310 # We need that when migration VMs to other nodes (files already moved)
6311 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6312 sub vm_stop {
6313     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6314
6315     $force = 1 if !defined($force) && !$shutdown;
6316
6317     if ($migratedfrom){
6318         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6319         kill 15, $pid if $pid;
6320         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6321         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6322         return;
6323     }
6324
6325     PVE::QemuConfig->lock_config($vmid, sub {
6326         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6327    });
6328 }
6329
6330 sub vm_reboot {
6331     my ($vmid, $timeout) = @_;
6332
6333     PVE::QemuConfig->lock_config($vmid, sub {
6334         eval {
6335
6336             # only reboot if running, as qmeventd starts it again on a stop event
6337             return if !check_running($vmid);
6338
6339             create_reboot_request($vmid);
6340
6341             my $storecfg = PVE::Storage::config();
6342             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6343
6344         };
6345         if (my $err = $@) {
6346             # avoid that the next normal shutdown will be confused for a reboot
6347             clear_reboot_request($vmid);
6348             die $err;
6349         }
6350    });
6351 }
6352
6353 # note: if using the statestorage parameter, the caller has to check privileges
6354 sub vm_suspend {
6355     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6356
6357     my $conf;
6358     my $path;
6359     my $storecfg;
6360     my $vmstate;
6361
6362     PVE::QemuConfig->lock_config($vmid, sub {
6363
6364         $conf = PVE::QemuConfig->load_config($vmid);
6365
6366         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6367         PVE::QemuConfig->check_lock($conf)
6368             if !($skiplock || $is_backing_up);
6369
6370         die "cannot suspend to disk during backup\n"
6371             if $is_backing_up && $includestate;
6372
6373         if ($includestate) {
6374             $conf->{lock} = 'suspending';
6375             my $date = strftime("%Y-%m-%d", localtime(time()));
6376             $storecfg = PVE::Storage::config();
6377             if (!$statestorage) {
6378                 $statestorage = find_vmstate_storage($conf, $storecfg);
6379                 # check permissions for the storage
6380                 my $rpcenv = PVE::RPCEnvironment::get();
6381                 if ($rpcenv->{type} ne 'cli') {
6382                     my $authuser = $rpcenv->get_user();
6383                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6384                 }
6385             }
6386
6387
6388             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6389                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6390             $path = PVE::Storage::path($storecfg, $vmstate);
6391             PVE::QemuConfig->write_config($vmid, $conf);
6392         } else {
6393             mon_cmd($vmid, "stop");
6394         }
6395     });
6396
6397     if ($includestate) {
6398         # save vm state
6399         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6400
6401         eval {
6402             set_migration_caps($vmid, 1);
6403             mon_cmd($vmid, "savevm-start", statefile => $path);
6404             for(;;) {
6405                 my $state = mon_cmd($vmid, "query-savevm");
6406                 if (!$state->{status}) {
6407                     die "savevm not active\n";
6408                 } elsif ($state->{status} eq 'active') {
6409                     sleep(1);
6410                     next;
6411                 } elsif ($state->{status} eq 'completed') {
6412                     print "State saved, quitting\n";
6413                     last;
6414                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6415                     die "query-savevm failed with error '$state->{error}'\n"
6416                 } else {
6417                     die "query-savevm returned status '$state->{status}'\n";
6418                 }
6419             }
6420         };
6421         my $err = $@;
6422
6423         PVE::QemuConfig->lock_config($vmid, sub {
6424             $conf = PVE::QemuConfig->load_config($vmid);
6425             if ($err) {
6426                 # cleanup, but leave suspending lock, to indicate something went wrong
6427                 eval {
6428                     mon_cmd($vmid, "savevm-end");
6429                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6430                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6431                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6432                     PVE::QemuConfig->write_config($vmid, $conf);
6433                 };
6434                 warn $@ if $@;
6435                 die $err;
6436             }
6437
6438             die "lock changed unexpectedly\n"
6439                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6440
6441             mon_cmd($vmid, "quit");
6442             $conf->{lock} = 'suspended';
6443             PVE::QemuConfig->write_config($vmid, $conf);
6444         });
6445     }
6446 }
6447
6448 # $nocheck is set when called as part of a migration - in this context the
6449 # location of the config file (source or target node) is not deterministic,
6450 # since migration cannot wait for pmxcfs to process the rename
6451 sub vm_resume {
6452     my ($vmid, $skiplock, $nocheck) = @_;
6453
6454     PVE::QemuConfig->lock_config($vmid, sub {
6455         my $res = mon_cmd($vmid, 'query-status');
6456         my $resume_cmd = 'cont';
6457         my $reset = 0;
6458         my $conf;
6459         if ($nocheck) {
6460             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6461             if ($@) {
6462                 my $vmlist = PVE::Cluster::get_vmlist();
6463                 if (exists($vmlist->{ids}->{$vmid})) {
6464                     my $node = $vmlist->{ids}->{$vmid}->{node};
6465                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6466                 }
6467                 if (!$conf) {
6468                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6469                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6470                 }
6471             }
6472         } else {
6473             $conf = PVE::QemuConfig->load_config($vmid);
6474         }
6475
6476         if ($res->{status}) {
6477             return if $res->{status} eq 'running'; # job done, go home
6478             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6479             $reset = 1 if $res->{status} eq 'shutdown';
6480         }
6481
6482         if (!$nocheck) {
6483             PVE::QemuConfig->check_lock($conf)
6484                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6485         }
6486
6487         if ($reset) {
6488             # required if a VM shuts down during a backup and we get a resume
6489             # request before the backup finishes for example
6490             mon_cmd($vmid, "system_reset");
6491         }
6492
6493         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6494
6495         mon_cmd($vmid, $resume_cmd);
6496     });
6497 }
6498
6499 sub vm_sendkey {
6500     my ($vmid, $skiplock, $key) = @_;
6501
6502     PVE::QemuConfig->lock_config($vmid, sub {
6503
6504         my $conf = PVE::QemuConfig->load_config($vmid);
6505
6506         # there is no qmp command, so we use the human monitor command
6507         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6508         die $res if $res ne '';
6509     });
6510 }
6511
6512 sub check_bridge_access {
6513     my ($rpcenv, $authuser, $conf) = @_;
6514
6515     return 1 if $authuser eq 'root@pam';
6516
6517     for my $opt (sort keys $conf->%*) {
6518         next if $opt !~ m/^net\d+$/;
6519         my $net = parse_net($conf->{$opt});
6520         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6521         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6522     }
6523     return 1;
6524 };
6525
6526 sub check_mapping_access {
6527     my ($rpcenv, $user, $conf) = @_;
6528
6529     for my $opt (keys $conf->%*) {
6530         if ($opt =~ m/^usb\d+$/) {
6531             my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6532             if (my $host = $device->{host}) {
6533                 die "only root can set '$opt' config for real devices\n"
6534                     if $host !~ m/^spice$/i && $user ne 'root@pam';
6535             } elsif ($device->{mapping}) {
6536                 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6537             } else {
6538                 die "either 'host' or 'mapping' must be set.\n";
6539             }
6540         } elsif ($opt =~ m/^hostpci\d+$/) {
6541             my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6542             if ($device->{host}) {
6543                 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6544             } elsif ($device->{mapping}) {
6545                 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6546             } else {
6547                 die "either 'host' or 'mapping' must be set.\n";
6548             }
6549        }
6550    }
6551 };
6552
6553 sub check_restore_permissions {
6554     my ($rpcenv, $user, $conf) = @_;
6555
6556     check_bridge_access($rpcenv, $user, $conf);
6557     check_mapping_access($rpcenv, $user, $conf);
6558 }
6559 # vzdump restore implementaion
6560
6561 sub tar_archive_read_firstfile {
6562     my $archive = shift;
6563
6564     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6565
6566     # try to detect archive type first
6567     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6568         die "unable to open file '$archive'\n";
6569     my $firstfile = <$fh>;
6570     kill 15, $pid;
6571     close $fh;
6572
6573     die "ERROR: archive contaions no data\n" if !$firstfile;
6574     chomp $firstfile;
6575
6576     return $firstfile;
6577 }
6578
6579 sub tar_restore_cleanup {
6580     my ($storecfg, $statfile) = @_;
6581
6582     print STDERR "starting cleanup\n";
6583
6584     if (my $fd = IO::File->new($statfile, "r")) {
6585         while (defined(my $line = <$fd>)) {
6586             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6587                 my $volid = $2;
6588                 eval {
6589                     if ($volid =~ m|^/|) {
6590                         unlink $volid || die 'unlink failed\n';
6591                     } else {
6592                         PVE::Storage::vdisk_free($storecfg, $volid);
6593                     }
6594                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6595                 };
6596                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6597             } else {
6598                 print STDERR "unable to parse line in statfile - $line";
6599             }
6600         }
6601         $fd->close();
6602     }
6603 }
6604
6605 sub restore_file_archive {
6606     my ($archive, $vmid, $user, $opts) = @_;
6607
6608     return restore_vma_archive($archive, $vmid, $user, $opts)
6609         if $archive eq '-';
6610
6611     my $info = PVE::Storage::archive_info($archive);
6612     my $format = $opts->{format} // $info->{format};
6613     my $comp = $info->{compression};
6614
6615     # try to detect archive format
6616     if ($format eq 'tar') {
6617         return restore_tar_archive($archive, $vmid, $user, $opts);
6618     } else {
6619         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6620     }
6621 }
6622
6623 # hepler to remove disks that will not be used after restore
6624 my $restore_cleanup_oldconf = sub {
6625     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6626
6627     my $kept_disks = {};
6628
6629     PVE::QemuConfig->foreach_volume($oldconf, sub {
6630         my ($ds, $drive) = @_;
6631
6632         return if drive_is_cdrom($drive, 1);
6633
6634         my $volid = $drive->{file};
6635         return if !$volid || $volid =~ m|^/|;
6636
6637         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6638         return if !$path || !$owner || ($owner != $vmid);
6639
6640         # Note: only delete disk we want to restore
6641         # other volumes will become unused
6642         if ($virtdev_hash->{$ds}) {
6643             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6644             if (my $err = $@) {
6645                 warn $err;
6646             }
6647         } else {
6648             $kept_disks->{$volid} = 1;
6649         }
6650     });
6651
6652     # after the restore we have no snapshots anymore
6653     for my $snapname (keys $oldconf->{snapshots}->%*) {
6654         my $snap = $oldconf->{snapshots}->{$snapname};
6655         if ($snap->{vmstate}) {
6656             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6657             if (my $err = $@) {
6658                 warn $err;
6659             }
6660         }
6661
6662         for my $volid (keys $kept_disks->%*) {
6663             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6664             warn $@ if $@;
6665         }
6666     }
6667 };
6668
6669 # Helper to parse vzdump backup device hints
6670 #
6671 # $rpcenv: Environment, used to ckeck storage permissions
6672 # $user: User ID, to check storage permissions
6673 # $storecfg: Storage configuration
6674 # $fh: the file handle for reading the configuration
6675 # $devinfo: should contain device sizes for all backu-up'ed devices
6676 # $options: backup options (pool, default storage)
6677 #
6678 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6679 my $parse_backup_hints = sub {
6680     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6681
6682     my $check_storage = sub { # assert if an image can be allocate
6683         my ($storeid, $scfg) = @_;
6684         die "Content type 'images' is not available on storage '$storeid'\n"
6685             if !$scfg->{content}->{images};
6686         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6687             if $user ne 'root@pam';
6688     };
6689
6690     my $virtdev_hash = {};
6691     while (defined(my $line = <$fh>)) {
6692         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6693             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6694             die "archive does not contain data for drive '$virtdev'\n"
6695                 if !$devinfo->{$devname};
6696
6697             if (defined($options->{storage})) {
6698                 $storeid = $options->{storage} || 'local';
6699             } elsif (!$storeid) {
6700                 $storeid = 'local';
6701             }
6702             $format = 'raw' if !$format;
6703             $devinfo->{$devname}->{devname} = $devname;
6704             $devinfo->{$devname}->{virtdev} = $virtdev;
6705             $devinfo->{$devname}->{format} = $format;
6706             $devinfo->{$devname}->{storeid} = $storeid;
6707
6708             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6709             $check_storage->($storeid, $scfg); # permission and content type check
6710
6711             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6712         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6713             my $virtdev = $1;
6714             my $drive = parse_drive($virtdev, $2);
6715
6716             if (drive_is_cloudinit($drive)) {
6717                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6718                 $storeid = $options->{storage} if defined ($options->{storage});
6719                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6720                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6721
6722                 $check_storage->($storeid, $scfg); # permission and content type check
6723
6724                 $virtdev_hash->{$virtdev} = {
6725                     format => $format,
6726                     storeid => $storeid,
6727                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6728                     is_cloudinit => 1,
6729                 };
6730             }
6731         }
6732     }
6733
6734     return $virtdev_hash;
6735 };
6736
6737 # Helper to allocate and activate all volumes required for a restore
6738 #
6739 # $storecfg: Storage configuration
6740 # $virtdev_hash: as returned by parse_backup_hints()
6741 #
6742 # Returns: { $virtdev => $volid }
6743 my $restore_allocate_devices = sub {
6744     my ($storecfg, $virtdev_hash, $vmid) = @_;
6745
6746     my $map = {};
6747     foreach my $virtdev (sort keys %$virtdev_hash) {
6748         my $d = $virtdev_hash->{$virtdev};
6749         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6750         my $storeid = $d->{storeid};
6751         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6752
6753         # test if requested format is supported
6754         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6755         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6756         $d->{format} = $defFormat if !$supported;
6757
6758         my $name;
6759         if ($d->{is_cloudinit}) {
6760             $name = "vm-$vmid-cloudinit";
6761             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6762             if ($scfg->{path}) {
6763                 $name .= ".$d->{format}";
6764             }
6765         }
6766
6767         my $volid = PVE::Storage::vdisk_alloc(
6768             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6769
6770         print STDERR "new volume ID is '$volid'\n";
6771         $d->{volid} = $volid;
6772
6773         PVE::Storage::activate_volumes($storecfg, [$volid]);
6774
6775         $map->{$virtdev} = $volid;
6776     }
6777
6778     return $map;
6779 };
6780
6781 sub restore_update_config_line {
6782     my ($cookie, $map, $line, $unique) = @_;
6783
6784     return '' if $line =~ m/^\#qmdump\#/;
6785     return '' if $line =~ m/^\#vzdump\#/;
6786     return '' if $line =~ m/^lock:/;
6787     return '' if $line =~ m/^unused\d+:/;
6788     return '' if $line =~ m/^parent:/;
6789
6790     my $res = '';
6791
6792     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6793     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6794         # try to convert old 1.X settings
6795         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6796         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6797             my ($model, $macaddr) = split(/\=/, $devconfig);
6798             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6799             my $net = {
6800                 model => $model,
6801                 bridge => "vmbr$ind",
6802                 macaddr => $macaddr,
6803             };
6804             my $netstr = print_net($net);
6805
6806             $res .= "net$cookie->{netcount}: $netstr\n";
6807             $cookie->{netcount}++;
6808         }
6809     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6810         my ($id, $netstr) = ($1, $2);
6811         my $net = parse_net($netstr);
6812         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6813         $netstr = print_net($net);
6814         $res .= "$id: $netstr\n";
6815     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6816         my $virtdev = $1;
6817         my $value = $3;
6818         my $di = parse_drive($virtdev, $value);
6819         if (defined($di->{backup}) && !$di->{backup}) {
6820             $res .= "#$line";
6821         } elsif ($map->{$virtdev}) {
6822             delete $di->{format}; # format can change on restore
6823             $di->{file} = $map->{$virtdev};
6824             $value = print_drive($di);
6825             $res .= "$virtdev: $value\n";
6826         } else {
6827             $res .= $line;
6828         }
6829     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6830         my $vmgenid = $1;
6831         if ($vmgenid ne '0') {
6832             # always generate a new vmgenid if there was a valid one setup
6833             $vmgenid = generate_uuid();
6834         }
6835         $res .= "vmgenid: $vmgenid\n";
6836     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6837         my ($uuid, $uuid_str);
6838         UUID::generate($uuid);
6839         UUID::unparse($uuid, $uuid_str);
6840         my $smbios1 = parse_smbios1($2);
6841         $smbios1->{uuid} = $uuid_str;
6842         $res .= $1.print_smbios1($smbios1)."\n";
6843     } else {
6844         $res .= $line;
6845     }
6846
6847     return $res;
6848 }
6849
6850 my $restore_deactivate_volumes = sub {
6851     my ($storecfg, $virtdev_hash) = @_;
6852
6853     my $vollist = [];
6854     for my $dev (values $virtdev_hash->%*) {
6855         push $vollist->@*, $dev->{volid} if $dev->{volid};
6856     }
6857
6858     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6859     print STDERR $@ if $@;
6860 };
6861
6862 my $restore_destroy_volumes = sub {
6863     my ($storecfg, $virtdev_hash) = @_;
6864
6865     for my $dev (values $virtdev_hash->%*) {
6866         my $volid = $dev->{volid} or next;
6867         eval {
6868             PVE::Storage::vdisk_free($storecfg, $volid);
6869             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6870         };
6871         print STDERR "unable to cleanup '$volid' - $@" if $@;
6872     }
6873 };
6874
6875 sub restore_merge_config {
6876     my ($filename, $backup_conf_raw, $override_conf) = @_;
6877
6878     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6879     for my $key (keys $override_conf->%*) {
6880         $backup_conf->{$key} = $override_conf->{$key};
6881     }
6882
6883     return $backup_conf;
6884 }
6885
6886 sub scan_volids {
6887     my ($cfg, $vmid) = @_;
6888
6889     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6890
6891     my $volid_hash = {};
6892     foreach my $storeid (keys %$info) {
6893         foreach my $item (@{$info->{$storeid}}) {
6894             next if !($item->{volid} && $item->{size});
6895             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6896             $volid_hash->{$item->{volid}} = $item;
6897         }
6898     }
6899
6900     return $volid_hash;
6901 }
6902
6903 sub update_disk_config {
6904     my ($vmid, $conf, $volid_hash) = @_;
6905
6906     my $changes;
6907     my $prefix = "VM $vmid";
6908
6909     # used and unused disks
6910     my $referenced = {};
6911
6912     # Note: it is allowed to define multiple storages with same path (alias), so
6913     # we need to check both 'volid' and real 'path' (two different volid can point
6914     # to the same path).
6915
6916     my $referencedpath = {};
6917
6918     # update size info
6919     PVE::QemuConfig->foreach_volume($conf, sub {
6920         my ($opt, $drive) = @_;
6921
6922         my $volid = $drive->{file};
6923         return if !$volid;
6924         my $volume = $volid_hash->{$volid};
6925
6926         # mark volid as "in-use" for next step
6927         $referenced->{$volid} = 1;
6928         if ($volume && (my $path = $volume->{path})) {
6929             $referencedpath->{$path} = 1;
6930         }
6931
6932         return if drive_is_cdrom($drive);
6933         return if !$volume;
6934
6935         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6936         if (defined($updated)) {
6937             $changes = 1;
6938             $conf->{$opt} = print_drive($updated);
6939             print "$prefix ($opt): $msg\n";
6940         }
6941     });
6942
6943     # remove 'unusedX' entry if volume is used
6944     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6945         my ($opt, $drive) = @_;
6946
6947         my $volid = $drive->{file};
6948         return if !$volid;
6949
6950         my $path;
6951         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6952         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6953             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6954             $changes = 1;
6955             delete $conf->{$opt};
6956         }
6957
6958         $referenced->{$volid} = 1;
6959         $referencedpath->{$path} = 1 if $path;
6960     });
6961
6962     foreach my $volid (sort keys %$volid_hash) {
6963         next if $volid =~ m/vm-$vmid-state-/;
6964         next if $referenced->{$volid};
6965         my $path = $volid_hash->{$volid}->{path};
6966         next if !$path; # just to be sure
6967         next if $referencedpath->{$path};
6968         $changes = 1;
6969         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6970         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6971         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6972     }
6973
6974     return $changes;
6975 }
6976
6977 sub rescan {
6978     my ($vmid, $nolock, $dryrun) = @_;
6979
6980     my $cfg = PVE::Storage::config();
6981
6982     print "rescan volumes...\n";
6983     my $volid_hash = scan_volids($cfg, $vmid);
6984
6985     my $updatefn =  sub {
6986         my ($vmid) = @_;
6987
6988         my $conf = PVE::QemuConfig->load_config($vmid);
6989
6990         PVE::QemuConfig->check_lock($conf);
6991
6992         my $vm_volids = {};
6993         foreach my $volid (keys %$volid_hash) {
6994             my $info = $volid_hash->{$volid};
6995             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6996         }
6997
6998         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6999
7000         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
7001     };
7002
7003     if (defined($vmid)) {
7004         if ($nolock) {
7005             &$updatefn($vmid);
7006         } else {
7007             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7008         }
7009     } else {
7010         my $vmlist = config_list();
7011         foreach my $vmid (keys %$vmlist) {
7012             if ($nolock) {
7013                 &$updatefn($vmid);
7014             } else {
7015                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
7016             }
7017         }
7018     }
7019 }
7020
7021 sub restore_proxmox_backup_archive {
7022     my ($archive, $vmid, $user, $options) = @_;
7023
7024     my $storecfg = PVE::Storage::config();
7025
7026     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
7027     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7028
7029     my $fingerprint = $scfg->{fingerprint};
7030     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7031
7032     my $repo = PVE::PBSClient::get_repository($scfg);
7033     my $namespace = $scfg->{namespace};
7034
7035     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7036     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7037     local $ENV{PBS_PASSWORD} = $password;
7038     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7039
7040     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7041         PVE::Storage::parse_volname($storecfg, $archive);
7042
7043     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7044
7045     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7046
7047     my $tmpdir = "/var/tmp/vzdumptmp$$";
7048     rmtree $tmpdir;
7049     mkpath $tmpdir;
7050
7051     my $conffile = PVE::QemuConfig->config_file($vmid);
7052      # disable interrupts (always do cleanups)
7053     local $SIG{INT} =
7054         local $SIG{TERM} =
7055         local $SIG{QUIT} =
7056         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7057
7058     # Note: $oldconf is undef if VM does not exists
7059     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7060     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7061     my $new_conf_raw = '';
7062
7063     my $rpcenv = PVE::RPCEnvironment::get();
7064     my $devinfo = {}; # info about drives included in backup
7065     my $virtdev_hash = {}; # info about allocated drives
7066
7067     eval {
7068         # enable interrupts
7069         local $SIG{INT} =
7070             local $SIG{TERM} =
7071             local $SIG{QUIT} =
7072             local $SIG{HUP} =
7073             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7074
7075         my $cfgfn = "$tmpdir/qemu-server.conf";
7076         my $firewall_config_fn = "$tmpdir/fw.conf";
7077         my $index_fn = "$tmpdir/index.json";
7078
7079         my $cmd = "restore";
7080
7081         my $param = [$pbs_backup_name, "index.json", $index_fn];
7082         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7083         my $index = PVE::Tools::file_get_contents($index_fn);
7084         $index = decode_json($index);
7085
7086         foreach my $info (@{$index->{files}}) {
7087             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7088                 my $devname = $1;
7089                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7090                     $devinfo->{$devname}->{size} = $1;
7091                 } else {
7092                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7093                 }
7094             }
7095         }
7096
7097         my $is_qemu_server_backup = scalar(
7098             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7099         );
7100         if (!$is_qemu_server_backup) {
7101             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7102         }
7103         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7104
7105         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7106         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7107
7108         if ($has_firewall_config) {
7109             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7110             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7111
7112             my $pve_firewall_dir = '/etc/pve/firewall';
7113             mkdir $pve_firewall_dir; # make sure the dir exists
7114             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7115         }
7116
7117         my $fh = IO::File->new($cfgfn, "r") ||
7118             die "unable to read qemu-server.conf - $!\n";
7119
7120         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7121
7122         # fixme: rate limit?
7123
7124         # create empty/temp config
7125         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7126
7127         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7128
7129         # allocate volumes
7130         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7131
7132         foreach my $virtdev (sort keys %$virtdev_hash) {
7133             my $d = $virtdev_hash->{$virtdev};
7134             next if $d->{is_cloudinit}; # no need to restore cloudinit
7135
7136             # this fails if storage is unavailable
7137             my $volid = $d->{volid};
7138             my $path = PVE::Storage::path($storecfg, $volid);
7139
7140             # for live-restore we only want to preload the efidisk and TPM state
7141             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7142
7143             my @ns_arg;
7144             if (defined(my $ns = $scfg->{namespace})) {
7145                 @ns_arg = ('--ns', $ns);
7146             }
7147
7148             my $pbs_restore_cmd = [
7149                 '/usr/bin/pbs-restore',
7150                 '--repository', $repo,
7151                 @ns_arg,
7152                 $pbs_backup_name,
7153                 "$d->{devname}.img.fidx",
7154                 $path,
7155                 '--verbose',
7156                 ];
7157
7158             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7159             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7160
7161             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7162                 push @$pbs_restore_cmd, '--skip-zero';
7163             }
7164
7165             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7166             print "restore proxmox backup image: $dbg_cmdstring\n";
7167             run_command($pbs_restore_cmd);
7168         }
7169
7170         $fh->seek(0, 0) || die "seek failed - $!\n";
7171
7172         my $cookie = { netcount => 0 };
7173         while (defined(my $line = <$fh>)) {
7174             $new_conf_raw .= restore_update_config_line(
7175                 $cookie,
7176                 $map,
7177                 $line,
7178                 $options->{unique},
7179             );
7180         }
7181
7182         $fh->close();
7183     };
7184     my $err = $@;
7185
7186     if ($err || !$options->{live}) {
7187         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7188     }
7189
7190     rmtree $tmpdir;
7191
7192     if ($err) {
7193         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7194         die $err;
7195     }
7196
7197     if ($options->{live}) {
7198         # keep lock during live-restore
7199         $new_conf_raw .= "\nlock: create";
7200     }
7201
7202     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7203     check_restore_permissions($rpcenv, $user, $new_conf);
7204     PVE::QemuConfig->write_config($vmid, $new_conf);
7205
7206     eval { rescan($vmid, 1); };
7207     warn $@ if $@;
7208
7209     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7210
7211     if ($options->{live}) {
7212         # enable interrupts
7213         local $SIG{INT} =
7214             local $SIG{TERM} =
7215             local $SIG{QUIT} =
7216             local $SIG{HUP} =
7217             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7218
7219         my $conf = PVE::QemuConfig->load_config($vmid);
7220         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7221
7222         # these special drives are already restored before start
7223         delete $devinfo->{'drive-efidisk0'};
7224         delete $devinfo->{'drive-tpmstate0-backup'};
7225
7226         my $pbs_opts = {
7227             repo => $repo,
7228             keyfile => $keyfile,
7229             snapshot => $pbs_backup_name,
7230             namespace => $namespace,
7231         };
7232         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7233
7234         PVE::QemuConfig->remove_lock($vmid, "create");
7235     }
7236 }
7237
7238 sub pbs_live_restore {
7239     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7240
7241     print "starting VM for live-restore\n";
7242     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7243
7244     my $pbs_backing = {};
7245     for my $ds (keys %$restored_disks) {
7246         $ds =~ m/^drive-(.*)$/;
7247         my $confname = $1;
7248         $pbs_backing->{$confname} = {
7249             repository => $opts->{repo},
7250             snapshot => $opts->{snapshot},
7251             archive => "$ds.img.fidx",
7252         };
7253         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7254         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7255
7256         my $drive = parse_drive($confname, $conf->{$confname});
7257         print "restoring '$ds' to '$drive->{file}'\n";
7258     }
7259
7260     my $drives_streamed = 0;
7261     eval {
7262         # make sure HA doesn't interrupt our restore by stopping the VM
7263         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7264             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7265         }
7266
7267         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7268         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7269         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7270
7271         my $qmeventd_fd = register_qmeventd_handle($vmid);
7272
7273         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7274         # this will effectively collapse the backing image chain consisting of
7275         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7276         # removes itself once all backing images vanish with 'auto-remove=on')
7277         my $jobs = {};
7278         for my $ds (sort keys %$restored_disks) {
7279             my $job_id = "restore-$ds";
7280             mon_cmd($vmid, 'block-stream',
7281                 'job-id' => $job_id,
7282                 device => "$ds",
7283             );
7284             $jobs->{$job_id} = {};
7285         }
7286
7287         mon_cmd($vmid, 'cont');
7288         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7289
7290         print "restore-drive jobs finished successfully, removing all tracking block devices"
7291             ." to disconnect from Proxmox Backup Server\n";
7292
7293         for my $ds (sort keys %$restored_disks) {
7294             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7295         }
7296
7297         close($qmeventd_fd);
7298     };
7299
7300     my $err = $@;
7301
7302     if ($err) {
7303         warn "An error occurred during live-restore: $err\n";
7304         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7305         die "live-restore failed\n";
7306     }
7307 }
7308
7309 sub restore_vma_archive {
7310     my ($archive, $vmid, $user, $opts, $comp) = @_;
7311
7312     my $readfrom = $archive;
7313
7314     my $cfg = PVE::Storage::config();
7315     my $commands = [];
7316     my $bwlimit = $opts->{bwlimit};
7317
7318     my $dbg_cmdstring = '';
7319     my $add_pipe = sub {
7320         my ($cmd) = @_;
7321         push @$commands, $cmd;
7322         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7323         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7324         $readfrom = '-';
7325     };
7326
7327     my $input = undef;
7328     if ($archive eq '-') {
7329         $input = '<&STDIN';
7330     } else {
7331         # If we use a backup from a PVE defined storage we also consider that
7332         # storage's rate limit:
7333         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7334         if (defined($volid)) {
7335             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7336             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7337             if ($readlimit) {
7338                 print STDERR "applying read rate limit: $readlimit\n";
7339                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7340                 $add_pipe->($cstream);
7341             }
7342         }
7343     }
7344
7345     if ($comp) {
7346         my $info = PVE::Storage::decompressor_info('vma', $comp);
7347         my $cmd = $info->{decompressor};
7348         push @$cmd, $readfrom;
7349         $add_pipe->($cmd);
7350     }
7351
7352     my $tmpdir = "/var/tmp/vzdumptmp$$";
7353     rmtree $tmpdir;
7354
7355     # disable interrupts (always do cleanups)
7356     local $SIG{INT} =
7357         local $SIG{TERM} =
7358         local $SIG{QUIT} =
7359         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7360
7361     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7362     POSIX::mkfifo($mapfifo, 0600);
7363     my $fifofh;
7364     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7365
7366     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7367
7368     my $oldtimeout;
7369     my $timeout = 5;
7370
7371     my $devinfo = {}; # info about drives included in backup
7372     my $virtdev_hash = {}; # info about allocated drives
7373
7374     my $rpcenv = PVE::RPCEnvironment::get();
7375
7376     my $conffile = PVE::QemuConfig->config_file($vmid);
7377
7378     # Note: $oldconf is undef if VM does not exist
7379     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7380     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7381     my $new_conf_raw = '';
7382
7383     my %storage_limits;
7384
7385     my $print_devmap = sub {
7386         my $cfgfn = "$tmpdir/qemu-server.conf";
7387
7388         # we can read the config - that is already extracted
7389         my $fh = IO::File->new($cfgfn, "r") ||
7390             die "unable to read qemu-server.conf - $!\n";
7391
7392         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7393         if (-f $fwcfgfn) {
7394             my $pve_firewall_dir = '/etc/pve/firewall';
7395             mkdir $pve_firewall_dir; # make sure the dir exists
7396             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7397         }
7398
7399         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7400
7401         foreach my $info (values %{$virtdev_hash}) {
7402             my $storeid = $info->{storeid};
7403             next if defined($storage_limits{$storeid});
7404
7405             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7406             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7407             $storage_limits{$storeid} = $limit * 1024;
7408         }
7409
7410         foreach my $devname (keys %$devinfo) {
7411             die "found no device mapping information for device '$devname'\n"
7412                 if !$devinfo->{$devname}->{virtdev};
7413         }
7414
7415         # create empty/temp config
7416         if ($oldconf) {
7417             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7418             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7419         }
7420
7421         # allocate volumes
7422         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7423
7424         # print restore information to $fifofh
7425         foreach my $virtdev (sort keys %$virtdev_hash) {
7426             my $d = $virtdev_hash->{$virtdev};
7427             next if $d->{is_cloudinit}; # no need to restore cloudinit
7428
7429             my $storeid = $d->{storeid};
7430             my $volid = $d->{volid};
7431
7432             my $map_opts = '';
7433             if (my $limit = $storage_limits{$storeid}) {
7434                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7435             }
7436
7437             my $write_zeros = 1;
7438             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7439                 $write_zeros = 0;
7440             }
7441
7442             my $path = PVE::Storage::path($cfg, $volid);
7443
7444             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7445
7446             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7447         }
7448
7449         $fh->seek(0, 0) || die "seek failed - $!\n";
7450
7451         my $cookie = { netcount => 0 };
7452         while (defined(my $line = <$fh>)) {
7453             $new_conf_raw .= restore_update_config_line(
7454                 $cookie,
7455                 $map,
7456                 $line,
7457                 $opts->{unique},
7458             );
7459         }
7460
7461         $fh->close();
7462     };
7463
7464     eval {
7465         # enable interrupts
7466         local $SIG{INT} =
7467             local $SIG{TERM} =
7468             local $SIG{QUIT} =
7469             local $SIG{HUP} =
7470             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7471         local $SIG{ALRM} = sub { die "got timeout\n"; };
7472
7473         $oldtimeout = alarm($timeout);
7474
7475         my $parser = sub {
7476             my $line = shift;
7477
7478             print "$line\n";
7479
7480             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7481                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7482                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7483             } elsif ($line =~ m/^CTIME: /) {
7484                 # we correctly received the vma config, so we can disable
7485                 # the timeout now for disk allocation (set to 10 minutes, so
7486                 # that we always timeout if something goes wrong)
7487                 alarm(600);
7488                 &$print_devmap();
7489                 print $fifofh "done\n";
7490                 my $tmp = $oldtimeout || 0;
7491                 $oldtimeout = undef;
7492                 alarm($tmp);
7493                 close($fifofh);
7494                 $fifofh = undef;
7495             }
7496         };
7497
7498         print "restore vma archive: $dbg_cmdstring\n";
7499         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7500     };
7501     my $err = $@;
7502
7503     alarm($oldtimeout) if $oldtimeout;
7504
7505     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7506
7507     close($fifofh) if $fifofh;
7508     unlink $mapfifo;
7509     rmtree $tmpdir;
7510
7511     if ($err) {
7512         $restore_destroy_volumes->($cfg, $virtdev_hash);
7513         die $err;
7514     }
7515
7516     my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7517     check_restore_permissions($rpcenv, $user, $new_conf);
7518     PVE::QemuConfig->write_config($vmid, $new_conf);
7519
7520     eval { rescan($vmid, 1); };
7521     warn $@ if $@;
7522
7523     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7524 }
7525
7526 sub restore_tar_archive {
7527     my ($archive, $vmid, $user, $opts) = @_;
7528
7529     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7530         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7531         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7532     }
7533
7534     if ($archive ne '-') {
7535         my $firstfile = tar_archive_read_firstfile($archive);
7536         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7537             if $firstfile ne 'qemu-server.conf';
7538     }
7539
7540     my $storecfg = PVE::Storage::config();
7541
7542     # avoid zombie disks when restoring over an existing VM -> cleanup first
7543     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7544     # skiplock=1 because qmrestore has set the 'create' lock itself already
7545     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7546     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7547
7548     my $tocmd = "/usr/lib/qemu-server/qmextract";
7549
7550     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7551     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7552     $tocmd .= ' --prealloc' if $opts->{prealloc};
7553     $tocmd .= ' --info' if $opts->{info};
7554
7555     # tar option "xf" does not autodetect compression when read from STDIN,
7556     # so we pipe to zcat
7557     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7558         PVE::Tools::shellquote("--to-command=$tocmd");
7559
7560     my $tmpdir = "/var/tmp/vzdumptmp$$";
7561     mkpath $tmpdir;
7562
7563     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7564     local $ENV{VZDUMP_VMID} = $vmid;
7565     local $ENV{VZDUMP_USER} = $user;
7566
7567     my $conffile = PVE::QemuConfig->config_file($vmid);
7568     my $new_conf_raw = '';
7569
7570     # disable interrupts (always do cleanups)
7571     local $SIG{INT} =
7572         local $SIG{TERM} =
7573         local $SIG{QUIT} =
7574         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7575
7576     eval {
7577         # enable interrupts
7578         local $SIG{INT} =
7579             local $SIG{TERM} =
7580             local $SIG{QUIT} =
7581             local $SIG{HUP} =
7582             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7583
7584         if ($archive eq '-') {
7585             print "extracting archive from STDIN\n";
7586             run_command($cmd, input => "<&STDIN");
7587         } else {
7588             print "extracting archive '$archive'\n";
7589             run_command($cmd);
7590         }
7591
7592         return if $opts->{info};
7593
7594         # read new mapping
7595         my $map = {};
7596         my $statfile = "$tmpdir/qmrestore.stat";
7597         if (my $fd = IO::File->new($statfile, "r")) {
7598             while (defined (my $line = <$fd>)) {
7599                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7600                     $map->{$1} = $2 if $1;
7601                 } else {
7602                     print STDERR "unable to parse line in statfile - $line\n";
7603                 }
7604             }
7605             $fd->close();
7606         }
7607
7608         my $confsrc = "$tmpdir/qemu-server.conf";
7609
7610         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7611
7612         my $cookie = { netcount => 0 };
7613         while (defined (my $line = <$srcfd>)) {
7614             $new_conf_raw .= restore_update_config_line(
7615                 $cookie,
7616                 $map,
7617                 $line,
7618                 $opts->{unique},
7619             );
7620         }
7621
7622         $srcfd->close();
7623     };
7624     if (my $err = $@) {
7625         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7626         die $err;
7627     }
7628
7629     rmtree $tmpdir;
7630
7631     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7632
7633     PVE::Cluster::cfs_update(); # make sure we read new file
7634
7635     eval { rescan($vmid, 1); };
7636     warn $@ if $@;
7637 };
7638
7639 sub foreach_storage_used_by_vm {
7640     my ($conf, $func) = @_;
7641
7642     my $sidhash = {};
7643
7644     PVE::QemuConfig->foreach_volume($conf, sub {
7645         my ($ds, $drive) = @_;
7646         return if drive_is_cdrom($drive);
7647
7648         my $volid = $drive->{file};
7649
7650         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7651         $sidhash->{$sid} = $sid if $sid;
7652     });
7653
7654     foreach my $sid (sort keys %$sidhash) {
7655         &$func($sid);
7656     }
7657 }
7658
7659 my $qemu_snap_storage = {
7660     rbd => 1,
7661 };
7662 sub do_snapshots_with_qemu {
7663     my ($storecfg, $volid, $deviceid) = @_;
7664
7665     return if $deviceid =~ m/tpmstate0/;
7666
7667     my $storage_name = PVE::Storage::parse_volume_id($volid);
7668     my $scfg = $storecfg->{ids}->{$storage_name};
7669     die "could not find storage '$storage_name'\n" if !defined($scfg);
7670
7671     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7672         return 1;
7673     }
7674
7675     if ($volid =~ m/\.(qcow2|qed)$/){
7676         return 1;
7677     }
7678
7679     return;
7680 }
7681
7682 sub qga_check_running {
7683     my ($vmid, $nowarn) = @_;
7684
7685     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7686     if ($@) {
7687         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7688         return 0;
7689     }
7690     return 1;
7691 }
7692
7693 sub template_create {
7694     my ($vmid, $conf, $disk) = @_;
7695
7696     my $storecfg = PVE::Storage::config();
7697
7698     PVE::QemuConfig->foreach_volume($conf, sub {
7699         my ($ds, $drive) = @_;
7700
7701         return if drive_is_cdrom($drive);
7702         return if $disk && $ds ne $disk;
7703
7704         my $volid = $drive->{file};
7705         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7706
7707         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7708         $drive->{file} = $voliddst;
7709         $conf->{$ds} = print_drive($drive);
7710         PVE::QemuConfig->write_config($vmid, $conf);
7711     });
7712 }
7713
7714 sub convert_iscsi_path {
7715     my ($path) = @_;
7716
7717     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7718         my $portal = $1;
7719         my $target = $2;
7720         my $lun = $3;
7721
7722         my $initiator_name = get_initiator_name();
7723
7724         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7725                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7726     }
7727
7728     die "cannot convert iscsi path '$path', unkown format\n";
7729 }
7730
7731 sub qemu_img_convert {
7732     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7733
7734     my $storecfg = PVE::Storage::config();
7735     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7736     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7737
7738     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7739
7740     my $cachemode;
7741     my $src_path;
7742     my $src_is_iscsi = 0;
7743     my $src_format;
7744
7745     if ($src_storeid) {
7746         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7747         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7748         $src_format = qemu_img_format($src_scfg, $src_volname);
7749         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7750         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7751         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7752     } elsif (-f $src_volid || -b $src_volid) {
7753         $src_path = $src_volid;
7754         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7755             $src_format = $1;
7756         }
7757     }
7758
7759     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7760
7761     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7762     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7763     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7764     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7765
7766     my $cmd = [];
7767     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7768     push @$cmd, '-l', "snapshot.name=$snapname"
7769         if $snapname && $src_format && $src_format eq "qcow2";
7770     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7771     push @$cmd, '-T', $cachemode if defined($cachemode);
7772     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7773
7774     if ($src_is_iscsi) {
7775         push @$cmd, '--image-opts';
7776         $src_path = convert_iscsi_path($src_path);
7777     } elsif ($src_format) {
7778         push @$cmd, '-f', $src_format;
7779     }
7780
7781     if ($dst_is_iscsi) {
7782         push @$cmd, '--target-image-opts';
7783         $dst_path = convert_iscsi_path($dst_path);
7784     } else {
7785         push @$cmd, '-O', $dst_format;
7786     }
7787
7788     push @$cmd, $src_path;
7789
7790     if (!$dst_is_iscsi && $is_zero_initialized) {
7791         push @$cmd, "zeroinit:$dst_path";
7792     } else {
7793         push @$cmd, $dst_path;
7794     }
7795
7796     my $parser = sub {
7797         my $line = shift;
7798         if($line =~ m/\((\S+)\/100\%\)/){
7799             my $percent = $1;
7800             my $transferred = int($size * $percent / 100);
7801             my $total_h = render_bytes($size, 1);
7802             my $transferred_h = render_bytes($transferred, 1);
7803
7804             print "transferred $transferred_h of $total_h ($percent%)\n";
7805         }
7806
7807     };
7808
7809     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7810     my $err = $@;
7811     die "copy failed: $err" if $err;
7812 }
7813
7814 sub qemu_img_format {
7815     my ($scfg, $volname) = @_;
7816
7817     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7818         return $1;
7819     } else {
7820         return "raw";
7821     }
7822 }
7823
7824 sub qemu_drive_mirror {
7825     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7826
7827     $jobs = {} if !$jobs;
7828
7829     my $qemu_target;
7830     my $format;
7831     $jobs->{"drive-$drive"} = {};
7832
7833     if ($dst_volid =~ /^nbd:/) {
7834         $qemu_target = $dst_volid;
7835         $format = "nbd";
7836     } else {
7837         my $storecfg = PVE::Storage::config();
7838         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7839
7840         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7841
7842         $format = qemu_img_format($dst_scfg, $dst_volname);
7843
7844         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7845
7846         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7847     }
7848
7849     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7850     $opts->{format} = $format if $format;
7851
7852     if (defined($src_bitmap)) {
7853         $opts->{sync} = 'incremental';
7854         $opts->{bitmap} = $src_bitmap;
7855         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7856     }
7857
7858     if (defined($bwlimit)) {
7859         $opts->{speed} = $bwlimit * 1024;
7860         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7861     } else {
7862         print "drive mirror is starting for drive-$drive\n";
7863     }
7864
7865     # if a job already runs for this device we get an error, catch it for cleanup
7866     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7867     if (my $err = $@) {
7868         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7869         warn "$@\n" if $@;
7870         die "mirroring error: $err\n";
7871     }
7872
7873     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7874 }
7875
7876 # $completion can be either
7877 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7878 # 'cancel': wait until all jobs are ready, block-job-cancel them
7879 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7880 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7881 sub qemu_drive_mirror_monitor {
7882     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7883
7884     $completion //= 'complete';
7885     $op //= "mirror";
7886
7887     eval {
7888         my $err_complete = 0;
7889
7890         my $starttime = time ();
7891         while (1) {
7892             die "block job ('$op') timed out\n" if $err_complete > 300;
7893
7894             my $stats = mon_cmd($vmid, "query-block-jobs");
7895             my $ctime = time();
7896
7897             my $running_jobs = {};
7898             for my $stat (@$stats) {
7899                 next if $stat->{type} ne $op;
7900                 $running_jobs->{$stat->{device}} = $stat;
7901             }
7902
7903             my $readycounter = 0;
7904
7905             for my $job_id (sort keys %$jobs) {
7906                 my $job = $running_jobs->{$job_id};
7907
7908                 my $vanished = !defined($job);
7909                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7910                 if($complete || ($vanished && $completion eq 'auto')) {
7911                     print "$job_id: $op-job finished\n";
7912                     delete $jobs->{$job_id};
7913                     next;
7914                 }
7915
7916                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7917
7918                 my $busy = $job->{busy};
7919                 my $ready = $job->{ready};
7920                 if (my $total = $job->{len}) {
7921                     my $transferred = $job->{offset} || 0;
7922                     my $remaining = $total - $transferred;
7923                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7924
7925                     my $duration = $ctime - $starttime;
7926                     my $total_h = render_bytes($total, 1);
7927                     my $transferred_h = render_bytes($transferred, 1);
7928
7929                     my $status = sprintf(
7930                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7931                         render_duration($duration),
7932                     );
7933
7934                     if ($ready) {
7935                         if ($busy) {
7936                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7937                         } else {
7938                             $status .= ", ready";
7939                         }
7940                     }
7941                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7942                     $jobs->{$job_id}->{ready} = $ready;
7943                 }
7944
7945                 $readycounter++ if $job->{ready};
7946             }
7947
7948             last if scalar(keys %$jobs) == 0;
7949
7950             if ($readycounter == scalar(keys %$jobs)) {
7951                 print "all '$op' jobs are ready\n";
7952
7953                 # do the complete later (or has already been done)
7954                 last if $completion eq 'skip' || $completion eq 'auto';
7955
7956                 if ($vmiddst && $vmiddst != $vmid) {
7957                     my $agent_running = $qga && qga_check_running($vmid);
7958                     if ($agent_running) {
7959                         print "freeze filesystem\n";
7960                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7961                         warn $@ if $@;
7962                     } else {
7963                         print "suspend vm\n";
7964                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7965                         warn $@ if $@;
7966                     }
7967
7968                     # if we clone a disk for a new target vm, we don't switch the disk
7969                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7970
7971                     if ($agent_running) {
7972                         print "unfreeze filesystem\n";
7973                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7974                         warn $@ if $@;
7975                     } else {
7976                         print "resume vm\n";
7977                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7978                         warn $@ if $@;
7979                     }
7980
7981                     last;
7982                 } else {
7983
7984                     for my $job_id (sort keys %$jobs) {
7985                         # try to switch the disk if source and destination are on the same guest
7986                         print "$job_id: Completing block job_id...\n";
7987
7988                         my $op;
7989                         if ($completion eq 'complete') {
7990                             $op = 'block-job-complete';
7991                         } elsif ($completion eq 'cancel') {
7992                             $op = 'block-job-cancel';
7993                         } else {
7994                             die "invalid completion value: $completion\n";
7995                         }
7996                         eval { mon_cmd($vmid, $op, device => $job_id) };
7997                         if ($@ =~ m/cannot be completed/) {
7998                             print "$job_id: block job cannot be completed, trying again.\n";
7999                             $err_complete++;
8000                         }else {
8001                             print "$job_id: Completed successfully.\n";
8002                             $jobs->{$job_id}->{complete} = 1;
8003                         }
8004                     }
8005                 }
8006             }
8007             sleep 1;
8008         }
8009     };
8010     my $err = $@;
8011
8012     if ($err) {
8013         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
8014         die "block job ($op) error: $err";
8015     }
8016 }
8017
8018 sub qemu_blockjobs_cancel {
8019     my ($vmid, $jobs) = @_;
8020
8021     foreach my $job (keys %$jobs) {
8022         print "$job: Cancelling block job\n";
8023         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
8024         $jobs->{$job}->{cancel} = 1;
8025     }
8026
8027     while (1) {
8028         my $stats = mon_cmd($vmid, "query-block-jobs");
8029
8030         my $running_jobs = {};
8031         foreach my $stat (@$stats) {
8032             $running_jobs->{$stat->{device}} = $stat;
8033         }
8034
8035         foreach my $job (keys %$jobs) {
8036
8037             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8038                 print "$job: Done.\n";
8039                 delete $jobs->{$job};
8040             }
8041         }
8042
8043         last if scalar(keys %$jobs) == 0;
8044
8045         sleep 1;
8046     }
8047 }
8048
8049 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8050 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8051 my sub clone_disk_check_io_uring {
8052     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8053
8054     return if !$use_drive_mirror;
8055
8056     # Don't complain when not changing storage.
8057     # Assume if it works for the source, it'll work for the target too.
8058     return if $src_storeid eq $dst_storeid;
8059
8060     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8061     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8062
8063     my $cache_direct = drive_uses_cache_direct($src_drive);
8064
8065     my $src_uses_io_uring;
8066     if ($src_drive->{aio}) {
8067         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8068     } else {
8069         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8070     }
8071
8072     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8073         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8074 }
8075
8076 sub clone_disk {
8077     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8078
8079     my ($vmid, $running) = $source->@{qw(vmid running)};
8080     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8081
8082     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8083     my ($storage, $format) = $dest->@{qw(storage format)};
8084
8085     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8086
8087     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8088         die "cloning from/to EFI disk requires EFI disk\n"
8089             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8090         die "cloning from/to TPM state requires TPM state\n"
8091             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8092
8093         # This would lead to two device nodes in QEMU pointing to the same backing image!
8094         die "cannot change drive name when cloning disk from/to the same VM\n"
8095             if $use_drive_mirror && $vmid == $newvmid;
8096     }
8097
8098     die "cannot move TPM state while VM is running\n"
8099         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8100
8101     my $newvolid;
8102
8103     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8104     print "$src_drivename " if $src_drivename;
8105     print "($drive->{file})\n";
8106
8107     if (!$full) {
8108         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8109         push @$newvollist, $newvolid;
8110     } else {
8111         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8112         my $storeid = $storage || $src_storeid;
8113
8114         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8115
8116         my $name = undef;
8117         my $size = undef;
8118         if (drive_is_cloudinit($drive)) {
8119             $name = "vm-$newvmid-cloudinit";
8120             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8121             if ($scfg->{path}) {
8122                 $name .= ".$dst_format";
8123             }
8124             $snapname = undef;
8125             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8126         } elsif ($dst_drivename eq 'efidisk0') {
8127             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8128         } elsif ($dst_drivename eq 'tpmstate0') {
8129             $dst_format = 'raw';
8130             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8131         } else {
8132             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8133
8134             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8135         }
8136         $newvolid = PVE::Storage::vdisk_alloc(
8137             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8138         );
8139         push @$newvollist, $newvolid;
8140
8141         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8142
8143         if (drive_is_cloudinit($drive)) {
8144             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8145             # if this is the case, we have to complete any block-jobs still there from
8146             # previous drive-mirrors
8147             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8148                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8149             }
8150             goto no_data_clone;
8151         }
8152
8153         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8154         if ($use_drive_mirror) {
8155             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8156                 $completion, $qga, $bwlimit);
8157         } else {
8158             if ($dst_drivename eq 'efidisk0') {
8159                 # the relevant data on the efidisk may be smaller than the source
8160                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8161                 # that is given by the OVMF_VARS.fd
8162                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8163                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8164
8165                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8166
8167                 # better for Ceph if block size is not too small, see bug #3324
8168                 my $bs = 1024*1024;
8169
8170                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8171
8172                 if ($src_format eq 'qcow2' && $snapname) {
8173                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8174                         if !min_version(kvm_user_version(), 6, 2);
8175                     push $cmd->@*, '-l', $snapname;
8176                 }
8177                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8178                 run_command($cmd);
8179             } else {
8180                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8181             }
8182         }
8183     }
8184
8185 no_data_clone:
8186     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8187
8188     my $disk = dclone($drive);
8189     delete $disk->{format};
8190     $disk->{file} = $newvolid;
8191     $disk->{size} = $size if defined($size);
8192
8193     return $disk;
8194 }
8195
8196 sub get_running_qemu_version {
8197     my ($vmid) = @_;
8198     my $res = mon_cmd($vmid, "query-version");
8199     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8200 }
8201
8202 sub qemu_use_old_bios_files {
8203     my ($machine_type) = @_;
8204
8205     return if !$machine_type;
8206
8207     my $use_old_bios_files = undef;
8208
8209     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8210         $machine_type = $1;
8211         $use_old_bios_files = 1;
8212     } else {
8213         my $version = extract_version($machine_type, kvm_user_version());
8214         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8215         # load new efi bios files on migration. So this hack is required to allow
8216         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8217         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8218         $use_old_bios_files = !min_version($version, 2, 4);
8219     }
8220
8221     return ($use_old_bios_files, $machine_type);
8222 }
8223
8224 sub get_efivars_size {
8225     my ($conf, $efidisk) = @_;
8226
8227     my $arch = get_vm_arch($conf);
8228     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8229     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8230     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8231     return -s $ovmf_vars;
8232 }
8233
8234 sub update_efidisk_size {
8235     my ($conf) = @_;
8236
8237     return if !defined($conf->{efidisk0});
8238
8239     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8240     $disk->{size} = get_efivars_size($conf);
8241     $conf->{efidisk0} = print_drive($disk);
8242
8243     return;
8244 }
8245
8246 sub update_tpmstate_size {
8247     my ($conf) = @_;
8248
8249     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8250     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8251     $conf->{tpmstate0} = print_drive($disk);
8252 }
8253
8254 sub create_efidisk($$$$$$$) {
8255     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8256
8257     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8258
8259     my $vars_size_b = -s $ovmf_vars;
8260     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8261     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8262     PVE::Storage::activate_volumes($storecfg, [$volid]);
8263
8264     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8265     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8266
8267     return ($volid, $size/1024);
8268 }
8269
8270 sub vm_iothreads_list {
8271     my ($vmid) = @_;
8272
8273     my $res = mon_cmd($vmid, 'query-iothreads');
8274
8275     my $iothreads = {};
8276     foreach my $iothread (@$res) {
8277         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8278     }
8279
8280     return $iothreads;
8281 }
8282
8283 sub scsihw_infos {
8284     my ($conf, $drive) = @_;
8285
8286     my $maxdev = 0;
8287
8288     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8289         $maxdev = 7;
8290     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8291         $maxdev = 1;
8292     } else {
8293         $maxdev = 256;
8294     }
8295
8296     my $controller = int($drive->{index} / $maxdev);
8297     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8298         ? "virtioscsi"
8299         : "scsihw";
8300
8301     return ($maxdev, $controller, $controller_prefix);
8302 }
8303
8304 sub resolve_dst_disk_format {
8305         my ($storecfg, $storeid, $src_volname, $format) = @_;
8306         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8307
8308         if (!$format) {
8309             # if no target format is specified, use the source disk format as hint
8310             if ($src_volname) {
8311                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8312                 $format = qemu_img_format($scfg, $src_volname);
8313             } else {
8314                 return $defFormat;
8315             }
8316         }
8317
8318         # test if requested format is supported - else use default
8319         my $supported = grep { $_ eq $format } @$validFormats;
8320         $format = $defFormat if !$supported;
8321         return $format;
8322 }
8323
8324 # NOTE: if this logic changes, please update docs & possibly gui logic
8325 sub find_vmstate_storage {
8326     my ($conf, $storecfg) = @_;
8327
8328     # first, return storage from conf if set
8329     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8330
8331     my ($target, $shared, $local);
8332
8333     foreach_storage_used_by_vm($conf, sub {
8334         my ($sid) = @_;
8335         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8336         my $dst = $scfg->{shared} ? \$shared : \$local;
8337         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8338     });
8339
8340     # second, use shared storage where VM has at least one disk
8341     # third, use local storage where VM has at least one disk
8342     # fall back to local storage
8343     $target = $shared // $local // 'local';
8344
8345     return $target;
8346 }
8347
8348 sub generate_uuid {
8349     my ($uuid, $uuid_str);
8350     UUID::generate($uuid);
8351     UUID::unparse($uuid, $uuid_str);
8352     return $uuid_str;
8353 }
8354
8355 sub generate_smbios1_uuid {
8356     return "uuid=".generate_uuid();
8357 }
8358
8359 sub nbd_stop {
8360     my ($vmid) = @_;
8361
8362     mon_cmd($vmid, 'nbd-server-stop');
8363 }
8364
8365 sub create_reboot_request {
8366     my ($vmid) = @_;
8367     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8368         or die "failed to create reboot trigger file: $!\n";
8369     close($fh);
8370 }
8371
8372 sub clear_reboot_request {
8373     my ($vmid) = @_;
8374     my $path = "/run/qemu-server/$vmid.reboot";
8375     my $res = 0;
8376
8377     $res = unlink($path);
8378     die "could not remove reboot request for $vmid: $!"
8379         if !$res && $! != POSIX::ENOENT;
8380
8381     return $res;
8382 }
8383
8384 sub bootorder_from_legacy {
8385     my ($conf, $bootcfg) = @_;
8386
8387     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8388     my $bootindex_hash = {};
8389     my $i = 1;
8390     foreach my $o (split(//, $boot)) {
8391         $bootindex_hash->{$o} = $i*100;
8392         $i++;
8393     }
8394
8395     my $bootorder = {};
8396
8397     PVE::QemuConfig->foreach_volume($conf, sub {
8398         my ($ds, $drive) = @_;
8399
8400         if (drive_is_cdrom ($drive, 1)) {
8401             if ($bootindex_hash->{d}) {
8402                 $bootorder->{$ds} = $bootindex_hash->{d};
8403                 $bootindex_hash->{d} += 1;
8404             }
8405         } elsif ($bootindex_hash->{c}) {
8406             $bootorder->{$ds} = $bootindex_hash->{c}
8407                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8408             $bootindex_hash->{c} += 1;
8409         }
8410     });
8411
8412     if ($bootindex_hash->{n}) {
8413         for (my $i = 0; $i < $MAX_NETS; $i++) {
8414             my $netname = "net$i";
8415             next if !$conf->{$netname};
8416             $bootorder->{$netname} = $bootindex_hash->{n};
8417             $bootindex_hash->{n} += 1;
8418         }
8419     }
8420
8421     return $bootorder;
8422 }
8423
8424 # Generate default device list for 'boot: order=' property. Matches legacy
8425 # default boot order, but with explicit device names. This is important, since
8426 # the fallback for when neither 'order' nor the old format is specified relies
8427 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8428 sub get_default_bootdevices {
8429     my ($conf) = @_;
8430
8431     my @ret = ();
8432
8433     # harddisk
8434     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8435     push @ret, $first if $first;
8436
8437     # cdrom
8438     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8439     push @ret, $first if $first;
8440
8441     # network
8442     for (my $i = 0; $i < $MAX_NETS; $i++) {
8443         my $netname = "net$i";
8444         next if !$conf->{$netname};
8445         push @ret, $netname;
8446         last;
8447     }
8448
8449     return \@ret;
8450 }
8451
8452 sub device_bootorder {
8453     my ($conf) = @_;
8454
8455     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8456
8457     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8458
8459     my $bootorder = {};
8460     if (!defined($boot) || $boot->{legacy}) {
8461         $bootorder = bootorder_from_legacy($conf, $boot);
8462     } elsif ($boot->{order}) {
8463         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8464         for my $dev (PVE::Tools::split_list($boot->{order})) {
8465             $bootorder->{$dev} = $i++;
8466         }
8467     }
8468
8469     return $bootorder;
8470 }
8471
8472 sub register_qmeventd_handle {
8473     my ($vmid) = @_;
8474
8475     my $fh;
8476     my $peer = "/var/run/qmeventd.sock";
8477     my $count = 0;
8478
8479     for (;;) {
8480         $count++;
8481         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8482         last if $fh;
8483         if ($! != EINTR && $! != EAGAIN) {
8484             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8485         }
8486         if ($count > 4) {
8487             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8488               . "after $count retries\n";
8489         }
8490         usleep(25000);
8491     }
8492
8493     # send handshake to mark VM as backing up
8494     print $fh to_json({vzdump => {vmid => "$vmid"}});
8495
8496     # return handle to be closed later when inhibit is no longer required
8497     return $fh;
8498 }
8499
8500 # bash completion helper
8501
8502 sub complete_backup_archives {
8503     my ($cmdname, $pname, $cvalue) = @_;
8504
8505     my $cfg = PVE::Storage::config();
8506
8507     my $storeid;
8508
8509     if ($cvalue =~ m/^([^:]+):/) {
8510         $storeid = $1;
8511     }
8512
8513     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8514
8515     my $res = [];
8516     foreach my $id (keys %$data) {
8517         foreach my $item (@{$data->{$id}}) {
8518             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8519             push @$res, $item->{volid} if defined($item->{volid});
8520         }
8521     }
8522
8523     return $res;
8524 }
8525
8526 my $complete_vmid_full = sub {
8527     my ($running) = @_;
8528
8529     my $idlist = vmstatus();
8530
8531     my $res = [];
8532
8533     foreach my $id (keys %$idlist) {
8534         my $d = $idlist->{$id};
8535         if (defined($running)) {
8536             next if $d->{template};
8537             next if $running && $d->{status} ne 'running';
8538             next if !$running && $d->{status} eq 'running';
8539         }
8540         push @$res, $id;
8541
8542     }
8543     return $res;
8544 };
8545
8546 sub complete_vmid {
8547     return &$complete_vmid_full();
8548 }
8549
8550 sub complete_vmid_stopped {
8551     return &$complete_vmid_full(0);
8552 }
8553
8554 sub complete_vmid_running {
8555     return &$complete_vmid_full(1);
8556 }
8557
8558 sub complete_storage {
8559
8560     my $cfg = PVE::Storage::config();
8561     my $ids = $cfg->{ids};
8562
8563     my $res = [];
8564     foreach my $sid (keys %$ids) {
8565         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8566         next if !$ids->{$sid}->{content}->{images};
8567         push @$res, $sid;
8568     }
8569
8570     return $res;
8571 }
8572
8573 sub complete_migration_storage {
8574     my ($cmd, $param, $current_value, $all_args) = @_;
8575
8576     my $targetnode = @$all_args[1];
8577
8578     my $cfg = PVE::Storage::config();
8579     my $ids = $cfg->{ids};
8580
8581     my $res = [];
8582     foreach my $sid (keys %$ids) {
8583         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8584         next if !$ids->{$sid}->{content}->{images};
8585         push @$res, $sid;
8586     }
8587
8588     return $res;
8589 }
8590
8591 sub vm_is_paused {
8592     my ($vmid) = @_;
8593     my $qmpstatus = eval {
8594         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8595         mon_cmd($vmid, "query-status");
8596     };
8597     warn "$@\n" if $@;
8598     return $qmpstatus && $qmpstatus->{status} eq "paused";
8599 }
8600
8601 sub check_volume_storage_type {
8602     my ($storecfg, $vol) = @_;
8603
8604     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8605     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8606     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8607
8608     die "storage '$storeid' does not support content-type '$vtype'\n"
8609         if !$scfg->{content}->{$vtype};
8610
8611     return 1;
8612 }
8613
8614 sub add_nets_bridge_fdb {
8615     my ($conf, $vmid) = @_;
8616
8617     for my $opt (keys %$conf) {
8618         next if $opt !~ m/^net(\d+)$/;
8619         my $iface = "tap${vmid}i$1";
8620         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8621         my $net = parse_net($conf->{$opt}, 1) or next;
8622
8623         my $mac = $net->{macaddr};
8624         if (!$mac) {
8625             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8626                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8627             next;
8628         }
8629
8630         my $bridge = $net->{bridge};
8631         if (!$bridge) {
8632             log_warn("Interface '$iface' not attached to any bridge.");
8633             next;
8634         }
8635         if ($have_sdn) {
8636             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8637         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8638             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8639         }
8640     }
8641 }
8642
8643 sub del_nets_bridge_fdb {
8644     my ($conf, $vmid) = @_;
8645
8646     for my $opt (keys %$conf) {
8647         next if $opt !~ m/^net(\d+)$/;
8648         my $iface = "tap${vmid}i$1";
8649
8650         my $net = parse_net($conf->{$opt}) or next;
8651         my $mac = $net->{macaddr} or next;
8652
8653         my $bridge = $net->{bridge};
8654         if ($have_sdn) {
8655             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8656         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8657             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8658         }
8659     }
8660 }
8661
8662 1;