PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196 };
 197
 198 my $ivshmem_fmt = {
 199     size => {
 200         type => 'integer',
 201         minimum => 1,
 202         description => "The size of the file in MB.",
 203     },
 204     name => {
 205         type => 'string',
 206         pattern => '[a-zA-Z0-9\-]+',
 207         optional => 1,
 208         format_description => 'string',
 209         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 210     },
 211 };
 212
 213 my $audio_fmt = {
 214     device => {
 215         type => 'string',
 216         enum => [qw(ich9-intel-hda intel-hda AC97)],
 217         description =>  "Configure an audio device."
 218     },
 219     driver =>  {
 220         type => 'string',
 221         enum => ['spice', 'none'],
 222         default => 'spice',
 223         optional => 1,
 224         description => "Driver backend for the audio device."
 225     },
 226 };
 227
 228 my $spice_enhancements_fmt = {
 229     foldersharing => {
 230         type => 'boolean',
 231         optional => 1,
 232         default => '0',
 233         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 234     },
 235     videostreaming =>  {
 236         type => 'string',
 237         enum => ['off', 'all', 'filter'],
 238         default => 'off',
 239         optional => 1,
 240         description => "Enable video streaming. Uses compression for detected video streams."
 241     },
 242 };
 243
 244 my $rng_fmt = {
 245     source => {
 246         type => 'string',
 247         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 248         default_key => 1,
 249         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 250             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 251             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 252             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 253             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 254             ." a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 259             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 260             ." `0` to disable limiting (potentially dangerous!).",
 261         optional => 1,
 262
 263         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 264         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 265         # reading from /dev/urandom
 266         default => 1024,
 267     },
 268     period => {
 269         type => 'integer',
 270         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 271             ." the guest to retrieve another 'max_bytes' of entropy.",
 272         optional => 1,
 273         default => 1000,
 274     },
 275 };
 276
 277 my $meta_info_fmt = {
 278     'ctime' => {
 279         type => 'integer',
 280         description => "The guest creation timestamp as UNIX epoch time",
 281         minimum => 0,
 282         optional => 1,
 283     },
 284     'creation-qemu' => {
 285         type => 'string',
 286         description => "The QEMU (machine) version from the time this VM was created.",
 287         pattern => '\d+(\.\d+)+',
 288         optional => 1,
 289     },
 290 };
 291
 292 my $confdesc = {
 293     onboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Specifies whether a VM will be started during system bootup.",
 297         default => 0,
 298     },
 299     autostart => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Automatic restart after crash (currently ignored).",
 303         default => 0,
 304     },
 305     hotplug => {
 306         optional => 1,
 307         type => 'string', format => 'pve-hotplug-features',
 308         description => "Selectively enable hotplug features. This is a comma separated list of"
 309             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 310             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 311             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 312             ." windows > 7.",
 313         default => 'network,disk,usb',
 314     },
 315     reboot => {
 316         optional => 1,
 317         type => 'boolean',
 318         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 319         default => 1,
 320     },
 321     lock => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Lock/unlock the VM.",
 325         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 326     },
 327     cpulimit => {
 328         optional => 1,
 329         type => 'number',
 330         description => "Limit of CPU usage.",
 331         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 332             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 333         minimum => 0,
 334         maximum => 128,
 335         default => 0,
 336     },
 337     cpuunits => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 341         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 342             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 343             ." weights of all the other running VMs.",
 344         minimum => 1,
 345         maximum => 262144,
 346         default => 'cgroup v1: 1024, cgroup v2: 100',
 347     },
 348     memory => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 352             ." you use the balloon device.",
 353         minimum => 16,
 354         default => 512,
 355     },
 356     balloon => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 360         minimum => 0,
 361     },
 362     shares => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 366             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 367             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 368         minimum => 0,
 369         maximum => 50000,
 370         default => 1000,
 371     },
 372     keyboard => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Keyboard layout for VNC server. This option is generally not required and"
 376         ." is often better handled from within the guest OS.",
 377         enum => PVE::Tools::kvmkeymaplist(),
 378         default => undef,
 379     },
 380     name => {
 381         optional => 1,
 382         type => 'string', format => 'dns-name',
 383         description => "Set a name for the VM. Only used on the configuration web interface.",
 384     },
 385     scsihw => {
 386         optional => 1,
 387         type => 'string',
 388         description => "SCSI controller model",
 389         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 390         default => 'lsi',
 391     },
 392     description => {
 393         optional => 1,
 394         type => 'string',
 395         description => "Description for the VM. Shown in the web-interface VM's summary."
 396             ." This is saved as comment inside the configuration file.",
 397         maxLength => 1024 * 8,
 398     },
 399     ostype => {
 400         optional => 1,
 401         type => 'string',
 402         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 403         description => "Specify guest operating system.",
 404         verbose_description => <<EODESC,
 405 Specify guest operating system. This is used to enable special
 406 optimization/features for specific operating systems:
 407
 408 [horizontal]
 409 other;; unspecified OS
 410 wxp;; Microsoft Windows XP
 411 w2k;; Microsoft Windows 2000
 412 w2k3;; Microsoft Windows 2003
 413 w2k8;; Microsoft Windows 2008
 414 wvista;; Microsoft Windows Vista
 415 win7;; Microsoft Windows 7
 416 win8;; Microsoft Windows 8/2012/2012r2
 417 win10;; Microsoft Windows 10/2016/2019
 418 win11;; Microsoft Windows 11/2022
 419 l24;; Linux 2.4 Kernel
 420 l26;; Linux 2.6 - 6.X Kernel
 421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 422 EODESC
 423     },
 424     boot => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-boot',
 427         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 428             ." key or 'legacy=' is deprecated.",
 429     },
 430     bootdisk => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-bootdisk',
 433         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 434         pattern => '(ide|sata|scsi|virtio)\d+',
 435     },
 436     smp => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPUs. Please use option -sockets instead.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     sockets => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of CPU sockets.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     cores => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of cores per socket.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     numa => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable NUMA.",
 461         default => 0,
 462     },
 463     hugepages => {
 464         optional => 1,
 465         type => 'string',
 466         description => "Enable/disable hugepages memory.",
 467         enum => [qw(any 2 1024)],
 468     },
 469     keephugepages => {
 470         optional => 1,
 471         type => 'boolean',
 472         default => 0,
 473         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 474             ." after VM shutdown and can be used for subsequent starts.",
 475     },
 476     vcpus => {
 477         optional => 1,
 478         type => 'integer',
 479         description => "Number of hotplugged vcpus.",
 480         minimum => 1,
 481         default => 0,
 482     },
 483     acpi => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Enable/disable ACPI.",
 487         default => 1,
 488     },
 489     agent => {
 490         optional => 1,
 491         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 492         type => 'string',
 493         format => $agent_fmt,
 494     },
 495     kvm => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable KVM hardware virtualization.",
 499         default => 1,
 500     },
 501     tdf => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable time drift fix.",
 505         default => 0,
 506     },
 507     localtime => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 511             ." the `ostype` indicates a Microsoft Windows OS.",
 512     },
 513     freeze => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 517     },
 518     vga => {
 519         optional => 1,
 520         type => 'string', format => $vga_fmt,
 521         description => "Configure the VGA hardware.",
 522         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 523             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 524             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 525             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 526             ." display server. For win* OS you can select how many independent displays you want,"
 527             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 528             ." using a serial device as terminal.",
 529     },
 530     watchdog => {
 531         optional => 1,
 532         type => 'string', format => 'pve-qm-watchdog',
 533         description => "Create a virtual hardware watchdog device.",
 534         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 535             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 536             ." else the watchdog will reset the guest (or execute the respective action specified)",
 537     },
 538     startdate => {
 539         optional => 1,
 540         type => 'string',
 541         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 542         description => "Set the initial date of the real time clock. Valid format for date are:"
 543             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 544         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 545         default => 'now',
 546     },
 547     startup =>  get_standard_option('pve-startup-order'),
 548     template => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Enable/disable Template.",
 552         default => 0,
 553     },
 554     args => {
 555         optional => 1,
 556         type => 'string',
 557         description => "Arbitrary arguments passed to kvm.",
 558         verbose_description => <<EODESCR,
 559 Arbitrary arguments passed to kvm, for example:
 560
 561 args: -no-reboot -no-hpet
 562
 563 NOTE: this option is for experts only.
 564 EODESCR
 565     },
 566     tablet => {
 567         optional => 1,
 568         type => 'boolean',
 569         default => 1,
 570         description => "Enable/disable the USB tablet device.",
 571         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 572             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 573             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 574             ." may consider disabling this to save some context switches. This is turned off by"
 575             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 576     },
 577     migrate_speed => {
 578         optional => 1,
 579         type => 'integer',
 580         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 581         minimum => 0,
 582         default => 0,
 583     },
 584     migrate_downtime => {
 585         optional => 1,
 586         type => 'number',
 587         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 588         minimum => 0,
 589         default => 0.1,
 590     },
 591     cdrom => {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-ide',
 594         typetext => '<volume>',
 595         description => "This is an alias for option -ide2",
 596     },
 597     cpu => {
 598         optional => 1,
 599         description => "Emulated CPU type.",
 600         type => 'string',
 601         format => 'pve-vm-cpu-conf',
 602     },
 603     parent => get_standard_option('pve-snapshot-name', {
 604         optional => 1,
 605         description => "Parent snapshot name. This is used internally, and should not be modified.",
 606     }),
 607     snaptime => {
 608         optional => 1,
 609         description => "Timestamp for snapshots.",
 610         type => 'integer',
 611         minimum => 0,
 612     },
 613     vmstate => {
 614         optional => 1,
 615         type => 'string', format => 'pve-volume-id',
 616         description => "Reference to a volume which stores the VM state. This is used internally"
 617             ." for snapshots.",
 618     },
 619     vmstatestorage => get_standard_option('pve-storage-id', {
 620         description => "Default storage for VM state volumes/files.",
 621         optional => 1,
 622     }),
 623     runningmachine => get_standard_option('pve-qemu-machine', {
 624         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 625             ." for snapshots.",
 626     }),
 627     runningcpu => {
 628         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 629             ." internally for snapshots.",
 630         optional => 1,
 631         type => 'string',
 632         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 633         format_description => 'QEMU -cpu parameter'
 634     },
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 652             ." remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 667             ." to disable explicitly.",
 668         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 669             ." value identifier to the guest OS. This allows to notify the guest operating system"
 670             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 671             ." execution or creation from a template). The guest operating system notices the"
 672             ." change, and is then able to react as appropriate by marking its copies of"
 673             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 674             ."Note that auto-creation only works when done through API/CLI create or update methods"
 675             .", but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 689             ." the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704     tags => {
 705         type => 'string', format => 'pve-tag-list',
 706         description => 'Tags of the VM. This is only meta information.',
 707         optional => 1,
 708     },
 709     rng0 => {
 710         type => 'string',
 711         format => $rng_fmt,
 712         description => "Configure a VirtIO-based Random Number Generator.",
 713         optional => 1,
 714     },
 715     meta => {
 716         type => 'string',
 717         format => $meta_info_fmt,
 718         description => "Some (read-only) meta-information about this guest.",
 719         optional => 1,
 720     },
 721     affinity => {
 722         type => 'string', format => 'pve-cpuset',
 723         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 724         optional => 1,
 725     },
 726 };
 727
 728 my $cicustom_fmt = {
 729     meta => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all meta data passed to the VM via"
 733             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     network => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     user => {
 745         type => 'string',
 746         optional => 1,
 747         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 748         format => 'pve-volume-id',
 749         format_description => 'volume',
 750     },
 751     vendor => {
 752         type => 'string',
 753         optional => 1,
 754         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 760
 761 my $confdesc_cloudinit = {
 762     citype => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'Specifies the cloud-init configuration format. The default depends on the'
 766             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 767             .' and `configdrive2` for windows.',
 768         enum => ['configdrive2', 'nocloud', 'opennebula'],
 769     },
 770     ciuser => {
 771         optional => 1,
 772         type => 'string',
 773         description => "cloud-init: User name to change ssh keys and password for instead of the"
 774             ." image's configured default user.",
 775     },
 776     cipassword => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Password to assign the user. Using this is generally not'
 780             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 781             .' support hashed passwords.',
 782     },
 783     cicustom => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Specify custom files to replace the automatically generated'
 787             .' ones at start.',
 788         format => 'pve-qm-cicustom',
 789     },
 790     searchdomain => {
 791         optional => 1,
 792         type => 'string',
 793         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     nameserver => {
 798         optional => 1,
 799         type => 'string', format => 'address-list',
 800         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 801             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 802             .' are set.',
 803     },
 804     sshkeys => {
 805         optional => 1,
 806         type => 'string',
 807         format => 'urlencoded',
 808         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 809     },
 810 };
 811
 812 # what about other qemu settings ?
 813 #cpu => 'string',
 814 #machine => 'string',
 815 #fda => 'file',
 816 #fdb => 'file',
 817 #mtdblock => 'file',
 818 #sd => 'file',
 819 #pflash => 'file',
 820 #snapshot => 'bool',
 821 #bootp => 'file',
 822 ##tftp => 'dir',
 823 ##smb => 'dir',
 824 #kernel => 'file',
 825 #append => 'string',
 826 #initrd => 'file',
 827 ##soundhw => 'string',
 828
 829 while (my ($k, $v) = each %$confdesc) {
 830     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 831 }
 832
 833 my $MAX_USB_DEVICES = 14;
 834 my $MAX_NETS = 32;
 835 my $MAX_SERIAL_PORTS = 4;
 836 my $MAX_PARALLEL_PORTS = 3;
 837 my $MAX_NUMA = 8;
 838
 839 my $numa_fmt = {
 840     cpus => {
 841         type => "string",
 842         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 843         description => "CPUs accessing this NUMA node.",
 844         format_description => "id[-id];...",
 845     },
 846     memory => {
 847         type => "number",
 848         description => "Amount of memory this NUMA node provides.",
 849         optional => 1,
 850     },
 851     hostnodes => {
 852         type => "string",
 853         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 854         description => "Host NUMA nodes to use.",
 855         format_description => "id[-id];...",
 856         optional => 1,
 857     },
 858     policy => {
 859         type => 'string',
 860         enum => [qw(preferred bind interleave)],
 861         description => "NUMA allocation policy.",
 862         optional => 1,
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 866 my $numadesc = {
 867     optional => 1,
 868     type => 'string', format => $numa_fmt,
 869     description => "NUMA topology.",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 872
 873 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 874     $confdesc->{"numa$i"} = $numadesc;
 875 }
 876
 877 my $nic_model_list = [
 878     'e1000',
 879     'e1000-82540em',
 880     'e1000-82544gc',
 881     'e1000-82545em',
 882     'e1000e',
 883     'i82551',
 884     'i82557b',
 885     'i82559er',
 886     'ne2k_isa',
 887     'ne2k_pci',
 888     'pcnet',
 889     'rtl8139',
 890     'virtio',
 891     'vmxnet3',
 892 ];
 893 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 894
 895 my $net_fmt_bridge_descr = <<__EOD__;
 896 Bridge to attach the network device to. The Proxmox VE standard bridge
 897 is called 'vmbr0'.
 898
 899 If you do not specify a bridge, we create a kvm user (NATed) network
 900 device, which provides DHCP and DNS services. The following addresses
 901 are used:
 902
 903  10.0.2.2   Gateway
 904  10.0.2.3   DNS Server
 905  10.0.2.4   SMB Server
 906
 907 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 908 __EOD__
 909
 910 my $net_fmt = {
 911     macaddr  => get_standard_option('mac-addr', {
 912         description => "MAC address. That address must be unique withing your network. This is"
 913             ." automatically generated if not specified.",
 914     }),
 915     model => {
 916         type => 'string',
 917         description => "Network Card Model. The 'virtio' model provides the best performance with"
 918             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 919             ." best to use 'e1000'.",
 920         enum => $nic_model_list,
 921         default_key => 1,
 922     },
 923     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 924     bridge => get_standard_option('pve-bridge-id', {
 925         description => $net_fmt_bridge_descr,
 926         optional => 1,
 927     }),
 928     queues => {
 929         type => 'integer',
 930         minimum => 0, maximum => 64,
 931         description => 'Number of packet queues to be used on the device.',
 932         optional => 1,
 933     },
 934     rate => {
 935         type => 'number',
 936         minimum => 0,
 937         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 938         optional => 1,
 939     },
 940     tag => {
 941         type => 'integer',
 942         minimum => 1, maximum => 4094,
 943         description => 'VLAN tag to apply to packets on this interface.',
 944         optional => 1,
 945     },
 946     trunks => {
 947         type => 'string',
 948         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 949         description => 'VLAN trunks to pass through this interface.',
 950         format_description => 'vlanid[;vlanid...]',
 951         optional => 1,
 952     },
 953     firewall => {
 954         type => 'boolean',
 955         description => 'Whether this interface should be protected by the firewall.',
 956         optional => 1,
 957     },
 958     link_down => {
 959         type => 'boolean',
 960         description => 'Whether this interface should be disconnected (like pulling the plug).',
 961         optional => 1,
 962     },
 963     mtu => {
 964         type => 'integer',
 965         minimum => 1, maximum => 65520,
 966         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 967         optional => 1,
 968     },
 969 };
 970
 971 my $netdesc = {
 972     optional => 1,
 973     type => 'string', format => $net_fmt,
 974     description => "Specify network devices.",
 975 };
 976
 977 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 978
 979 my $ipconfig_fmt = {
 980     ip => {
 981         type => 'string',
 982         format => 'pve-ipv4-config',
 983         format_description => 'IPv4Format/CIDR',
 984         description => 'IPv4 address in CIDR format.',
 985         optional => 1,
 986         default => 'dhcp',
 987     },
 988     gw => {
 989         type => 'string',
 990         format => 'ipv4',
 991         format_description => 'GatewayIPv4',
 992         description => 'Default gateway for IPv4 traffic.',
 993         optional => 1,
 994         requires => 'ip',
 995     },
 996     ip6 => {
 997         type => 'string',
 998         format => 'pve-ipv6-config',
 999         format_description => 'IPv6Format/CIDR',
1000         description => 'IPv6 address in CIDR format.',
1001         optional => 1,
1002         default => 'dhcp',
1003     },
1004     gw6 => {
1005         type => 'string',
1006         format => 'ipv6',
1007         format_description => 'GatewayIPv6',
1008         description => 'Default gateway for IPv6 traffic.',
1009         optional => 1,
1010         requires => 'ip6',
1011     },
1012 };
1013 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1014 my $ipconfigdesc = {
1015     optional => 1,
1016     type => 'string', format => 'pve-qm-ipconfig',
1017     description => <<'EODESCR',
1018 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1019
1020 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1021
1022 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1023 gateway should be provided.
1024 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1025 cloud-init 19.4 or newer.
1026
1027 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1028 dhcp on IPv4.
1029 EODESCR
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1032
1033 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1034     $confdesc->{"net$i"} = $netdesc;
1035     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1036 }
1037
1038 foreach my $key (keys %$confdesc_cloudinit) {
1039     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1040 }
1041
1042 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1043 sub pve_verify_cpuset {
1044     my ($set_text, $noerr) = @_;
1045
1046     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1047
1048     if ($@) {
1049         return if $noerr;
1050         die "unable to parse cpuset option\n";
1051     }
1052
1053     return PVE::CpuSet->new($members)->short_string();
1054 }
1055
1056 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1057 sub verify_volume_id_or_qm_path {
1058     my ($volid, $noerr) = @_;
1059
1060     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1061
1062     return verify_volume_id_or_absolute_path($volid, $noerr);
1063 }
1064
1065 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1066 sub verify_volume_id_or_absolute_path {
1067     my ($volid, $noerr) = @_;
1068
1069     return $volid if $volid =~ m|^/|;
1070
1071     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1072     if ($@) {
1073         return if $noerr;
1074         die $@;
1075     }
1076     return $volid;
1077 }
1078
1079 my $usb_fmt = {
1080     host => {
1081         default_key => 1,
1082         type => 'string', format => 'pve-qm-usb-device',
1083         format_description => 'HOSTUSBDEVICE|spice',
1084         description => <<EODESCR,
1085 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1086
1087  'bus-port(.port)*' (decimal numbers) or
1088  'vendor_id:product_id' (hexadeciaml numbers) or
1089  'spice'
1090
1091 You can use the 'lsusb -t' command to list existing usb devices.
1092
1093 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1094 machines - use with special care.
1095
1096 The value 'spice' can be used to add a usb redirection devices for spice.
1097 EODESCR
1098     },
1099     usb3 => {
1100         optional => 1,
1101         type => 'boolean',
1102         description => "Specifies whether if given host option is a USB3 device or port."
1103             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1104             ." is irrelevant (all devices are plugged into a xhci controller).",
1105         default => 0,
1106     },
1107 };
1108
1109 my $usbdesc = {
1110     optional => 1,
1111     type => 'string', format => $usb_fmt,
1112     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1113         ." l26 or windows > 7, n can be up to 14).",
1114 };
1115 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1116
1117 my $serialdesc = {
1118         optional => 1,
1119         type => 'string',
1120         pattern => '(/dev/.+|socket)',
1121         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1122         verbose_description =>  <<EODESCR,
1123 Create a serial device inside the VM (n is 0 to 3), and pass through a
1124 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1125 host side (use 'qm terminal' to open a terminal connection).
1126
1127 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1128 use with special care.
1129
1130 CAUTION: Experimental! User reported problems with this option.
1131 EODESCR
1132 };
1133
1134 my $paralleldesc= {
1135         optional => 1,
1136         type => 'string',
1137         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1138         description =>  "Map host parallel devices (n is 0 to 2).",
1139         verbose_description =>  <<EODESCR,
1140 Map host parallel devices (n is 0 to 2).
1141
1142 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1143 machines - use with special care.
1144
1145 CAUTION: Experimental! User reported problems with this option.
1146 EODESCR
1147 };
1148
1149 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1150     $confdesc->{"parallel$i"} = $paralleldesc;
1151 }
1152
1153 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1154     $confdesc->{"serial$i"} = $serialdesc;
1155 }
1156
1157 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1158     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1159 }
1160
1161 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1162     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1163 }
1164
1165 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1166     $confdesc->{"usb$i"} = $usbdesc;
1167 }
1168
1169 my $boot_fmt = {
1170     legacy => {
1171         optional => 1,
1172         default_key => 1,
1173         type => 'string',
1174         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1175                      . " Deprecated, use 'order=' instead.",
1176         pattern => '[acdn]{1,4}',
1177         format_description => "[acdn]{1,4}",
1178
1179         # note: this is also the fallback if boot: is not given at all
1180         default => 'cdn',
1181     },
1182     order => {
1183         optional => 1,
1184         type => 'string',
1185         format => 'pve-qm-bootdev-list',
1186         format_description => "device[;device...]",
1187         description => <<EODESC,
1188 The guest will attempt to boot from devices in the order they appear here.
1189
1190 Disks, optical drives and passed-through storage USB devices will be directly
1191 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1192 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1193
1194 Note that only devices in this list will be marked as bootable and thus loaded
1195 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1196 (e.g. software-raid), you need to specify all of them here.
1197
1198 Overrides the deprecated 'legacy=[acdn]*' value when given.
1199 EODESC
1200     },
1201 };
1202 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1203
1204 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1205 sub verify_bootdev {
1206     my ($dev, $noerr) = @_;
1207
1208     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1209     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1210
1211     my $check = sub {
1212         my ($base) = @_;
1213         return 0 if $dev !~ m/^$base\d+$/;
1214         return 0 if !$confdesc->{$dev};
1215         return 1;
1216     };
1217
1218     return $dev if $check->("net");
1219     return $dev if $check->("usb");
1220     return $dev if $check->("hostpci");
1221
1222     return if $noerr;
1223     die "invalid boot device '$dev'\n";
1224 }
1225
1226 sub print_bootorder {
1227     my ($devs) = @_;
1228     return "" if !@$devs;
1229     my $data = { order => join(';', @$devs) };
1230     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1231 }
1232
1233 my $kvm_api_version = 0;
1234
1235 sub kvm_version {
1236     return $kvm_api_version if $kvm_api_version;
1237
1238     open my $fh, '<', '/dev/kvm' or return;
1239
1240     # 0xae00 => KVM_GET_API_VERSION
1241     $kvm_api_version = ioctl($fh, 0xae00, 0);
1242     close($fh);
1243
1244     return $kvm_api_version;
1245 }
1246
1247 my $kvm_user_version = {};
1248 my $kvm_mtime = {};
1249
1250 sub kvm_user_version {
1251     my ($binary) = @_;
1252
1253     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1254     my $st = stat($binary);
1255
1256     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1257     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1258         $cachedmtime == $st->mtime;
1259
1260     $kvm_user_version->{$binary} = 'unknown';
1261     $kvm_mtime->{$binary} = $st->mtime;
1262
1263     my $code = sub {
1264         my $line = shift;
1265         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1266             $kvm_user_version->{$binary} = $2;
1267         }
1268     };
1269
1270     eval { run_command([$binary, '--version'], outfunc => $code); };
1271     warn $@ if $@;
1272
1273     return $kvm_user_version->{$binary};
1274
1275 }
1276 my sub extract_version {
1277     my ($machine_type, $version) = @_;
1278     $version = kvm_user_version() if !defined($version);
1279     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1280 }
1281
1282 sub kernel_has_vhost_net {
1283     return -c '/dev/vhost-net';
1284 }
1285
1286 sub option_exists {
1287     my $key = shift;
1288     return defined($confdesc->{$key});
1289 }
1290
1291 my $cdrom_path;
1292 sub get_cdrom_path {
1293
1294     return $cdrom_path if defined($cdrom_path);
1295
1296     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1297
1298     if (!defined($cdrom_path)) {
1299         log_warn("no physical CD-ROM available, ignoring");
1300         $cdrom_path = '';
1301     }
1302
1303     return $cdrom_path;
1304 }
1305
1306 sub get_iso_path {
1307     my ($storecfg, $vmid, $cdrom) = @_;
1308
1309     if ($cdrom eq 'cdrom') {
1310         return get_cdrom_path();
1311     } elsif ($cdrom eq 'none') {
1312         return '';
1313     } elsif ($cdrom =~ m|^/|) {
1314         return $cdrom;
1315     } else {
1316         return PVE::Storage::path($storecfg, $cdrom);
1317     }
1318 }
1319
1320 # try to convert old style file names to volume IDs
1321 sub filename_to_volume_id {
1322     my ($vmid, $file, $media) = @_;
1323
1324      if (!($file eq 'none' || $file eq 'cdrom' ||
1325           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1326
1327         return if $file =~ m|/|;
1328
1329         if ($media && $media eq 'cdrom') {
1330             $file = "local:iso/$file";
1331         } else {
1332             $file = "local:$vmid/$file";
1333         }
1334     }
1335
1336     return $file;
1337 }
1338
1339 sub verify_media_type {
1340     my ($opt, $vtype, $media) = @_;
1341
1342     return if !$media;
1343
1344     my $etype;
1345     if ($media eq 'disk') {
1346         $etype = 'images';
1347     } elsif ($media eq 'cdrom') {
1348         $etype = 'iso';
1349     } else {
1350         die "internal error";
1351     }
1352
1353     return if ($vtype eq $etype);
1354
1355     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1356 }
1357
1358 sub cleanup_drive_path {
1359     my ($opt, $storecfg, $drive) = @_;
1360
1361     # try to convert filesystem paths to volume IDs
1362
1363     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1364         ($drive->{file} !~ m|^/dev/.+|) &&
1365         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1366         ($drive->{file} !~ m/^\d+$/)) {
1367         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1368         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1369             if !$vtype;
1370         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1371         verify_media_type($opt, $vtype, $drive->{media});
1372         $drive->{file} = $volid;
1373     }
1374
1375     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1376 }
1377
1378 sub parse_hotplug_features {
1379     my ($data) = @_;
1380
1381     my $res = {};
1382
1383     return $res if $data eq '0';
1384
1385     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1386
1387     foreach my $feature (PVE::Tools::split_list($data)) {
1388         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1389             $res->{$1} = 1;
1390         } else {
1391             die "invalid hotplug feature '$feature'\n";
1392         }
1393     }
1394     return $res;
1395 }
1396
1397 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1398 sub pve_verify_hotplug_features {
1399     my ($value, $noerr) = @_;
1400
1401     return $value if parse_hotplug_features($value);
1402
1403     return if $noerr;
1404
1405     die "unable to parse hotplug option\n";
1406 }
1407
1408 sub scsi_inquiry {
1409     my($fh, $noerr) = @_;
1410
1411     my $SG_IO = 0x2285;
1412     my $SG_GET_VERSION_NUM = 0x2282;
1413
1414     my $versionbuf = "\x00" x 8;
1415     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1416     if (!$ret) {
1417         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1418         return;
1419     }
1420     my $version = unpack("I", $versionbuf);
1421     if ($version < 30000) {
1422         die "scsi generic interface too old\n"  if !$noerr;
1423         return;
1424     }
1425
1426     my $buf = "\x00" x 36;
1427     my $sensebuf = "\x00" x 8;
1428     my $cmd = pack("C x3 C x1", 0x12, 36);
1429
1430     # see /usr/include/scsi/sg.h
1431     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1432
1433     my $packet = pack(
1434         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1435     );
1436
1437     $ret = ioctl($fh, $SG_IO, $packet);
1438     if (!$ret) {
1439         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1440         return;
1441     }
1442
1443     my @res = unpack($sg_io_hdr_t, $packet);
1444     if ($res[17] || $res[18]) {
1445         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1446         return;
1447     }
1448
1449     my $res = {};
1450     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1451
1452     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1453     $res->{type} &= 0x1F;
1454
1455     return $res;
1456 }
1457
1458 sub path_is_scsi {
1459     my ($path) = @_;
1460
1461     my $fh = IO::File->new("+<$path") || return;
1462     my $res = scsi_inquiry($fh, 1);
1463     close($fh);
1464
1465     return $res;
1466 }
1467
1468 sub print_tabletdevice_full {
1469     my ($conf, $arch) = @_;
1470
1471     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1472
1473     # we use uhci for old VMs because tablet driver was buggy in older qemu
1474     my $usbbus;
1475     if ($q35 || $arch eq 'aarch64') {
1476         $usbbus = 'ehci';
1477     } else {
1478         $usbbus = 'uhci';
1479     }
1480
1481     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1482 }
1483
1484 sub print_keyboarddevice_full {
1485     my ($conf, $arch) = @_;
1486
1487     return if $arch ne 'aarch64';
1488
1489     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1490 }
1491
1492 my sub get_drive_id {
1493     my ($drive) = @_;
1494     return "$drive->{interface}$drive->{index}";
1495 }
1496
1497 sub print_drivedevice_full {
1498     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1499
1500     my $device = '';
1501     my $maxdev = 0;
1502
1503     my $drive_id = get_drive_id($drive);
1504     if ($drive->{interface} eq 'virtio') {
1505         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1506         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1507         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1508     } elsif ($drive->{interface} eq 'scsi') {
1509
1510         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1511         my $unit = $drive->{index} % $maxdev;
1512         my $devicetype = 'hd';
1513         my $path = '';
1514         if (drive_is_cdrom($drive)) {
1515             $devicetype = 'cd';
1516         } else {
1517             if ($drive->{file} =~ m|^/|) {
1518                 $path = $drive->{file};
1519                 if (my $info = path_is_scsi($path)) {
1520                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1521                         $devicetype = 'block';
1522                     } elsif ($info->{type} == 1) { # tape
1523                         $devicetype = 'generic';
1524                     }
1525                 }
1526             } else {
1527                  $path = PVE::Storage::path($storecfg, $drive->{file});
1528             }
1529
1530             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1531             my $version = extract_version($machine_type, kvm_user_version());
1532             if ($path =~ m/^iscsi\:\/\// &&
1533                !min_version($version, 4, 1)) {
1534                 $devicetype = 'generic';
1535             }
1536         }
1537
1538         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1539             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1540         } else {
1541             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1542                 .",lun=$drive->{index}";
1543         }
1544         $device .= ",drive=drive-$drive_id,id=$drive_id";
1545
1546         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1547             $device .= ",rotation_rate=1";
1548         }
1549         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1550
1551     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1552         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1553         my $controller = int($drive->{index} / $maxdev);
1554         my $unit = $drive->{index} % $maxdev;
1555         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1556
1557         $device = "ide-$devicetype";
1558         if ($drive->{interface} eq 'ide') {
1559             $device .= ",bus=ide.$controller,unit=$unit";
1560         } else {
1561             $device .= ",bus=ahci$controller.$unit";
1562         }
1563         $device .= ",drive=drive-$drive_id,id=$drive_id";
1564
1565         if ($devicetype eq 'hd') {
1566             if (my $model = $drive->{model}) {
1567                 $model = URI::Escape::uri_unescape($model);
1568                 $device .= ",model=$model";
1569             }
1570             if ($drive->{ssd}) {
1571                 $device .= ",rotation_rate=1";
1572             }
1573         }
1574         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1575     } elsif ($drive->{interface} eq 'usb') {
1576         die "implement me";
1577         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1578     } else {
1579         die "unsupported interface type";
1580     }
1581
1582     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1583
1584     if (my $serial = $drive->{serial}) {
1585         $serial = URI::Escape::uri_unescape($serial);
1586         $device .= ",serial=$serial";
1587     }
1588
1589
1590     return $device;
1591 }
1592
1593 sub get_initiator_name {
1594     my $initiator;
1595
1596     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1597     while (defined(my $line = <$fh>)) {
1598         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1599         $initiator = $1;
1600         last;
1601     }
1602     $fh->close();
1603
1604     return $initiator;
1605 }
1606
1607 my sub storage_allows_io_uring_default {
1608     my ($scfg, $cache_direct) = @_;
1609
1610     # io_uring with cache mode writeback or writethrough on krbd will hang...
1611     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1612
1613     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1614     # sometimes, just plain disable...
1615     return if $scfg && $scfg->{type} eq 'lvm';
1616
1617     # io_uring causes problems when used with CIFS since kernel 5.15
1618     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1619     return if $scfg && $scfg->{type} eq 'cifs';
1620
1621     return 1;
1622 }
1623
1624 my sub drive_uses_cache_direct {
1625     my ($drive, $scfg) = @_;
1626
1627     my $cache_direct = 0;
1628
1629     if (my $cache = $drive->{cache}) {
1630         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1631     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1632         $cache_direct = 1;
1633     }
1634
1635     return $cache_direct;
1636 }
1637
1638 sub print_drive_commandline_full {
1639     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1640
1641     my $path;
1642     my $volid = $drive->{file};
1643     my $format = $drive->{format};
1644     my $drive_id = get_drive_id($drive);
1645
1646     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1647     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1648
1649     if (drive_is_cdrom($drive)) {
1650         $path = get_iso_path($storecfg, $vmid, $volid);
1651         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1652     } else {
1653         if ($storeid) {
1654             $path = PVE::Storage::path($storecfg, $volid);
1655             $format //= qemu_img_format($scfg, $volname);
1656         } else {
1657             $path = $volid;
1658             $format //= "raw";
1659         }
1660    }
1661
1662    my $is_rbd = $path =~ m/^rbd:/;
1663
1664     my $opts = '';
1665     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1666     foreach my $o (@qemu_drive_options) {
1667         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1668     }
1669
1670     # snapshot only accepts on|off
1671     if (defined($drive->{snapshot})) {
1672         my $v = $drive->{snapshot} ? 'on' : 'off';
1673         $opts .= ",snapshot=$v";
1674     }
1675
1676     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1677         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1678     }
1679
1680     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1681         my ($dir, $qmpname) = @$type;
1682         if (my $v = $drive->{"mbps$dir"}) {
1683             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1684         }
1685         if (my $v = $drive->{"mbps${dir}_max"}) {
1686             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1687         }
1688         if (my $v = $drive->{"bps${dir}_max_length"}) {
1689             $opts .= ",throttling.bps$qmpname-max-length=$v";
1690         }
1691         if (my $v = $drive->{"iops${dir}"}) {
1692             $opts .= ",throttling.iops$qmpname=$v";
1693         }
1694         if (my $v = $drive->{"iops${dir}_max"}) {
1695             $opts .= ",throttling.iops$qmpname-max=$v";
1696         }
1697         if (my $v = $drive->{"iops${dir}_max_length"}) {
1698             $opts .= ",throttling.iops$qmpname-max-length=$v";
1699         }
1700     }
1701
1702     if ($pbs_name) {
1703         $format = "rbd" if $is_rbd;
1704         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1705             if !$format;
1706         $opts .= ",format=alloc-track,file.driver=$format";
1707     } elsif ($format) {
1708         $opts .= ",format=$format";
1709     }
1710
1711     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1712
1713     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1714
1715     if (!$drive->{aio}) {
1716         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1717             # io_uring supports all cache modes
1718             $opts .= ",aio=io_uring";
1719         } else {
1720             # aio native works only with O_DIRECT
1721             if($cache_direct) {
1722                 $opts .= ",aio=native";
1723             } else {
1724                 $opts .= ",aio=threads";
1725             }
1726         }
1727     }
1728
1729     if (!drive_is_cdrom($drive)) {
1730         my $detectzeroes;
1731         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1732             $detectzeroes = 'off';
1733         } elsif ($drive->{discard}) {
1734             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1735         } else {
1736             # This used to be our default with discard not being specified:
1737             $detectzeroes = 'on';
1738         }
1739
1740         # note: 'detect-zeroes' works per blockdev and we want it to persist
1741         # after the alloc-track is removed, so put it on 'file' directly
1742         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1743         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1744     }
1745
1746     if ($pbs_name) {
1747         $opts .= ",backing=$pbs_name";
1748         $opts .= ",auto-remove=on";
1749     }
1750
1751     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1752     my $file_param = "file";
1753     if ($pbs_name) {
1754         # non-rbd drivers require the underlying file to be a seperate block
1755         # node, so add a second .file indirection
1756         $file_param .= ".file" if !$is_rbd;
1757         $file_param .= ".filename";
1758     }
1759     my $pathinfo = $path ? "$file_param=$path," : '';
1760
1761     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1762 }
1763
1764 sub print_pbs_blockdev {
1765     my ($pbs_conf, $pbs_name) = @_;
1766     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1767     $blockdev .= ",repository=$pbs_conf->{repository}";
1768     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1769     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1770     $blockdev .= ",archive=$pbs_conf->{archive}";
1771     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1772     return $blockdev;
1773 }
1774
1775 sub print_netdevice_full {
1776     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1777
1778     my $device = $net->{model};
1779     if ($net->{model} eq 'virtio') {
1780          $device = 'virtio-net-pci';
1781      };
1782
1783     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1784     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1785     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1786         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1787         # and out of each queue plus one config interrupt and control vector queue
1788         my $vectors = $net->{queues} * 2 + 2;
1789         $tmpstr .= ",vectors=$vectors,mq=on";
1790         if (min_version($machine_version, 7, 1)) {
1791             $tmpstr .= ",packed=on";
1792         }
1793     }
1794
1795     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1796         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1797     }
1798
1799     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1800
1801     if (my $mtu = $net->{mtu}) {
1802         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1803             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1804             if ($mtu == 1) {
1805                  $mtu = $bridge_mtu;
1806             } elsif ($mtu < 576) {
1807                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1808             } elsif ($mtu > $bridge_mtu) {
1809                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1810             }
1811             $tmpstr .= ",host_mtu=$mtu";
1812         } else {
1813             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1814         }
1815     }
1816
1817     if ($use_old_bios_files) {
1818         my $romfile;
1819         if ($device eq 'virtio-net-pci') {
1820             $romfile = 'pxe-virtio.rom';
1821         } elsif ($device eq 'e1000') {
1822             $romfile = 'pxe-e1000.rom';
1823         } elsif ($device eq 'e1000e') {
1824             $romfile = 'pxe-e1000e.rom';
1825         } elsif ($device eq 'ne2k') {
1826             $romfile = 'pxe-ne2k_pci.rom';
1827         } elsif ($device eq 'pcnet') {
1828             $romfile = 'pxe-pcnet.rom';
1829         } elsif ($device eq 'rtl8139') {
1830             $romfile = 'pxe-rtl8139.rom';
1831         }
1832         $tmpstr .= ",romfile=$romfile" if $romfile;
1833     }
1834
1835     return $tmpstr;
1836 }
1837
1838 sub print_netdev_full {
1839     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1840
1841     my $i = '';
1842     if ($netid =~ m/^net(\d+)$/) {
1843         $i = int($1);
1844     }
1845
1846     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1847
1848     my $ifname = "tap${vmid}i$i";
1849
1850     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1851     die "interface name '$ifname' is too long (max 15 character)\n"
1852         if length($ifname) >= 16;
1853
1854     my $vhostparam = '';
1855     if (is_native($arch)) {
1856         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1857     }
1858
1859     my $vmname = $conf->{name} || "vm$vmid";
1860
1861     my $netdev = "";
1862     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1863
1864     if ($net->{bridge}) {
1865         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1866             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1867     } else {
1868         $netdev = "type=user,id=$netid,hostname=$vmname";
1869     }
1870
1871     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1872
1873     return $netdev;
1874 }
1875
1876 my $vga_map = {
1877     'cirrus' => 'cirrus-vga',
1878     'std' => 'VGA',
1879     'vmware' => 'vmware-svga',
1880     'virtio' => 'virtio-vga',
1881     'virtio-gl' => 'virtio-vga-gl',
1882 };
1883
1884 sub print_vga_device {
1885     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1886
1887     my $type = $vga_map->{$vga->{type}};
1888     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1889         $type = 'virtio-gpu';
1890     }
1891     my $vgamem_mb = $vga->{memory};
1892
1893     my $max_outputs = '';
1894     if ($qxlnum) {
1895         $type = $id ? 'qxl' : 'qxl-vga';
1896
1897         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1898             # set max outputs so linux can have up to 4 qxl displays with one device
1899             if (min_version($machine_version, 4, 1)) {
1900                 $max_outputs = ",max_outputs=4";
1901             }
1902         }
1903     }
1904
1905     die "no devicetype for $vga->{type}\n" if !$type;
1906
1907     my $memory = "";
1908     if ($vgamem_mb) {
1909         if ($vga->{type} =~ /^virtio/) {
1910             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1911             $memory = ",max_hostmem=$bytes";
1912         } elsif ($qxlnum) {
1913             # from https://www.spice-space.org/multiple-monitors.html
1914             $memory = ",vgamem_mb=$vga->{memory}";
1915             my $ram = $vgamem_mb * 4;
1916             my $vram = $vgamem_mb * 2;
1917             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1918         } else {
1919             $memory = ",vgamem_mb=$vga->{memory}";
1920         }
1921     } elsif ($qxlnum && $id) {
1922         $memory = ",ram_size=67108864,vram_size=33554432";
1923     }
1924
1925     my $edidoff = "";
1926     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1927         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1928     }
1929
1930     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1931     my $vgaid = "vga" . ($id // '');
1932     my $pciaddr;
1933     if ($q35 && $vgaid eq 'vga') {
1934         # the first display uses pcie.0 bus on q35 machines
1935         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1936     } else {
1937         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1938     }
1939
1940     if ($vga->{type} eq 'virtio-gl') {
1941         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1942         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1943             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1944
1945         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1946             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1947     }
1948
1949     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1950 }
1951
1952 sub parse_number_sets {
1953     my ($set) = @_;
1954     my $res = [];
1955     foreach my $part (split(/;/, $set)) {
1956         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1957             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1958             push @$res, [ $1, $2 ];
1959         } else {
1960             die "invalid range: $part\n";
1961         }
1962     }
1963     return $res;
1964 }
1965
1966 sub parse_numa {
1967     my ($data) = @_;
1968
1969     my $res = parse_property_string($numa_fmt, $data);
1970     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1971     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1972     return $res;
1973 }
1974
1975 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1976 sub parse_net {
1977     my ($data, $disable_mac_autogen) = @_;
1978
1979     my $res = eval { parse_property_string($net_fmt, $data) };
1980     if ($@) {
1981         warn $@;
1982         return;
1983     }
1984     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1985         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1986         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1987     }
1988     return $res;
1989 }
1990
1991 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1992 sub parse_ipconfig {
1993     my ($data) = @_;
1994
1995     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1996     if ($@) {
1997         warn $@;
1998         return;
1999     }
2000
2001     if ($res->{gw} && !$res->{ip}) {
2002         warn 'gateway specified without specifying an IP address';
2003         return;
2004     }
2005     if ($res->{gw6} && !$res->{ip6}) {
2006         warn 'IPv6 gateway specified without specifying an IPv6 address';
2007         return;
2008     }
2009     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2010         warn 'gateway specified together with DHCP';
2011         return;
2012     }
2013     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2014         # gw6 + auto/dhcp
2015         warn "IPv6 gateway specified together with $res->{ip6} address";
2016         return;
2017     }
2018
2019     if (!$res->{ip} && !$res->{ip6}) {
2020         return { ip => 'dhcp', ip6 => 'dhcp' };
2021     }
2022
2023     return $res;
2024 }
2025
2026 sub print_net {
2027     my $net = shift;
2028
2029     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2030 }
2031
2032 sub add_random_macs {
2033     my ($settings) = @_;
2034
2035     foreach my $opt (keys %$settings) {
2036         next if $opt !~ m/^net(\d+)$/;
2037         my $net = parse_net($settings->{$opt});
2038         next if !$net;
2039         $settings->{$opt} = print_net($net);
2040     }
2041 }
2042
2043 sub vm_is_volid_owner {
2044     my ($storecfg, $vmid, $volid) = @_;
2045
2046     if ($volid !~  m|^/|) {
2047         my ($path, $owner);
2048         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2049         if ($owner && ($owner == $vmid)) {
2050             return 1;
2051         }
2052     }
2053
2054     return;
2055 }
2056
2057 sub vmconfig_register_unused_drive {
2058     my ($storecfg, $vmid, $conf, $drive) = @_;
2059
2060     if (drive_is_cloudinit($drive)) {
2061         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2062         warn $@ if $@;
2063         delete $conf->{cloudinit};
2064     } elsif (!drive_is_cdrom($drive)) {
2065         my $volid = $drive->{file};
2066         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2067             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2068         }
2069     }
2070 }
2071
2072 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2073 my $smbios1_fmt = {
2074     uuid => {
2075         type => 'string',
2076         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2077         format_description => 'UUID',
2078         description => "Set SMBIOS1 UUID.",
2079         optional => 1,
2080     },
2081     version => {
2082         type => 'string',
2083         pattern => '[A-Za-z0-9+\/]+={0,2}',
2084         format_description => 'Base64 encoded string',
2085         description => "Set SMBIOS1 version.",
2086         optional => 1,
2087     },
2088     serial => {
2089         type => 'string',
2090         pattern => '[A-Za-z0-9+\/]+={0,2}',
2091         format_description => 'Base64 encoded string',
2092         description => "Set SMBIOS1 serial number.",
2093         optional => 1,
2094     },
2095     manufacturer => {
2096         type => 'string',
2097         pattern => '[A-Za-z0-9+\/]+={0,2}',
2098         format_description => 'Base64 encoded string',
2099         description => "Set SMBIOS1 manufacturer.",
2100         optional => 1,
2101     },
2102     product => {
2103         type => 'string',
2104         pattern => '[A-Za-z0-9+\/]+={0,2}',
2105         format_description => 'Base64 encoded string',
2106         description => "Set SMBIOS1 product ID.",
2107         optional => 1,
2108     },
2109     sku => {
2110         type => 'string',
2111         pattern => '[A-Za-z0-9+\/]+={0,2}',
2112         format_description => 'Base64 encoded string',
2113         description => "Set SMBIOS1 SKU string.",
2114         optional => 1,
2115     },
2116     family => {
2117         type => 'string',
2118         pattern => '[A-Za-z0-9+\/]+={0,2}',
2119         format_description => 'Base64 encoded string',
2120         description => "Set SMBIOS1 family string.",
2121         optional => 1,
2122     },
2123     base64 => {
2124         type => 'boolean',
2125         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2126         optional => 1,
2127     },
2128 };
2129
2130 sub parse_smbios1 {
2131     my ($data) = @_;
2132
2133     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2134     warn $@ if $@;
2135     return $res;
2136 }
2137
2138 sub print_smbios1 {
2139     my ($smbios1) = @_;
2140     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2141 }
2142
2143 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2144
2145 sub parse_watchdog {
2146     my ($value) = @_;
2147
2148     return if !$value;
2149
2150     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2151     warn $@ if $@;
2152     return $res;
2153 }
2154
2155 sub parse_guest_agent {
2156     my ($conf) = @_;
2157
2158     return {} if !defined($conf->{agent});
2159
2160     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2161     warn $@ if $@;
2162
2163     # if the agent is disabled ignore the other potentially set properties
2164     return {} if !$res->{enabled};
2165     return $res;
2166 }
2167
2168 sub get_qga_key {
2169     my ($conf, $key) = @_;
2170     return undef if !defined($conf->{agent});
2171
2172     my $agent = parse_guest_agent($conf);
2173     return $agent->{$key};
2174 }
2175
2176 sub parse_vga {
2177     my ($value) = @_;
2178
2179     return {} if !$value;
2180     my $res = eval { parse_property_string($vga_fmt, $value) };
2181     warn $@ if $@;
2182     return $res;
2183 }
2184
2185 sub parse_rng {
2186     my ($value) = @_;
2187
2188     return if !$value;
2189
2190     my $res = eval { parse_property_string($rng_fmt, $value) };
2191     warn $@ if $@;
2192     return $res;
2193 }
2194
2195 sub parse_meta_info {
2196     my ($value) = @_;
2197
2198     return if !$value;
2199
2200     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2201     warn $@ if $@;
2202     return $res;
2203 }
2204
2205 sub new_meta_info_string {
2206     my () = @_; # for now do not allow to override any value
2207
2208     return PVE::JSONSchema::print_property_string(
2209         {
2210             'creation-qemu' => kvm_user_version(),
2211             ctime => "". int(time()),
2212         },
2213         $meta_info_fmt
2214     );
2215 }
2216
2217 sub qemu_created_version_fixups {
2218     my ($conf, $forcemachine, $kvmver) = @_;
2219
2220     my $meta = parse_meta_info($conf->{meta}) // {};
2221     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2222
2223     # check if we need to apply some handling for VMs that always use the latest machine version but
2224     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2225     # would be required (we do not want to pin machine version for non-windows OS type)
2226     if (
2227         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2228         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2229         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2230         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2231     ) {
2232         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2233         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2234             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2235             # and thus with the predictable interface naming of systemd
2236             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2237         }
2238     }
2239     return;
2240 }
2241
2242 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2243 sub verify_usb_device {
2244     my ($value, $noerr) = @_;
2245
2246     return $value if parse_usb_device($value);
2247
2248     return if $noerr;
2249
2250     die "unable to parse usb device\n";
2251 }
2252
2253 # add JSON properties for create and set function
2254 sub json_config_properties {
2255     my ($prop, $with_disk_alloc) = @_;
2256
2257     my $skip_json_config_opts = {
2258         parent => 1,
2259         snaptime => 1,
2260         vmstate => 1,
2261         runningmachine => 1,
2262         runningcpu => 1,
2263         meta => 1,
2264     };
2265
2266     foreach my $opt (keys %$confdesc) {
2267         next if $skip_json_config_opts->{$opt};
2268
2269         if ($with_disk_alloc && is_valid_drivename($opt)) {
2270             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2271         } else {
2272             $prop->{$opt} = $confdesc->{$opt};
2273         }
2274     }
2275
2276     return $prop;
2277 }
2278
2279 # Properties that we can read from an OVF file
2280 sub json_ovf_properties {
2281     my $prop = {};
2282
2283     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2284         $prop->{$device} = {
2285             type => 'string',
2286             format => 'pve-volume-id-or-absolute-path',
2287             description => "Disk image that gets imported to $device",
2288             optional => 1,
2289         };
2290     }
2291
2292     $prop->{cores} = {
2293         type => 'integer',
2294         description => "The number of CPU cores.",
2295         optional => 1,
2296     };
2297     $prop->{memory} = {
2298         type => 'integer',
2299         description => "Amount of RAM for the VM in MB.",
2300         optional => 1,
2301     };
2302     $prop->{name} = {
2303         type => 'string',
2304         description => "Name of the VM.",
2305         optional => 1,
2306     };
2307
2308     return $prop;
2309 }
2310
2311 # return copy of $confdesc_cloudinit to generate documentation
2312 sub cloudinit_config_properties {
2313
2314     return dclone($confdesc_cloudinit);
2315 }
2316
2317 sub cloudinit_pending_properties {
2318     my $p = {
2319         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2320         name => 1,
2321     };
2322     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2323     return $p;
2324 }
2325
2326 sub check_type {
2327     my ($key, $value) = @_;
2328
2329     die "unknown setting '$key'\n" if !$confdesc->{$key};
2330
2331     my $type = $confdesc->{$key}->{type};
2332
2333     if (!defined($value)) {
2334         die "got undefined value\n";
2335     }
2336
2337     if ($value =~ m/[\n\r]/) {
2338         die "property contains a line feed\n";
2339     }
2340
2341     if ($type eq 'boolean') {
2342         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2343         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2344         die "type check ('boolean') failed - got '$value'\n";
2345     } elsif ($type eq 'integer') {
2346         return int($1) if $value =~ m/^(\d+)$/;
2347         die "type check ('integer') failed - got '$value'\n";
2348     } elsif ($type eq 'number') {
2349         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2350         die "type check ('number') failed - got '$value'\n";
2351     } elsif ($type eq 'string') {
2352         if (my $fmt = $confdesc->{$key}->{format}) {
2353             PVE::JSONSchema::check_format($fmt, $value);
2354             return $value;
2355         }
2356         $value =~ s/^\"(.*)\"$/$1/;
2357         return $value;
2358     } else {
2359         die "internal error"
2360     }
2361 }
2362
2363 sub destroy_vm {
2364     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2365
2366     my $conf = PVE::QemuConfig->load_config($vmid);
2367
2368     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2369         PVE::QemuConfig->check_lock($conf);
2370     }
2371
2372     if ($conf->{template}) {
2373         # check if any base image is still used by a linked clone
2374         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2375                 my ($ds, $drive) = @_;
2376                 return if drive_is_cdrom($drive);
2377
2378                 my $volid = $drive->{file};
2379                 return if !$volid || $volid =~ m|^/|;
2380
2381                 die "base volume '$volid' is still in use by linked cloned\n"
2382                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2383
2384         });
2385     }
2386
2387     my $volids = {};
2388     my $remove_owned_drive = sub {
2389         my ($ds, $drive) = @_;
2390         return if drive_is_cdrom($drive, 1);
2391
2392         my $volid = $drive->{file};
2393         return if !$volid || $volid =~ m|^/|;
2394         return if $volids->{$volid};
2395
2396         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2397         return if !$path || !$owner || ($owner != $vmid);
2398
2399         $volids->{$volid} = 1;
2400         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2401         warn "Could not remove disk '$volid', check manually: $@" if $@;
2402     };
2403
2404     # only remove disks owned by this VM (referenced in the config)
2405     my $include_opts = {
2406         include_unused => 1,
2407         extra_keys => ['vmstate'],
2408     };
2409     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2410
2411     for my $snap (values %{$conf->{snapshots}}) {
2412         next if !defined($snap->{vmstate});
2413         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2414         next if !defined($drive);
2415         $remove_owned_drive->('vmstate', $drive);
2416     }
2417
2418     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2419
2420     if ($purge_unreferenced) { # also remove unreferenced disk
2421         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2422         PVE::Storage::foreach_volid($vmdisks, sub {
2423             my ($volid, $sid, $volname, $d) = @_;
2424             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2425             warn $@ if $@;
2426         });
2427     }
2428
2429     if (defined $replacement_conf) {
2430         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2431     } else {
2432         PVE::QemuConfig->destroy_config($vmid);
2433     }
2434 }
2435
2436 sub parse_vm_config {
2437     my ($filename, $raw, $strict) = @_;
2438
2439     return if !defined($raw);
2440
2441     my $res = {
2442         digest => Digest::SHA::sha1_hex($raw),
2443         snapshots => {},
2444         pending => {},
2445         cloudinit => {},
2446     };
2447
2448     my $handle_error = sub {
2449         my ($msg) = @_;
2450
2451         if ($strict) {
2452             die $msg;
2453         } else {
2454             warn $msg;
2455         }
2456     };
2457
2458     $filename =~ m|/qemu-server/(\d+)\.conf$|
2459         || die "got strange filename '$filename'";
2460
2461     my $vmid = $1;
2462
2463     my $conf = $res;
2464     my $descr;
2465     my $finish_description = sub {
2466         if (defined($descr)) {
2467             $descr =~ s/\s+$//;
2468             $conf->{description} = $descr;
2469         }
2470         $descr = undef;
2471     };
2472     my $section = '';
2473
2474     my @lines = split(/\n/, $raw);
2475     foreach my $line (@lines) {
2476         next if $line =~ m/^\s*$/;
2477
2478         if ($line =~ m/^\[PENDING\]\s*$/i) {
2479             $section = 'pending';
2480             $finish_description->();
2481             $conf = $res->{$section} = {};
2482             next;
2483         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2484             $section = 'cloudinit';
2485             $finish_description->();
2486             $conf = $res->{$section} = {};
2487             next;
2488
2489         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2490             $section = $1;
2491             $finish_description->();
2492             $conf = $res->{snapshots}->{$section} = {};
2493             next;
2494         }
2495
2496         if ($line =~ m/^\#(.*)$/) {
2497             $descr = '' if !defined($descr);
2498             $descr .= PVE::Tools::decode_text($1) . "\n";
2499             next;
2500         }
2501
2502         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2503             $descr = '' if !defined($descr);
2504             $descr .= PVE::Tools::decode_text($2);
2505         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2506             $conf->{snapstate} = $1;
2507         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2508             my $key = $1;
2509             my $value = $2;
2510             $conf->{$key} = $value;
2511         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2512             my $value = $1;
2513             if ($section eq 'pending') {
2514                 $conf->{delete} = $value; # we parse this later
2515             } else {
2516                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2517             }
2518         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2519             my $key = $1;
2520             my $value = $2;
2521             if ($section eq 'cloudinit') {
2522                 # ignore validation only used for informative purpose
2523                 $conf->{$key} = $value;
2524                 next;
2525             }
2526             eval { $value = check_type($key, $value); };
2527             if ($@) {
2528                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2529             } else {
2530                 $key = 'ide2' if $key eq 'cdrom';
2531                 my $fmt = $confdesc->{$key}->{format};
2532                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2533                     my $v = parse_drive($key, $value);
2534                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2535                         $v->{file} = $volid;
2536                         $value = print_drive($v);
2537                     } else {
2538                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2539                         next;
2540                     }
2541                 }
2542
2543                 $conf->{$key} = $value;
2544             }
2545         } else {
2546             $handle_error->("vm $vmid - unable to parse config: $line\n");
2547         }
2548     }
2549
2550     $finish_description->();
2551     delete $res->{snapstate}; # just to be sure
2552
2553     return $res;
2554 }
2555
2556 sub write_vm_config {
2557     my ($filename, $conf) = @_;
2558
2559     delete $conf->{snapstate}; # just to be sure
2560
2561     if ($conf->{cdrom}) {
2562         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2563         $conf->{ide2} = $conf->{cdrom};
2564         delete $conf->{cdrom};
2565     }
2566
2567     # we do not use 'smp' any longer
2568     if ($conf->{sockets}) {
2569         delete $conf->{smp};
2570     } elsif ($conf->{smp}) {
2571         $conf->{sockets} = $conf->{smp};
2572         delete $conf->{cores};
2573         delete $conf->{smp};
2574     }
2575
2576     my $used_volids = {};
2577
2578     my $cleanup_config = sub {
2579         my ($cref, $pending, $snapname) = @_;
2580
2581         foreach my $key (keys %$cref) {
2582             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2583                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2584             my $value = $cref->{$key};
2585             if ($key eq 'delete') {
2586                 die "propertry 'delete' is only allowed in [PENDING]\n"
2587                     if !$pending;
2588                 # fixme: check syntax?
2589                 next;
2590             }
2591             eval { $value = check_type($key, $value); };
2592             die "unable to parse value of '$key' - $@" if $@;
2593
2594             $cref->{$key} = $value;
2595
2596             if (!$snapname && is_valid_drivename($key)) {
2597                 my $drive = parse_drive($key, $value);
2598                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2599             }
2600         }
2601     };
2602
2603     &$cleanup_config($conf);
2604
2605     &$cleanup_config($conf->{pending}, 1);
2606
2607     foreach my $snapname (keys %{$conf->{snapshots}}) {
2608         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2609         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2610     }
2611
2612     # remove 'unusedX' settings if we re-add a volume
2613     foreach my $key (keys %$conf) {
2614         my $value = $conf->{$key};
2615         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2616             delete $conf->{$key};
2617         }
2618     }
2619
2620     my $generate_raw_config = sub {
2621         my ($conf, $pending) = @_;
2622
2623         my $raw = '';
2624
2625         # add description as comment to top of file
2626         if (defined(my $descr = $conf->{description})) {
2627             if ($descr) {
2628                 foreach my $cl (split(/\n/, $descr)) {
2629                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2630                 }
2631             } else {
2632                 $raw .= "#\n" if $pending;
2633             }
2634         }
2635
2636         foreach my $key (sort keys %$conf) {
2637             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2638             $raw .= "$key: $conf->{$key}\n";
2639         }
2640         return $raw;
2641     };
2642
2643     my $raw = &$generate_raw_config($conf);
2644
2645     if (scalar(keys %{$conf->{pending}})){
2646         $raw .= "\n[PENDING]\n";
2647         $raw .= &$generate_raw_config($conf->{pending}, 1);
2648     }
2649
2650     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2651         $raw .= "\n[special:cloudinit]\n";
2652         $raw .= &$generate_raw_config($conf->{cloudinit});
2653     }
2654
2655     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2656         $raw .= "\n[$snapname]\n";
2657         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2658     }
2659
2660     return $raw;
2661 }
2662
2663 sub load_defaults {
2664
2665     my $res = {};
2666
2667     # we use static defaults from our JSON schema configuration
2668     foreach my $key (keys %$confdesc) {
2669         if (defined(my $default = $confdesc->{$key}->{default})) {
2670             $res->{$key} = $default;
2671         }
2672     }
2673
2674     return $res;
2675 }
2676
2677 sub config_list {
2678     my $vmlist = PVE::Cluster::get_vmlist();
2679     my $res = {};
2680     return $res if !$vmlist || !$vmlist->{ids};
2681     my $ids = $vmlist->{ids};
2682     my $nodename = nodename();
2683
2684     foreach my $vmid (keys %$ids) {
2685         my $d = $ids->{$vmid};
2686         next if !$d->{node} || $d->{node} ne $nodename;
2687         next if !$d->{type} || $d->{type} ne 'qemu';
2688         $res->{$vmid}->{exists} = 1;
2689     }
2690     return $res;
2691 }
2692
2693 # test if VM uses local resources (to prevent migration)
2694 sub check_local_resources {
2695     my ($conf, $noerr) = @_;
2696
2697     my @loc_res = ();
2698
2699     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2700     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2701
2702     push @loc_res, "ivshmem" if $conf->{ivshmem};
2703
2704     foreach my $k (keys %$conf) {
2705         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2706         # sockets are safe: they will recreated be on the target side post-migrate
2707         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2708         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2709     }
2710
2711     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2712
2713     return \@loc_res;
2714 }
2715
2716 # check if used storages are available on all nodes (use by migrate)
2717 sub check_storage_availability {
2718     my ($storecfg, $conf, $node) = @_;
2719
2720     PVE::QemuConfig->foreach_volume($conf, sub {
2721         my ($ds, $drive) = @_;
2722
2723         my $volid = $drive->{file};
2724         return if !$volid;
2725
2726         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2727         return if !$sid;
2728
2729         # check if storage is available on both nodes
2730         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2731         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2732
2733         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2734
2735         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2736             if !$scfg->{content}->{$vtype};
2737    });
2738 }
2739
2740 # list nodes where all VM images are available (used by has_feature API)
2741 sub shared_nodes {
2742     my ($conf, $storecfg) = @_;
2743
2744     my $nodelist = PVE::Cluster::get_nodelist();
2745     my $nodehash = { map { $_ => 1 } @$nodelist };
2746     my $nodename = nodename();
2747
2748     PVE::QemuConfig->foreach_volume($conf, sub {
2749         my ($ds, $drive) = @_;
2750
2751         my $volid = $drive->{file};
2752         return if !$volid;
2753
2754         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2755         if ($storeid) {
2756             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2757             if ($scfg->{disable}) {
2758                 $nodehash = {};
2759             } elsif (my $avail = $scfg->{nodes}) {
2760                 foreach my $node (keys %$nodehash) {
2761                     delete $nodehash->{$node} if !$avail->{$node};
2762                 }
2763             } elsif (!$scfg->{shared}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if $node ne $nodename
2766                 }
2767             }
2768         }
2769     });
2770
2771     return $nodehash
2772 }
2773
2774 sub check_local_storage_availability {
2775     my ($conf, $storecfg) = @_;
2776
2777     my $nodelist = PVE::Cluster::get_nodelist();
2778     my $nodehash = { map { $_ => {} } @$nodelist };
2779
2780     PVE::QemuConfig->foreach_volume($conf, sub {
2781         my ($ds, $drive) = @_;
2782
2783         my $volid = $drive->{file};
2784         return if !$volid;
2785
2786         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2787         if ($storeid) {
2788             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2789
2790             if ($scfg->{disable}) {
2791                 foreach my $node (keys %$nodehash) {
2792                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2793                 }
2794             } elsif (my $avail = $scfg->{nodes}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     if (!$avail->{$node}) {
2797                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2798                     }
2799                 }
2800             }
2801         }
2802     });
2803
2804     foreach my $node (values %$nodehash) {
2805         if (my $unavail = $node->{unavailable_storages}) {
2806             $node->{unavailable_storages} = [ sort keys %$unavail ];
2807         }
2808     }
2809
2810     return $nodehash
2811 }
2812
2813 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2814 sub check_running {
2815     my ($vmid, $nocheck, $node) = @_;
2816
2817     # $nocheck is set when called during a migration, in which case the config
2818     # file might still or already reside on the *other* node
2819     # - because rename has already happened, and current node is source
2820     # - because rename hasn't happened yet, and current node is target
2821     # - because rename has happened, current node is target, but hasn't yet
2822     # processed it yet
2823     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2824     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2825 }
2826
2827 sub vzlist {
2828
2829     my $vzlist = config_list();
2830
2831     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2832
2833     while (defined(my $de = $fd->read)) {
2834         next if $de !~ m/^(\d+)\.pid$/;
2835         my $vmid = $1;
2836         next if !defined($vzlist->{$vmid});
2837         if (my $pid = check_running($vmid)) {
2838             $vzlist->{$vmid}->{pid} = $pid;
2839         }
2840     }
2841
2842     return $vzlist;
2843 }
2844
2845 our $vmstatus_return_properties = {
2846     vmid => get_standard_option('pve-vmid'),
2847     status => {
2848         description => "QEMU process status.",
2849         type => 'string',
2850         enum => ['stopped', 'running'],
2851     },
2852     maxmem => {
2853         description => "Maximum memory in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     maxdisk => {
2859         description => "Root disk size in bytes.",
2860         type => 'integer',
2861         optional => 1,
2862         renderer => 'bytes',
2863     },
2864     name => {
2865         description => "VM name.",
2866         type => 'string',
2867         optional => 1,
2868     },
2869     qmpstatus => {
2870         description => "QEMU QMP agent status.",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     pid => {
2875         description => "PID of running qemu process.",
2876         type => 'integer',
2877         optional => 1,
2878     },
2879     uptime => {
2880         description => "Uptime.",
2881         type => 'integer',
2882         optional => 1,
2883         renderer => 'duration',
2884     },
2885     cpus => {
2886         description => "Maximum usable CPUs.",
2887         type => 'number',
2888         optional => 1,
2889     },
2890     lock => {
2891         description => "The current config lock, if any.",
2892         type => 'string',
2893         optional => 1,
2894     },
2895     tags => {
2896         description  => "The current configured tags, if any",
2897         type => 'string',
2898         optional => 1,
2899     },
2900     'running-machine' => {
2901         description => "The currently running machine type (if running).",
2902         type => 'string',
2903         optional => 1,
2904     },
2905     'running-qemu' => {
2906         description => "The currently running QEMU version (if running).",
2907         type => 'string',
2908         optional => 1,
2909     },
2910 };
2911
2912 my $last_proc_pid_stat;
2913
2914 # get VM status information
2915 # This must be fast and should not block ($full == false)
2916 # We only query KVM using QMP if $full == true (this can be slow)
2917 sub vmstatus {
2918     my ($opt_vmid, $full) = @_;
2919
2920     my $res = {};
2921
2922     my $storecfg = PVE::Storage::config();
2923
2924     my $list = vzlist();
2925     my $defaults = load_defaults();
2926
2927     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2928
2929     my $cpucount = $cpuinfo->{cpus} || 1;
2930
2931     foreach my $vmid (keys %$list) {
2932         next if $opt_vmid && ($vmid ne $opt_vmid);
2933
2934         my $conf = PVE::QemuConfig->load_config($vmid);
2935
2936         my $d = { vmid => int($vmid) };
2937         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2938
2939         # fixme: better status?
2940         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2941
2942         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2943         if (defined($size)) {
2944             $d->{disk} = 0; # no info available
2945             $d->{maxdisk} = $size;
2946         } else {
2947             $d->{disk} = 0;
2948             $d->{maxdisk} = 0;
2949         }
2950
2951         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2952             * ($conf->{cores} || $defaults->{cores});
2953         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2954         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2955
2956         $d->{name} = $conf->{name} || "VM $vmid";
2957         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2958             : $defaults->{memory}*(1024*1024);
2959
2960         if ($conf->{balloon}) {
2961             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2962             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2963                 : $defaults->{shares};
2964         }
2965
2966         $d->{uptime} = 0;
2967         $d->{cpu} = 0;
2968         $d->{mem} = 0;
2969
2970         $d->{netout} = 0;
2971         $d->{netin} = 0;
2972
2973         $d->{diskread} = 0;
2974         $d->{diskwrite} = 0;
2975
2976         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2977
2978         $d->{serial} = 1 if conf_has_serial($conf);
2979         $d->{lock} = $conf->{lock} if $conf->{lock};
2980         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2981
2982         $res->{$vmid} = $d;
2983     }
2984
2985     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2986     foreach my $dev (keys %$netdev) {
2987         next if $dev !~ m/^tap([1-9]\d*)i/;
2988         my $vmid = $1;
2989         my $d = $res->{$vmid};
2990         next if !$d;
2991
2992         $d->{netout} += $netdev->{$dev}->{receive};
2993         $d->{netin} += $netdev->{$dev}->{transmit};
2994
2995         if ($full) {
2996             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2997             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2998         }
2999
3000     }
3001
3002     my $ctime = gettimeofday;
3003
3004     foreach my $vmid (keys %$list) {
3005
3006         my $d = $res->{$vmid};
3007         my $pid = $d->{pid};
3008         next if !$pid;
3009
3010         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3011         next if !$pstat; # not running
3012
3013         my $used = $pstat->{utime} + $pstat->{stime};
3014
3015         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3016
3017         if ($pstat->{vsize}) {
3018             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3019         }
3020
3021         my $old = $last_proc_pid_stat->{$pid};
3022         if (!$old) {
3023             $last_proc_pid_stat->{$pid} = {
3024                 time => $ctime,
3025                 used => $used,
3026                 cpu => 0,
3027             };
3028             next;
3029         }
3030
3031         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3032
3033         if ($dtime > 1000) {
3034             my $dutime = $used -  $old->{used};
3035
3036             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3037             $last_proc_pid_stat->{$pid} = {
3038                 time => $ctime,
3039                 used => $used,
3040                 cpu => $d->{cpu},
3041             };
3042         } else {
3043             $d->{cpu} = $old->{cpu};
3044         }
3045     }
3046
3047     return $res if !$full;
3048
3049     my $qmpclient = PVE::QMPClient->new();
3050
3051     my $ballooncb = sub {
3052         my ($vmid, $resp) = @_;
3053
3054         my $info = $resp->{'return'};
3055         return if !$info->{max_mem};
3056
3057         my $d = $res->{$vmid};
3058
3059         # use memory assigned to VM
3060         $d->{maxmem} = $info->{max_mem};
3061         $d->{balloon} = $info->{actual};
3062
3063         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3064             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3065             $d->{freemem} = $info->{free_mem};
3066         }
3067
3068         $d->{ballooninfo} = $info;
3069     };
3070
3071     my $blockstatscb = sub {
3072         my ($vmid, $resp) = @_;
3073         my $data = $resp->{'return'} || [];
3074         my $totalrdbytes = 0;
3075         my $totalwrbytes = 0;
3076
3077         for my $blockstat (@$data) {
3078             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3079             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3080
3081             $blockstat->{device} =~ s/drive-//;
3082             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3083         }
3084         $res->{$vmid}->{diskread} = $totalrdbytes;
3085         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3086     };
3087
3088     my $machinecb = sub {
3089         my ($vmid, $resp) = @_;
3090         my $data = $resp->{'return'} || [];
3091
3092         $res->{$vmid}->{'running-machine'} =
3093             PVE::QemuServer::Machine::current_from_query_machines($data);
3094     };
3095
3096     my $versioncb = sub {
3097         my ($vmid, $resp) = @_;
3098         my $data = $resp->{'return'} // {};
3099         my $version = 'unknown';
3100
3101         if (my $v = $data->{qemu}) {
3102             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3103         }
3104
3105         $res->{$vmid}->{'running-qemu'} = $version;
3106     };
3107
3108     my $statuscb = sub {
3109         my ($vmid, $resp) = @_;
3110
3111         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3112         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3113         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3114         # this fails if ballon driver is not loaded, so this must be
3115         # the last commnand (following command are aborted if this fails).
3116         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3117
3118         my $status = 'unknown';
3119         if (!defined($status = $resp->{'return'}->{status})) {
3120             warn "unable to get VM status\n";
3121             return;
3122         }
3123
3124         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3125     };
3126
3127     foreach my $vmid (keys %$list) {
3128         next if $opt_vmid && ($vmid ne $opt_vmid);
3129         next if !$res->{$vmid}->{pid}; # not running
3130         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3131     }
3132
3133     $qmpclient->queue_execute(undef, 2);
3134
3135     foreach my $vmid (keys %$list) {
3136         next if $opt_vmid && ($vmid ne $opt_vmid);
3137         next if !$res->{$vmid}->{pid}; #not running
3138
3139         # we can't use the $qmpclient since it might have already aborted on
3140         # 'query-balloon', but this might also fail for older versions...
3141         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3142         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3143     }
3144
3145     foreach my $vmid (keys %$list) {
3146         next if $opt_vmid && ($vmid ne $opt_vmid);
3147         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3148     }
3149
3150     return $res;
3151 }
3152
3153 sub conf_has_serial {
3154     my ($conf) = @_;
3155
3156     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3157         if ($conf->{"serial$i"}) {
3158             return 1;
3159         }
3160     }
3161
3162     return 0;
3163 }
3164
3165 sub conf_has_audio {
3166     my ($conf, $id) = @_;
3167
3168     $id //= 0;
3169     my $audio = $conf->{"audio$id"};
3170     return if !defined($audio);
3171
3172     my $audioproperties = parse_property_string($audio_fmt, $audio);
3173     my $audiodriver = $audioproperties->{driver} // 'spice';
3174
3175     return {
3176         dev => $audioproperties->{device},
3177         dev_id => "audiodev$id",
3178         backend => $audiodriver,
3179         backend_id => "$audiodriver-backend${id}",
3180     };
3181 }
3182
3183 sub audio_devs {
3184     my ($audio, $audiopciaddr, $machine_version) = @_;
3185
3186     my $devs = [];
3187
3188     my $id = $audio->{dev_id};
3189     my $audiodev = "";
3190     if (min_version($machine_version, 4, 2)) {
3191         $audiodev = ",audiodev=$audio->{backend_id}";
3192     }
3193
3194     if ($audio->{dev} eq 'AC97') {
3195         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3196     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3197         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3198         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3199         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3200     } else {
3201         die "unkown audio device '$audio->{dev}', implement me!";
3202     }
3203
3204     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3205
3206     return $devs;
3207 }
3208
3209 sub get_tpm_paths {
3210     my ($vmid) = @_;
3211     return {
3212         socket => "/var/run/qemu-server/$vmid.swtpm",
3213         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3214     };
3215 }
3216
3217 sub add_tpm_device {
3218     my ($vmid, $devices, $conf) = @_;
3219
3220     return if !$conf->{tpmstate0};
3221
3222     my $paths = get_tpm_paths($vmid);
3223
3224     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3225     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3226     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3227 }
3228
3229 sub start_swtpm {
3230     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3231
3232     return if !$tpmdrive;
3233
3234     my $state;
3235     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3236     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3237     if ($storeid) {
3238         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3239     } else {
3240         $state = $tpm->{file};
3241     }
3242
3243     my $paths = get_tpm_paths($vmid);
3244
3245     # during migration, we will get state from remote
3246     #
3247     if (!$migration) {
3248         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3249         my $setup_cmd = [
3250             "swtpm_setup",
3251             "--tpmstate",
3252             "file://$state",
3253             "--createek",
3254             "--create-ek-cert",
3255             "--create-platform-cert",
3256             "--lock-nvram",
3257             "--config",
3258             "/etc/swtpm_setup.conf", # do not use XDG configs
3259             "--runas",
3260             "0", # force creation as root, error if not possible
3261             "--not-overwrite", # ignore existing state, do not modify
3262         ];
3263
3264         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3265         # TPM 2.0 supports ECC crypto, use if possible
3266         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3267
3268         run_command($setup_cmd, outfunc => sub {
3269             print "swtpm_setup: $1\n";
3270         });
3271     }
3272
3273     # Used to distinguish different invocations in the log.
3274     my $log_prefix = "[id=" . int(time()) . "] ";
3275
3276     my $emulator_cmd = [
3277         "swtpm",
3278         "socket",
3279         "--tpmstate",
3280         "backend-uri=file://$state,mode=0600",
3281         "--ctrl",
3282         "type=unixio,path=$paths->{socket},mode=0600",
3283         "--pid",
3284         "file=$paths->{pid}",
3285         "--terminate", # terminate on QEMU disconnect
3286         "--daemon",
3287         "--log",
3288         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3289     ];
3290     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3291     run_command($emulator_cmd, outfunc => sub { print $1; });
3292
3293     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3294     while (! -e $paths->{pid}) {
3295         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3296         usleep(50_000);
3297     }
3298
3299     # return untainted PID of swtpm daemon so it can be killed on error
3300     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3301     return $1;
3302 }
3303
3304 sub vga_conf_has_spice {
3305     my ($vga) = @_;
3306
3307     my $vgaconf = parse_vga($vga);
3308     my $vgatype = $vgaconf->{type};
3309     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3310
3311     return $1 || 1;
3312 }
3313
3314 sub is_native($) {
3315     my ($arch) = @_;
3316     return get_host_arch() eq $arch;
3317 }
3318
3319 sub get_vm_arch {
3320     my ($conf) = @_;
3321     return $conf->{arch} // get_host_arch();
3322 }
3323
3324 my $default_machines = {
3325     x86_64 => 'pc',
3326     aarch64 => 'virt',
3327 };
3328
3329 sub get_installed_machine_version {
3330     my ($kvmversion) = @_;
3331     $kvmversion = kvm_user_version() if !defined($kvmversion);
3332     $kvmversion =~ m/^(\d+\.\d+)/;
3333     return $1;
3334 }
3335
3336 sub windows_get_pinned_machine_version {
3337     my ($machine, $base_version, $kvmversion) = @_;
3338
3339     my $pin_version = $base_version;
3340     if (!defined($base_version) ||
3341         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3342     ) {
3343         $pin_version = get_installed_machine_version($kvmversion);
3344     }
3345     if (!$machine || $machine eq 'pc') {
3346         $machine = "pc-i440fx-$pin_version";
3347     } elsif ($machine eq 'q35') {
3348         $machine = "pc-q35-$pin_version";
3349     } elsif ($machine eq 'virt') {
3350         $machine = "virt-$pin_version";
3351     } else {
3352         warn "unknown machine type '$machine', not touching that!\n";
3353     }
3354
3355     return $machine;
3356 }
3357
3358 sub get_vm_machine {
3359     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3360
3361     my $machine = $forcemachine || $conf->{machine};
3362
3363     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3364         $kvmversion //= kvm_user_version();
3365         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3366         # layout which confuses windows quite a bit and may result in various regressions..
3367         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3368         if (windows_version($conf->{ostype})) {
3369             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3370         }
3371         $arch //= 'x86_64';
3372         $machine ||= $default_machines->{$arch};
3373         if ($add_pve_version) {
3374             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3375             $machine .= "+pve$pvever";
3376         }
3377     }
3378
3379     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3380         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3381         $machine = $1 if $is_pxe;
3382
3383         # for version-pinned machines that do not include a pve-version (e.g.
3384         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3385         $machine .= '+pve0';
3386
3387         $machine .= '.pxe' if $is_pxe;
3388     }
3389
3390     return $machine;
3391 }
3392
3393 sub get_ovmf_files($$$) {
3394     my ($arch, $efidisk, $smm) = @_;
3395
3396     my $types = $OVMF->{$arch}
3397         or die "no OVMF images known for architecture '$arch'\n";
3398
3399     my $type = 'default';
3400     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3401         $type = $smm ? "4m" : "4m-no-smm";
3402         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3403     }
3404
3405     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3406     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3407     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3408
3409     return ($ovmf_code, $ovmf_vars);
3410 }
3411
3412 my $Arch2Qemu = {
3413     aarch64 => '/usr/bin/qemu-system-aarch64',
3414     x86_64 => '/usr/bin/qemu-system-x86_64',
3415 };
3416 sub get_command_for_arch($) {
3417     my ($arch) = @_;
3418     return '/usr/bin/kvm' if is_native($arch);
3419
3420     my $cmd = $Arch2Qemu->{$arch}
3421         or die "don't know how to emulate architecture '$arch'\n";
3422     return $cmd;
3423 }
3424
3425 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3426 # to use in a QEMU command line (-cpu element), first array_intersect the result
3427 # of query_supported_ with query_understood_. This is necessary because:
3428 #
3429 # a) query_understood_ returns flags the host cannot use and
3430 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3431 #    flags, but CPU settings - with most of them being flags. Those settings
3432 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3433 #
3434 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3435 # expensive. If you need the value returned from this, you can get it much
3436 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3437 # $accel being 'kvm' or 'tcg'.
3438 #
3439 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3440 # changes, automatically populating pmxcfs.
3441 #
3442 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3443 # since kvm and tcg machines support different flags
3444 #
3445 sub query_supported_cpu_flags {
3446     my ($arch) = @_;
3447
3448     $arch //= get_host_arch();
3449     my $default_machine = $default_machines->{$arch};
3450
3451     my $flags = {};
3452
3453     # FIXME: Once this is merged, the code below should work for ARM as well:
3454     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3455     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3456         $arch eq "aarch64";
3457
3458     my $kvm_supported = defined(kvm_version());
3459     my $qemu_cmd = get_command_for_arch($arch);
3460     my $fakevmid = -1;
3461     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3462
3463     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3464     my $query_supported_run_qemu = sub {
3465         my ($kvm) = @_;
3466
3467         my $flags = {};
3468         my $cmd = [
3469             $qemu_cmd,
3470             '-machine', $default_machine,
3471             '-display', 'none',
3472             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3473             '-mon', 'chardev=qmp,mode=control',
3474             '-pidfile', $pidfile,
3475             '-S', '-daemonize'
3476         ];
3477
3478         if (!$kvm) {
3479             push @$cmd, '-accel', 'tcg';
3480         }
3481
3482         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3483         die "QEMU flag querying VM exited with code " . $rc if $rc;
3484
3485         eval {
3486             my $cmd_result = mon_cmd(
3487                 $fakevmid,
3488                 'query-cpu-model-expansion',
3489                 type => 'full',
3490                 model => { name => 'host' }
3491             );
3492
3493             my $props = $cmd_result->{model}->{props};
3494             foreach my $prop (keys %$props) {
3495                 next if $props->{$prop} ne '1';
3496                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3497                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3498                 # We only keep those with underscores, to match /proc/cpuinfo
3499                 $prop =~ s/\.|-/_/g;
3500                 $flags->{$prop} = 1;
3501             }
3502         };
3503         my $err = $@;
3504
3505         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3506         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3507
3508         die $err if $err;
3509
3510         return [ sort keys %$flags ];
3511     };
3512
3513     # We need to query QEMU twice, since KVM and TCG have different supported flags
3514     PVE::QemuConfig->lock_config($fakevmid, sub {
3515         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3516         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3517
3518         if ($kvm_supported) {
3519             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3520             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3521         }
3522     });
3523
3524     return $flags;
3525 }
3526
3527 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3528 my $understood_cpu_flag_dir = "/usr/share/kvm";
3529 sub query_understood_cpu_flags {
3530     my $arch = get_host_arch();
3531     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3532
3533     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3534         if ! -e $filepath;
3535
3536     my $raw = file_get_contents($filepath);
3537     $raw =~ s/^\s+|\s+$//g;
3538     my @flags = split(/\s+/, $raw);
3539
3540     return \@flags;
3541 }
3542
3543 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3544 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3545 my sub should_disable_smm {
3546     my ($conf, $vga) = @_;
3547
3548     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3549         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3550 }
3551
3552 my sub print_ovmf_drive_commandlines {
3553     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3554
3555     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3556
3557     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3558
3559     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3560     if ($d) {
3561         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3562         my ($path, $format) = $d->@{'file', 'format'};
3563         if ($storeid) {
3564             $path = PVE::Storage::path($storecfg, $d->{file});
3565             if (!defined($format)) {
3566                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3567                 $format = qemu_img_format($scfg, $volname);
3568             }
3569         } elsif (!defined($format)) {
3570             die "efidisk format must be specified\n";
3571         }
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         if ($path =~ m/^rbd:/) {
3574             $var_drive_str .= ',cache=writeback';
3575             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3576         }
3577         $var_drive_str .= ",format=$format,file=$path";
3578
3579         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3580         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3581     } else {
3582         log_warn("no efidisk configured! Using temporary efivars disk.");
3583         my $path = "/tmp/$vmid-ovmf.fd";
3584         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3585         $var_drive_str .= ",format=raw,file=$path";
3586         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3587     }
3588
3589     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3590 }
3591
3592 sub config_to_command {
3593     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3594         $pbs_backing) = @_;
3595
3596     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3597     my $devices = [];
3598     my $bridges = {};
3599     my $ostype = $conf->{ostype};
3600     my $winversion = windows_version($ostype);
3601     my $kvm = $conf->{kvm};
3602     my $nodename = nodename();
3603
3604     my $arch = get_vm_arch($conf);
3605     my $kvm_binary = get_command_for_arch($arch);
3606     my $kvmver = kvm_user_version($kvm_binary);
3607
3608     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3609         $kvmver //= "undefined";
3610         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3611     }
3612
3613     my $add_pve_version = min_version($kvmver, 4, 1);
3614
3615     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3616     my $machine_version = extract_version($machine_type, $kvmver);
3617     $kvm //= 1 if is_native($arch);
3618
3619     $machine_version =~ m/(\d+)\.(\d+)/;
3620     my ($machine_major, $machine_minor) = ($1, $2);
3621
3622     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3623         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3624     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3625         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3626             ." please upgrade node '$nodename'\n"
3627     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3628         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3629         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3630             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3631             ." node '$nodename'\n";
3632     }
3633
3634     # if a specific +pve version is required for a feature, use $version_guard
3635     # instead of min_version to allow machines to be run with the minimum
3636     # required version
3637     my $required_pve_version = 0;
3638     my $version_guard = sub {
3639         my ($major, $minor, $pve) = @_;
3640         return 0 if !min_version($machine_version, $major, $minor, $pve);
3641         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3642         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3643         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3644         return 1;
3645     };
3646
3647     if ($kvm && !defined kvm_version()) {
3648         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3649             ." or enable in BIOS.\n";
3650     }
3651
3652     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3653     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3654     my $use_old_bios_files = undef;
3655     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3656
3657     my $cmd = [];
3658     if ($conf->{affinity}) {
3659         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3660     }
3661
3662     push @$cmd, $kvm_binary;
3663
3664     push @$cmd, '-id', $vmid;
3665
3666     my $vmname = $conf->{name} || "vm$vmid";
3667
3668     push @$cmd, '-name', "$vmname,debug-threads=on";
3669
3670     push @$cmd, '-no-shutdown';
3671
3672     my $use_virtio = 0;
3673
3674     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3675     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3676     push @$cmd, '-mon', "chardev=qmp,mode=control";
3677
3678     if (min_version($machine_version, 2, 12)) {
3679         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3680         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3681     }
3682
3683     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3684
3685     push @$cmd, '-daemonize';
3686
3687     if ($conf->{smbios1}) {
3688         my $smbios_conf = parse_smbios1($conf->{smbios1});
3689         if ($smbios_conf->{base64}) {
3690             # Do not pass base64 flag to qemu
3691             delete $smbios_conf->{base64};
3692             my $smbios_string = "";
3693             foreach my $key (keys %$smbios_conf) {
3694                 my $value;
3695                 if ($key eq "uuid") {
3696                     $value = $smbios_conf->{uuid}
3697                 } else {
3698                     $value = decode_base64($smbios_conf->{$key});
3699                 }
3700                 # qemu accepts any binary data, only commas need escaping by double comma
3701                 $value =~ s/,/,,/g;
3702                 $smbios_string .= "," . $key . "=" . $value if $value;
3703             }
3704             push @$cmd, '-smbios', "type=1" . $smbios_string;
3705         } else {
3706             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3707         }
3708     }
3709
3710     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3711         my ($code_drive_str, $var_drive_str) =
3712             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3713         push $cmd->@*, '-drive', $code_drive_str;
3714         push $cmd->@*, '-drive', $var_drive_str;
3715     }
3716
3717     if ($q35) { # tell QEMU to load q35 config early
3718         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719         if (min_version($machine_version, 4, 0)) {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721         } else {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723         }
3724     }
3725
3726     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3727         push @$cmd, $fixups->@*;
3728     }
3729
3730     if ($conf->{vmgenid}) {
3731         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3732     }
3733
3734     # add usb controllers
3735     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3736         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3737     push @$devices, @usbcontrollers if @usbcontrollers;
3738     my $vga = parse_vga($conf->{vga});
3739
3740     my $qxlnum = vga_conf_has_spice($conf->{vga});
3741     $vga->{type} = 'qxl' if $qxlnum;
3742
3743     if (!$vga->{type}) {
3744         if ($arch eq 'aarch64') {
3745             $vga->{type} = 'virtio';
3746         } elsif (min_version($machine_version, 2, 9)) {
3747             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3748         } else {
3749             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3750         }
3751     }
3752
3753     # enable absolute mouse coordinates (needed by vnc)
3754     my $tablet = $conf->{tablet};
3755     if (!defined($tablet)) {
3756         $tablet = $defaults->{tablet};
3757         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3758         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3759     }
3760
3761     if ($tablet) {
3762         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3763         my $kbd = print_keyboarddevice_full($conf, $arch);
3764         push @$devices, '-device', $kbd if defined($kbd);
3765     }
3766
3767     my $bootorder = device_bootorder($conf);
3768
3769     # host pci device passthrough
3770     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3771         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3772
3773     # usb devices
3774     my $usb_dev_features = {};
3775     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3776
3777     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3778         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3779     push @$devices, @usbdevices if @usbdevices;
3780
3781     # serial devices
3782     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3783         my $path = $conf->{"serial$i"} or next;
3784         if ($path eq 'socket') {
3785             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3786             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3787             # On aarch64, serial0 is the UART device. QEMU only allows
3788             # connecting UART devices via the '-serial' command line, as
3789             # the device has a fixed slot on the hardware...
3790             if ($arch eq 'aarch64' && $i == 0) {
3791                 push @$devices, '-serial', "chardev:serial$i";
3792             } else {
3793                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3794             }
3795         } else {
3796             die "no such serial device\n" if ! -c $path;
3797             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3798             push @$devices, '-device', "isa-serial,chardev=serial$i";
3799         }
3800     }
3801
3802     # parallel devices
3803     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3804         if (my $path = $conf->{"parallel$i"}) {
3805             die "no such parallel device\n" if ! -c $path;
3806             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3807             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3808             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3809         }
3810     }
3811
3812     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3813         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3814         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3815         push @$devices, @$audio_devs;
3816     }
3817
3818     add_tpm_device($vmid, $devices, $conf);
3819
3820     my $sockets = 1;
3821     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3822     $sockets = $conf->{sockets} if  $conf->{sockets};
3823
3824     my $cores = $conf->{cores} || 1;
3825
3826     my $maxcpus = $sockets * $cores;
3827
3828     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3829
3830     my $allowed_vcpus = $cpuinfo->{cpus};
3831
3832     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3833
3834     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3835         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3836         for (my $i = 2; $i <= $vcpus; $i++)  {
3837             my $cpustr = print_cpu_device($conf,$i);
3838             push @$cmd, '-device', $cpustr;
3839         }
3840
3841     } else {
3842
3843         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3844     }
3845     push @$cmd, '-nodefaults';
3846
3847     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3848
3849     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3850
3851     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3852
3853     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3854         push @$devices, '-device', print_vga_device(
3855             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3856
3857         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3858
3859         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3860         push @$cmd,  '-vnc', "unix:$socket,password=on";
3861     } else {
3862         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3863         push @$cmd, '-nographic';
3864     }
3865
3866     # time drift fix
3867     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3868     my $useLocaltime = $conf->{localtime};
3869
3870     if ($winversion >= 5) { # windows
3871         $useLocaltime = 1 if !defined($conf->{localtime});
3872
3873         # use time drift fix when acpi is enabled
3874         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3875             $tdf = 1 if !defined($conf->{tdf});
3876         }
3877     }
3878
3879     if ($winversion >= 6) {
3880         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3881         push @$cmd, '-no-hpet';
3882     }
3883
3884     push @$rtcFlags, 'driftfix=slew' if $tdf;
3885
3886     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3887         push @$rtcFlags, "base=$conf->{startdate}";
3888     } elsif ($useLocaltime) {
3889         push @$rtcFlags, 'base=localtime';
3890     }
3891
3892     if ($forcecpu) {
3893         push @$cmd, '-cpu', $forcecpu;
3894     } else {
3895         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3896     }
3897
3898     PVE::QemuServer::Memory::config(
3899         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3900
3901     push @$cmd, '-S' if $conf->{freeze};
3902
3903     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3904
3905     my $guest_agent = parse_guest_agent($conf);
3906
3907     if ($guest_agent->{enabled}) {
3908         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3909         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3910
3911         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3912             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3913             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3914             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3915         } elsif ($guest_agent->{type} eq 'isa') {
3916             push @$devices, '-device', "isa-serial,chardev=qga0";
3917         }
3918     }
3919
3920     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3921     if ($rng && $version_guard->(4, 1, 2)) {
3922         check_rng_source($rng->{source});
3923
3924         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3925         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3926         my $limiter_str = "";
3927         if ($max_bytes) {
3928             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3929         }
3930
3931         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3932         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3933         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3934     }
3935
3936     my $spice_port;
3937
3938     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3939         if ($qxlnum > 1) {
3940             if ($winversion){
3941                 for (my $i = 1; $i < $qxlnum; $i++){
3942                     push @$devices, '-device', print_vga_device(
3943                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3944                 }
3945             } else {
3946                 # assume other OS works like Linux
3947                 my ($ram, $vram) = ("134217728", "67108864");
3948                 if ($vga->{memory}) {
3949                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3950                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3951                 }
3952                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3953                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3954             }
3955         }
3956
3957         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3958
3959         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3960         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3961         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3962
3963         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3964         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3965         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3966
3967         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3968         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3969
3970         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3971         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3972         if ($spice_enhancement->{foldersharing}) {
3973             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3974             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3975         }
3976
3977         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3978         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3979             if $spice_enhancement->{videostreaming};
3980
3981         push @$devices, '-spice', "$spice_opts";
3982     }
3983
3984     # enable balloon by default, unless explicitly disabled
3985     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3986         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3987         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3988         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3989         push @$devices, '-device', $ballooncmd;
3990     }
3991
3992     if ($conf->{watchdog}) {
3993         my $wdopts = parse_watchdog($conf->{watchdog});
3994         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3995         my $watchdog = $wdopts->{model} || 'i6300esb';
3996         push @$devices, '-device', "$watchdog$pciaddr";
3997         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3998     }
3999
4000     my $vollist = [];
4001     my $scsicontroller = {};
4002     my $ahcicontroller = {};
4003     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4004
4005     # Add iscsi initiator name if available
4006     if (my $initiator = get_initiator_name()) {
4007         push @$devices, '-iscsi', "initiator-name=$initiator";
4008     }
4009
4010     PVE::QemuConfig->foreach_volume($conf, sub {
4011         my ($ds, $drive) = @_;
4012
4013         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4014             check_volume_storage_type($storecfg, $drive->{file});
4015             push @$vollist, $drive->{file};
4016         }
4017
4018         # ignore efidisk here, already added in bios/fw handling code above
4019         return if $drive->{interface} eq 'efidisk';
4020         # similar for TPM
4021         return if $drive->{interface} eq 'tpmstate';
4022
4023         $use_virtio = 1 if $ds =~ m/^virtio/;
4024
4025         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4026
4027         if ($drive->{interface} eq 'virtio'){
4028            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4029         }
4030
4031         if ($drive->{interface} eq 'scsi') {
4032
4033             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4034
4035             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4036                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4037
4038             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4039             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4040
4041             my $iothread = '';
4042             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4043                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4044                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4045             } elsif ($drive->{iothread}) {
4046                 log_warn(
4047                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4048                 );
4049             }
4050
4051             my $queues = '';
4052             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4053                 $queues = ",num_queues=$drive->{queues}";
4054             }
4055
4056             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4057                 if !$scsicontroller->{$controller};
4058             $scsicontroller->{$controller}=1;
4059         }
4060
4061         if ($drive->{interface} eq 'sata') {
4062             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4063             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4064             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4065                 if !$ahcicontroller->{$controller};
4066             $ahcicontroller->{$controller}=1;
4067         }
4068
4069         my $pbs_conf = $pbs_backing->{$ds};
4070         my $pbs_name = undef;
4071         if ($pbs_conf) {
4072             $pbs_name = "drive-$ds-pbs";
4073             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4074         }
4075
4076         my $drive_cmd = print_drive_commandline_full(
4077             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4078
4079         # extra protection for templates, but SATA and IDE don't support it..
4080         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4081
4082         push @$devices, '-drive',$drive_cmd;
4083         push @$devices, '-device', print_drivedevice_full(
4084             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4085     });
4086
4087     for (my $i = 0; $i < $MAX_NETS; $i++) {
4088         my $netname = "net$i";
4089
4090         next if !$conf->{$netname};
4091         my $d = parse_net($conf->{$netname});
4092         next if !$d;
4093         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4094
4095         $use_virtio = 1 if $d->{model} eq 'virtio';
4096
4097         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4098
4099         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4100         push @$devices, '-netdev', $netdevfull;
4101
4102         my $netdevicefull = print_netdevice_full(
4103             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4104
4105         push @$devices, '-device', $netdevicefull;
4106     }
4107
4108     if ($conf->{ivshmem}) {
4109         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4110
4111         my $bus;
4112         if ($q35) {
4113             $bus = print_pcie_addr("ivshmem");
4114         } else {
4115             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4116         }
4117
4118         my $ivshmem_name = $ivshmem->{name} // $vmid;
4119         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4120
4121         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4122         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4123             .",size=$ivshmem->{size}M";
4124     }
4125
4126     # pci.4 is nested in pci.1
4127     $bridges->{1} = 1 if $bridges->{4};
4128
4129     if (!$q35) { # add pci bridges
4130         if (min_version($machine_version, 2, 3)) {
4131            $bridges->{1} = 1;
4132            $bridges->{2} = 1;
4133         }
4134         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4135     }
4136
4137     for my $k (sort {$b cmp $a} keys %$bridges) {
4138         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4139
4140         my $k_name = $k;
4141         if ($k == 2 && $legacy_igd) {
4142             $k_name = "$k-igd";
4143         }
4144         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4145         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4146
4147         if ($q35) { # add after -readconfig pve-q35.cfg
4148             splice @$devices, 2, 0, '-device', $devstr;
4149         } else {
4150             unshift @$devices, '-device', $devstr if $k > 0;
4151         }
4152     }
4153
4154     if (!$kvm) {
4155         push @$machineFlags, 'accel=tcg';
4156     }
4157
4158     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4159
4160     my $machine_type_min = $machine_type;
4161     if ($add_pve_version) {
4162         $machine_type_min =~ s/\+pve\d+$//;
4163         $machine_type_min .= "+pve$required_pve_version";
4164     }
4165     push @$machineFlags, "type=${machine_type_min}";
4166
4167     push @$cmd, @$devices;
4168     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4169     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4170     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4171
4172     if (my $vmstate = $conf->{vmstate}) {
4173         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4174         push @$vollist, $vmstate;
4175         push @$cmd, '-loadstate', $statepath;
4176         print "activating and using '$vmstate' as vmstate\n";
4177     }
4178
4179     if (PVE::QemuConfig->is_template($conf)) {
4180         # needed to workaround base volumes being read-only
4181         push @$cmd, '-snapshot';
4182     }
4183
4184     # add custom args
4185     if ($conf->{args}) {
4186         my $aa = PVE::Tools::split_args($conf->{args});
4187         push @$cmd, @$aa;
4188     }
4189
4190     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4191 }
4192
4193 sub check_rng_source {
4194     my ($source) = @_;
4195
4196     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4197     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4198         if ! -e $source;
4199
4200     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4201     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4202         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4203         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4204         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4205             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4206             ." to the host.\n";
4207     }
4208 }
4209
4210 sub spice_port {
4211     my ($vmid) = @_;
4212
4213     my $res = mon_cmd($vmid, 'query-spice');
4214
4215     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4216 }
4217
4218 sub vm_devices_list {
4219     my ($vmid) = @_;
4220
4221     my $res = mon_cmd($vmid, 'query-pci');
4222     my $devices_to_check = [];
4223     my $devices = {};
4224     foreach my $pcibus (@$res) {
4225         push @$devices_to_check, @{$pcibus->{devices}},
4226     }
4227
4228     while (@$devices_to_check) {
4229         my $to_check = [];
4230         for my $d (@$devices_to_check) {
4231             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4232             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4233
4234             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4235             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4236         }
4237         $devices_to_check = $to_check;
4238     }
4239
4240     my $resblock = mon_cmd($vmid, 'query-block');
4241     foreach my $block (@$resblock) {
4242         if($block->{device} =~ m/^drive-(\S+)/){
4243                 $devices->{$1} = 1;
4244         }
4245     }
4246
4247     my $resmice = mon_cmd($vmid, 'query-mice');
4248     foreach my $mice (@$resmice) {
4249         if ($mice->{name} eq 'QEMU HID Tablet') {
4250             $devices->{tablet} = 1;
4251             last;
4252         }
4253     }
4254
4255     # for usb devices there is no query-usb
4256     # but we can iterate over the entries in
4257     # qom-list path=/machine/peripheral
4258     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4259     foreach my $per (@$resperipheral) {
4260         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4261             $devices->{$per->{name}} = 1;
4262         }
4263     }
4264
4265     return $devices;
4266 }
4267
4268 sub vm_deviceplug {
4269     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4270
4271     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4272
4273     my $devices_list = vm_devices_list($vmid);
4274     return 1 if defined($devices_list->{$deviceid});
4275
4276     # add PCI bridge if we need it for the device
4277     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4278
4279     if ($deviceid eq 'tablet') {
4280         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4281     } elsif ($deviceid eq 'keyboard') {
4282         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4283     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4284         my $id = $1;
4285         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4286         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4287     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4288         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4289     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4290         qemu_iothread_add($vmid, $deviceid, $device);
4291
4292         qemu_driveadd($storecfg, $vmid, $device);
4293         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4294
4295         qemu_deviceadd($vmid, $devicefull);
4296         eval { qemu_deviceaddverify($vmid, $deviceid); };
4297         if (my $err = $@) {
4298             eval { qemu_drivedel($vmid, $deviceid); };
4299             warn $@ if $@;
4300             die $err;
4301         }
4302     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4303         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4304         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4305         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4306
4307         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4308
4309         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4310             qemu_iothread_add($vmid, $deviceid, $device);
4311             $devicefull .= ",iothread=iothread-$deviceid";
4312         }
4313
4314         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4315             $devicefull .= ",num_queues=$device->{queues}";
4316         }
4317
4318         qemu_deviceadd($vmid, $devicefull);
4319         qemu_deviceaddverify($vmid, $deviceid);
4320     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4321         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4322         qemu_driveadd($storecfg, $vmid, $device);
4323
4324         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4325         eval { qemu_deviceadd($vmid, $devicefull); };
4326         if (my $err = $@) {
4327             eval { qemu_drivedel($vmid, $deviceid); };
4328             warn $@ if $@;
4329             die $err;
4330         }
4331     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4332         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4333
4334         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4335         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4336         my $use_old_bios_files = undef;
4337         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4338
4339         my $netdevicefull = print_netdevice_full(
4340             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4341         qemu_deviceadd($vmid, $netdevicefull);
4342         eval {
4343             qemu_deviceaddverify($vmid, $deviceid);
4344             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4345         };
4346         if (my $err = $@) {
4347             eval { qemu_netdevdel($vmid, $deviceid); };
4348             warn $@ if $@;
4349             die $err;
4350         }
4351     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4352         my $bridgeid = $2;
4353         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4354         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4355
4356         qemu_deviceadd($vmid, $devicefull);
4357         qemu_deviceaddverify($vmid, $deviceid);
4358     } else {
4359         die "can't hotplug device '$deviceid'\n";
4360     }
4361
4362     return 1;
4363 }
4364
4365 # fixme: this should raise exceptions on error!
4366 sub vm_deviceunplug {
4367     my ($vmid, $conf, $deviceid) = @_;
4368
4369     my $devices_list = vm_devices_list($vmid);
4370     return 1 if !defined($devices_list->{$deviceid});
4371
4372     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4373     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4374
4375     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4376         qemu_devicedel($vmid, $deviceid);
4377     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4378         qemu_devicedel($vmid, $deviceid);
4379         qemu_devicedelverify($vmid, $deviceid);
4380     } elsif ($deviceid =~ m/^usb\d+$/) {
4381         qemu_devicedel($vmid, $deviceid);
4382         qemu_devicedelverify($vmid, $deviceid);
4383     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4384         my $device = parse_drive($deviceid, $conf->{$deviceid});
4385
4386         qemu_devicedel($vmid, $deviceid);
4387         qemu_devicedelverify($vmid, $deviceid);
4388         qemu_drivedel($vmid, $deviceid);
4389         qemu_iothread_del($vmid, $deviceid, $device);
4390     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4391         qemu_devicedel($vmid, $deviceid);
4392         qemu_devicedelverify($vmid, $deviceid);
4393     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4394         my $device = parse_drive($deviceid, $conf->{$deviceid});
4395
4396         qemu_devicedel($vmid, $deviceid);
4397         qemu_devicedelverify($vmid, $deviceid);
4398         qemu_drivedel($vmid, $deviceid);
4399         qemu_deletescsihw($conf, $vmid, $deviceid);
4400
4401         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4402             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4403     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406         qemu_netdevdel($vmid, $deviceid);
4407     } else {
4408         die "can't unplug device '$deviceid'\n";
4409     }
4410
4411     return 1;
4412 }
4413
4414 sub qemu_spice_usbredir_chardev_add {
4415     my ($vmid, $id) = @_;
4416
4417     mon_cmd($vmid, "chardev-add" , (
4418         id => $id,
4419         backend => {
4420             type => 'spicevmc',
4421             data => {
4422                 type => "usbredir",
4423             },
4424         },
4425     ));
4426 }
4427
4428 sub qemu_deviceadd {
4429     my ($vmid, $devicefull) = @_;
4430
4431     $devicefull = "driver=".$devicefull;
4432     my %options =  split(/[=,]/, $devicefull);
4433
4434     mon_cmd($vmid, "device_add" , %options);
4435 }
4436
4437 sub qemu_devicedel {
4438     my ($vmid, $deviceid) = @_;
4439
4440     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4441 }
4442
4443 sub qemu_iothread_add {
4444     my ($vmid, $deviceid, $device) = @_;
4445
4446     if ($device->{iothread}) {
4447         my $iothreads = vm_iothreads_list($vmid);
4448         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4449     }
4450 }
4451
4452 sub qemu_iothread_del {
4453     my ($vmid, $deviceid, $device) = @_;
4454
4455     if ($device->{iothread}) {
4456         my $iothreads = vm_iothreads_list($vmid);
4457         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4458     }
4459 }
4460
4461 sub qemu_objectadd {
4462     my ($vmid, $objectid, $qomtype) = @_;
4463
4464     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4465
4466     return 1;
4467 }
4468
4469 sub qemu_objectdel {
4470     my ($vmid, $objectid) = @_;
4471
4472     mon_cmd($vmid, "object-del", id => $objectid);
4473
4474     return 1;
4475 }
4476
4477 sub qemu_driveadd {
4478     my ($storecfg, $vmid, $device) = @_;
4479
4480     my $kvmver = get_running_qemu_version($vmid);
4481     my $io_uring = min_version($kvmver, 6, 0);
4482     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4483     $drive =~ s/\\/\\\\/g;
4484     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4485
4486     # If the command succeeds qemu prints: "OK"
4487     return 1 if $ret =~ m/OK/s;
4488
4489     die "adding drive failed: $ret\n";
4490 }
4491
4492 sub qemu_drivedel {
4493     my ($vmid, $deviceid) = @_;
4494
4495     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4496     $ret =~ s/^\s+//;
4497
4498     return 1 if $ret eq "";
4499
4500     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4501     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4502
4503     die "deleting drive $deviceid failed : $ret\n";
4504 }
4505
4506 sub qemu_deviceaddverify {
4507     my ($vmid, $deviceid) = @_;
4508
4509     for (my $i = 0; $i <= 5; $i++) {
4510          my $devices_list = vm_devices_list($vmid);
4511          return 1 if defined($devices_list->{$deviceid});
4512          sleep 1;
4513     }
4514
4515     die "error on hotplug device '$deviceid'\n";
4516 }
4517
4518
4519 sub qemu_devicedelverify {
4520     my ($vmid, $deviceid) = @_;
4521
4522     # need to verify that the device is correctly removed as device_del
4523     # is async and empty return is not reliable
4524
4525     for (my $i = 0; $i <= 5; $i++) {
4526          my $devices_list = vm_devices_list($vmid);
4527          return 1 if !defined($devices_list->{$deviceid});
4528          sleep 1;
4529     }
4530
4531     die "error on hot-unplugging device '$deviceid'\n";
4532 }
4533
4534 sub qemu_findorcreatescsihw {
4535     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4536
4537     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4538
4539     my $scsihwid="$controller_prefix$controller";
4540     my $devices_list = vm_devices_list($vmid);
4541
4542     if (!defined($devices_list->{$scsihwid})) {
4543         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4544     }
4545
4546     return 1;
4547 }
4548
4549 sub qemu_deletescsihw {
4550     my ($conf, $vmid, $opt) = @_;
4551
4552     my $device = parse_drive($opt, $conf->{$opt});
4553
4554     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4555         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4556         return 1;
4557     }
4558
4559     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4560
4561     my $devices_list = vm_devices_list($vmid);
4562     foreach my $opt (keys %{$devices_list}) {
4563         if (is_valid_drivename($opt)) {
4564             my $drive = parse_drive($opt, $conf->{$opt});
4565             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4566                 return 1;
4567             }
4568         }
4569     }
4570
4571     my $scsihwid="scsihw$controller";
4572
4573     vm_deviceunplug($vmid, $conf, $scsihwid);
4574
4575     return 1;
4576 }
4577
4578 sub qemu_add_pci_bridge {
4579     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4580
4581     my $bridges = {};
4582
4583     my $bridgeid;
4584
4585     print_pci_addr($device, $bridges, $arch, $machine_type);
4586
4587     while (my ($k, $v) = each %$bridges) {
4588         $bridgeid = $k;
4589     }
4590     return 1 if !defined($bridgeid) || $bridgeid < 1;
4591
4592     my $bridge = "pci.$bridgeid";
4593     my $devices_list = vm_devices_list($vmid);
4594
4595     if (!defined($devices_list->{$bridge})) {
4596         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4597     }
4598
4599     return 1;
4600 }
4601
4602 sub qemu_set_link_status {
4603     my ($vmid, $device, $up) = @_;
4604
4605     mon_cmd($vmid, "set_link", name => $device,
4606                up => $up ? JSON::true : JSON::false);
4607 }
4608
4609 sub qemu_netdevadd {
4610     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4611
4612     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4613     my %options =  split(/[=,]/, $netdev);
4614
4615     if (defined(my $vhost = $options{vhost})) {
4616         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4617     }
4618
4619     if (defined(my $queues = $options{queues})) {
4620         $options{queues} = $queues + 0;
4621     }
4622
4623     mon_cmd($vmid, "netdev_add",  %options);
4624     return 1;
4625 }
4626
4627 sub qemu_netdevdel {
4628     my ($vmid, $deviceid) = @_;
4629
4630     mon_cmd($vmid, "netdev_del", id => $deviceid);
4631 }
4632
4633 sub qemu_usb_hotplug {
4634     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4635
4636     return if !$device;
4637
4638     # remove the old one first
4639     vm_deviceunplug($vmid, $conf, $deviceid);
4640
4641     # check if xhci controller is necessary and available
4642     my $devicelist = vm_devices_list($vmid);
4643
4644     if (!$devicelist->{xhci}) {
4645         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4646         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4647     }
4648
4649     # print_usbdevice_full expects the parsed device
4650     my $d = parse_usb_device($device->{host});
4651     $d->{usb3} = $device->{usb3};
4652
4653     # add the new one
4654     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4655 }
4656
4657 sub qemu_cpu_hotplug {
4658     my ($vmid, $conf, $vcpus) = @_;
4659
4660     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4661
4662     my $sockets = 1;
4663     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4664     $sockets = $conf->{sockets} if  $conf->{sockets};
4665     my $cores = $conf->{cores} || 1;
4666     my $maxcpus = $sockets * $cores;
4667
4668     $vcpus = $maxcpus if !$vcpus;
4669
4670     die "you can't add more vcpus than maxcpus\n"
4671         if $vcpus > $maxcpus;
4672
4673     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4674
4675     if ($vcpus < $currentvcpus) {
4676
4677         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4678
4679             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4680                 qemu_devicedel($vmid, "cpu$i");
4681                 my $retry = 0;
4682                 my $currentrunningvcpus = undef;
4683                 while (1) {
4684                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4685                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4686                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4687                     $retry++;
4688                     sleep 1;
4689                 }
4690                 #update conf after each succesfull cpu unplug
4691                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4692                 PVE::QemuConfig->write_config($vmid, $conf);
4693             }
4694         } else {
4695             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4696         }
4697
4698         return;
4699     }
4700
4701     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4702     die "vcpus in running vm does not match its configuration\n"
4703         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4704
4705     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4706
4707         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4708             my $cpustr = print_cpu_device($conf, $i);
4709             qemu_deviceadd($vmid, $cpustr);
4710
4711             my $retry = 0;
4712             my $currentrunningvcpus = undef;
4713             while (1) {
4714                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4715                 last if scalar(@{$currentrunningvcpus}) == $i;
4716                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4717                 sleep 1;
4718                 $retry++;
4719             }
4720             #update conf after each succesfull cpu hotplug
4721             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4722             PVE::QemuConfig->write_config($vmid, $conf);
4723         }
4724     } else {
4725
4726         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4727             mon_cmd($vmid, "cpu-add", id => int($i));
4728         }
4729     }
4730 }
4731
4732 sub qemu_block_set_io_throttle {
4733     my ($vmid, $deviceid,
4734         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4735         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4736         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4737         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4738
4739     return if !check_running($vmid) ;
4740
4741     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4742         bps => int($bps),
4743         bps_rd => int($bps_rd),
4744         bps_wr => int($bps_wr),
4745         iops => int($iops),
4746         iops_rd => int($iops_rd),
4747         iops_wr => int($iops_wr),
4748         bps_max => int($bps_max),
4749         bps_rd_max => int($bps_rd_max),
4750         bps_wr_max => int($bps_wr_max),
4751         iops_max => int($iops_max),
4752         iops_rd_max => int($iops_rd_max),
4753         iops_wr_max => int($iops_wr_max),
4754         bps_max_length => int($bps_max_length),
4755         bps_rd_max_length => int($bps_rd_max_length),
4756         bps_wr_max_length => int($bps_wr_max_length),
4757         iops_max_length => int($iops_max_length),
4758         iops_rd_max_length => int($iops_rd_max_length),
4759         iops_wr_max_length => int($iops_wr_max_length),
4760     );
4761
4762 }
4763
4764 sub qemu_block_resize {
4765     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4766
4767     my $running = check_running($vmid);
4768
4769     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4770
4771     return if !$running;
4772
4773     my $padding = (1024 - $size % 1024) % 1024;
4774     $size = $size + $padding;
4775
4776     mon_cmd(
4777         $vmid,
4778         "block_resize",
4779         device => $deviceid,
4780         size => int($size),
4781         timeout => 60,
4782     );
4783 }
4784
4785 sub qemu_volume_snapshot {
4786     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4787
4788     my $running = check_running($vmid);
4789
4790     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4791         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4792     } else {
4793         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4794     }
4795 }
4796
4797 sub qemu_volume_snapshot_delete {
4798     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4799
4800     my $running = check_running($vmid);
4801
4802     if($running) {
4803
4804         $running = undef;
4805         my $conf = PVE::QemuConfig->load_config($vmid);
4806         PVE::QemuConfig->foreach_volume($conf, sub {
4807             my ($ds, $drive) = @_;
4808             $running = 1 if $drive->{file} eq $volid;
4809         });
4810     }
4811
4812     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4813         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4814     } else {
4815         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4816     }
4817 }
4818
4819 sub set_migration_caps {
4820     my ($vmid, $savevm) = @_;
4821
4822     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4823
4824     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4825     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4826
4827     my $cap_ref = [];
4828
4829     my $enabled_cap = {
4830         "auto-converge" => 1,
4831         "xbzrle" => 1,
4832         "x-rdma-pin-all" => 0,
4833         "zero-blocks" => 0,
4834         "compress" => 0,
4835         "dirty-bitmaps" => $dirty_bitmaps,
4836     };
4837
4838     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4839
4840     for my $supported_capability (@$supported_capabilities) {
4841         push @$cap_ref, {
4842             capability => $supported_capability->{capability},
4843             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4844         };
4845     }
4846
4847     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4848 }
4849
4850 sub foreach_volid {
4851     my ($conf, $func, @param) = @_;
4852
4853     my $volhash = {};
4854
4855     my $test_volid = sub {
4856         my ($key, $drive, $snapname) = @_;
4857
4858         my $volid = $drive->{file};
4859         return if !$volid;
4860
4861         $volhash->{$volid}->{cdrom} //= 1;
4862         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4863
4864         my $replicate = $drive->{replicate} // 1;
4865         $volhash->{$volid}->{replicate} //= 0;
4866         $volhash->{$volid}->{replicate} = 1 if $replicate;
4867
4868         $volhash->{$volid}->{shared} //= 0;
4869         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4870
4871         $volhash->{$volid}->{referenced_in_config} //= 0;
4872         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4873
4874         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4875             if defined($snapname);
4876
4877         my $size = $drive->{size};
4878         $volhash->{$volid}->{size} //= $size if $size;
4879
4880         $volhash->{$volid}->{is_vmstate} //= 0;
4881         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4882
4883         $volhash->{$volid}->{is_tpmstate} //= 0;
4884         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4885
4886         $volhash->{$volid}->{is_unused} //= 0;
4887         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4888
4889         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4890     };
4891
4892     my $include_opts = {
4893         extra_keys => ['vmstate'],
4894         include_unused => 1,
4895     };
4896
4897     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4898     foreach my $snapname (keys %{$conf->{snapshots}}) {
4899         my $snap = $conf->{snapshots}->{$snapname};
4900         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4901     }
4902
4903     foreach my $volid (keys %$volhash) {
4904         &$func($volid, $volhash->{$volid}, @param);
4905     }
4906 }
4907
4908 my $fast_plug_option = {
4909     'lock' => 1,
4910     'name' => 1,
4911     'onboot' => 1,
4912     'shares' => 1,
4913     'startup' => 1,
4914     'description' => 1,
4915     'protection' => 1,
4916     'vmstatestorage' => 1,
4917     'hookscript' => 1,
4918     'tags' => 1,
4919 };
4920
4921 for my $opt (keys %$confdesc_cloudinit) {
4922     $fast_plug_option->{$opt} = 1;
4923 };
4924
4925 # hotplug changes in [PENDING]
4926 # $selection hash can be used to only apply specified options, for
4927 # example: { cores => 1 } (only apply changed 'cores')
4928 # $errors ref is used to return error messages
4929 sub vmconfig_hotplug_pending {
4930     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4931
4932     my $defaults = load_defaults();
4933     my $arch = get_vm_arch($conf);
4934     my $machine_type = get_vm_machine($conf, undef, $arch);
4935
4936     # commit values which do not have any impact on running VM first
4937     # Note: those option cannot raise errors, we we do not care about
4938     # $selection and always apply them.
4939
4940     my $add_error = sub {
4941         my ($opt, $msg) = @_;
4942         $errors->{$opt} = "hotplug problem - $msg";
4943     };
4944
4945     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4946
4947     my $cloudinit_record_changed = sub {
4948         my ($conf, $opt, $old, $new) = @_;
4949         return if !$cloudinit_pending_properties->{$opt};
4950
4951         my $ci = ($conf->{cloudinit} //= {});
4952
4953         my $recorded = $ci->{$opt};
4954         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4955
4956         if (defined($new)) {
4957             if (defined($old)) {
4958                 # an existing value is being modified
4959                 if (defined($recorded)) {
4960                     # the value was already not in sync
4961                     if ($new eq $recorded) {
4962                         # a value is being reverted to the cloud-init state:
4963                         delete $ci->{$opt};
4964                         delete $added{$opt};
4965                     } else {
4966                         # the value was changed multiple times, do nothing
4967                     }
4968                 } elsif ($added{$opt}) {
4969                     # the value had been marked as added and is being changed, do nothing
4970                 } else {
4971                     # the value is new, record it:
4972                     $ci->{$opt} = $old;
4973                 }
4974             } else {
4975                 # a new value is being added
4976                 if (defined($recorded)) {
4977                     # it was already not in sync
4978                     if ($new eq $recorded) {
4979                         # a value is being reverted to the cloud-init state:
4980                         delete $ci->{$opt};
4981                         delete $added{$opt};
4982                     } else {
4983                         # the value had temporarily been removed, do nothing
4984                     }
4985                 } elsif ($added{$opt}) {
4986                     # the value had been marked as added already, do nothing
4987                 } else {
4988                     # the value is new, add it
4989                     $added{$opt} = 1;
4990                 }
4991             }
4992         } elsif (!defined($old)) {
4993             # a non-existent value is being removed? ignore...
4994         } else {
4995             # a value is being deleted
4996             if (defined($recorded)) {
4997                 # a value was already recorded, just keep it
4998             } elsif ($added{$opt}) {
4999                 # the value was marked as added, remove it
5000                 delete $added{$opt};
5001             } else {
5002                 # a previously unrecorded value is being removed, record the old value:
5003                 $ci->{$opt} = $old;
5004             }
5005         }
5006
5007         my $added = join(',', sort keys %added);
5008         $ci->{added} = $added if length($added);
5009     };
5010
5011     my $changes = 0;
5012     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5013         if ($fast_plug_option->{$opt}) {
5014             my $new = delete $conf->{pending}->{$opt};
5015             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5016             $conf->{$opt} = $new;
5017             $changes = 1;
5018         }
5019     }
5020
5021     if ($changes) {
5022         PVE::QemuConfig->write_config($vmid, $conf);
5023     }
5024
5025     my $ostype = $conf->{ostype};
5026     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5027     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5028     my $usb_hotplug = $hotplug_features->{usb}
5029         && min_version($version, 7, 1)
5030         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5031
5032     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5033     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5034
5035     foreach my $opt (sort keys %$pending_delete_hash) {
5036         next if $selection && !$selection->{$opt};
5037         my $force = $pending_delete_hash->{$opt}->{force};
5038         eval {
5039             if ($opt eq 'hotplug') {
5040                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5041             } elsif ($opt eq 'tablet') {
5042                 die "skip\n" if !$hotplug_features->{usb};
5043                 if ($defaults->{tablet}) {
5044                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5045                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5046                         if $arch eq 'aarch64';
5047                 } else {
5048                     vm_deviceunplug($vmid, $conf, 'tablet');
5049                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5050                 }
5051             } elsif ($opt =~ m/^usb(\d+)$/) {
5052                 my $index = $1;
5053                 die "skip\n" if !$usb_hotplug;
5054                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5055                 vm_deviceunplug($vmid, $conf, $opt);
5056             } elsif ($opt eq 'vcpus') {
5057                 die "skip\n" if !$hotplug_features->{cpu};
5058                 qemu_cpu_hotplug($vmid, $conf, undef);
5059             } elsif ($opt eq 'balloon') {
5060                 # enable balloon device is not hotpluggable
5061                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5062                 # here we reset the ballooning value to memory
5063                 my $balloon = $conf->{memory} || $defaults->{memory};
5064                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5065             } elsif ($fast_plug_option->{$opt}) {
5066                 # do nothing
5067             } elsif ($opt =~ m/^net(\d+)$/) {
5068                 die "skip\n" if !$hotplug_features->{network};
5069                 vm_deviceunplug($vmid, $conf, $opt);
5070             } elsif (is_valid_drivename($opt)) {
5071                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5072                 vm_deviceunplug($vmid, $conf, $opt);
5073                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5074             } elsif ($opt =~ m/^memory$/) {
5075                 die "skip\n" if !$hotplug_features->{memory};
5076                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5077             } elsif ($opt eq 'cpuunits') {
5078                 $cgroup->change_cpu_shares(undef);
5079             } elsif ($opt eq 'cpulimit') {
5080                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5081             } else {
5082                 die "skip\n";
5083             }
5084         };
5085         if (my $err = $@) {
5086             &$add_error($opt, $err) if $err ne "skip\n";
5087         } else {
5088             my $old = delete $conf->{$opt};
5089             $cloudinit_record_changed->($conf, $opt, $old, undef);
5090             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5091         }
5092     }
5093
5094     my $cloudinit_opt;
5095     foreach my $opt (keys %{$conf->{pending}}) {
5096         next if $selection && !$selection->{$opt};
5097         my $value = $conf->{pending}->{$opt};
5098         eval {
5099             if ($opt eq 'hotplug') {
5100                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5101             } elsif ($opt eq 'tablet') {
5102                 die "skip\n" if !$hotplug_features->{usb};
5103                 if ($value == 1) {
5104                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5105                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5106                         if $arch eq 'aarch64';
5107                 } elsif ($value == 0) {
5108                     vm_deviceunplug($vmid, $conf, 'tablet');
5109                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5110                 }
5111             } elsif ($opt =~ m/^usb(\d+)$/) {
5112                 my $index = $1;
5113                 die "skip\n" if !$usb_hotplug;
5114                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5115                 my $id = $opt;
5116                 if ($d->{host} eq 'spice')  {
5117                     $id = "usbredirdev$index";
5118                 }
5119                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5120             } elsif ($opt eq 'vcpus') {
5121                 die "skip\n" if !$hotplug_features->{cpu};
5122                 qemu_cpu_hotplug($vmid, $conf, $value);
5123             } elsif ($opt eq 'balloon') {
5124                 # enable/disable balloning device is not hotpluggable
5125                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5126                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5127                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5128
5129                 # allow manual ballooning if shares is set to zero
5130                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5131                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5132                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5133                 }
5134             } elsif ($opt =~ m/^net(\d+)$/) {
5135                 # some changes can be done without hotplug
5136                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5137                                     $vmid, $opt, $value, $arch, $machine_type);
5138             } elsif (is_valid_drivename($opt)) {
5139                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5140                 # some changes can be done without hotplug
5141                 my $drive = parse_drive($opt, $value);
5142                 if (drive_is_cloudinit($drive)) {
5143                     $cloudinit_opt = [$opt, $drive];
5144                     # apply all the other changes first, then generate the cloudinit disk
5145                     die "skip\n";
5146                 }
5147                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5148                                      $vmid, $opt, $value, $arch, $machine_type);
5149             } elsif ($opt =~ m/^memory$/) { #dimms
5150                 die "skip\n" if !$hotplug_features->{memory};
5151                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5152             } elsif ($opt eq 'cpuunits') {
5153                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5154                 $cgroup->change_cpu_shares($new_cpuunits);
5155             } elsif ($opt eq 'cpulimit') {
5156                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5157                 $cgroup->change_cpu_quota($cpulimit, 100000);
5158             } elsif ($opt eq 'agent') {
5159                 vmconfig_update_agent($conf, $opt, $value);
5160             } else {
5161                 die "skip\n";  # skip non-hot-pluggable options
5162             }
5163         };
5164         if (my $err = $@) {
5165             &$add_error($opt, $err) if $err ne "skip\n";
5166         } else {
5167             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5168             $conf->{$opt} = $value;
5169             delete $conf->{pending}->{$opt};
5170         }
5171     }
5172
5173     if (defined($cloudinit_opt)) {
5174         my ($opt, $drive) = @$cloudinit_opt;
5175         my $value = $conf->{pending}->{$opt};
5176         eval {
5177             my $temp = {%$conf, $opt => $value};
5178             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5179             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5180                                  $vmid, $opt, $value, $arch, $machine_type);
5181         };
5182         if (my $err = $@) {
5183             &$add_error($opt, $err) if $err ne "skip\n";
5184         } else {
5185             $conf->{$opt} = $value;
5186             delete $conf->{pending}->{$opt};
5187         }
5188     }
5189
5190     # unplug xhci controller if no usb device is left
5191     if ($usb_hotplug) {
5192         my $has_usb = 0;
5193         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5194             next if !defined($conf->{"usb$i"});
5195             $has_usb = 1;
5196             last;
5197         }
5198         if (!$has_usb) {
5199             vm_deviceunplug($vmid, $conf, 'xhci');
5200         }
5201     }
5202
5203     PVE::QemuConfig->write_config($vmid, $conf);
5204
5205     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5206         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5207     }
5208 }
5209
5210 sub try_deallocate_drive {
5211     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5212
5213     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5214         my $volid = $drive->{file};
5215         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5216             my $sid = PVE::Storage::parse_volume_id($volid);
5217             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5218
5219             # check if the disk is really unused
5220             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5221                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5222             PVE::Storage::vdisk_free($storecfg, $volid);
5223             return 1;
5224         } else {
5225             # If vm is not owner of this disk remove from config
5226             return 1;
5227         }
5228     }
5229
5230     return;
5231 }
5232
5233 sub vmconfig_delete_or_detach_drive {
5234     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5235
5236     my $drive = parse_drive($opt, $conf->{$opt});
5237
5238     my $rpcenv = PVE::RPCEnvironment::get();
5239     my $authuser = $rpcenv->get_user();
5240
5241     if ($force) {
5242         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5243         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5244     } else {
5245         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5246     }
5247 }
5248
5249
5250
5251 sub vmconfig_apply_pending {
5252     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5253
5254     return if !scalar(keys %{$conf->{pending}});
5255
5256     my $add_apply_error = sub {
5257         my ($opt, $msg) = @_;
5258         my $err_msg = "unable to apply pending change $opt : $msg";
5259         $errors->{$opt} = $err_msg;
5260         warn $err_msg;
5261     };
5262
5263     # cold plug
5264
5265     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5266     foreach my $opt (sort keys %$pending_delete_hash) {
5267         my $force = $pending_delete_hash->{$opt}->{force};
5268         eval {
5269             if ($opt =~ m/^unused/) {
5270                 die "internal error";
5271             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5272                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5273             }
5274         };
5275         if (my $err = $@) {
5276             $add_apply_error->($opt, $err);
5277         } else {
5278             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5279             delete $conf->{$opt};
5280         }
5281     }
5282
5283     PVE::QemuConfig->cleanup_pending($conf);
5284
5285     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5286
5287     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5288         next if $opt eq 'delete'; # just to be sure
5289         eval {
5290             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5291                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5292             }
5293         };
5294         if (my $err = $@) {
5295             $add_apply_error->($opt, $err);
5296         } else {
5297
5298             if (is_valid_drivename($opt)) {
5299                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5300                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5301             }
5302
5303             $conf->{$opt} = delete $conf->{pending}->{$opt};
5304         }
5305     }
5306
5307     # write all changes at once to avoid unnecessary i/o
5308     PVE::QemuConfig->write_config($vmid, $conf);
5309     if ($generate_cloudinit) {
5310         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5311             # After successful generation and if there were changes to be applied, update the
5312             # config to drop the {cloudinit} entry.
5313             PVE::QemuConfig->write_config($vmid, $conf);
5314         }
5315     }
5316 }
5317
5318 sub vmconfig_update_net {
5319     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5320
5321     my $newnet = parse_net($value);
5322
5323     if ($conf->{$opt}) {
5324         my $oldnet = parse_net($conf->{$opt});
5325
5326         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5327             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5328             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5329             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5330             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5331
5332             # for non online change, we try to hot-unplug
5333             die "skip\n" if !$hotplug;
5334             vm_deviceunplug($vmid, $conf, $opt);
5335         } else {
5336
5337             die "internal error" if $opt !~ m/net(\d+)/;
5338             my $iface = "tap${vmid}i$1";
5339
5340             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5341                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5342                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5343                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5344                 PVE::Network::tap_unplug($iface);
5345
5346                 if ($have_sdn) {
5347                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5348                 } else {
5349                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5350                 }
5351             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5352                 # Rate can be applied on its own but any change above needs to
5353                 # include the rate in tap_plug since OVS resets everything.
5354                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5355             }
5356
5357             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5358                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5359             }
5360
5361             return 1;
5362         }
5363     }
5364
5365     if ($hotplug) {
5366         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5367     } else {
5368         die "skip\n";
5369     }
5370 }
5371
5372 sub vmconfig_update_agent {
5373     my ($conf, $opt, $value) = @_;
5374
5375     die "skip\n" if !$conf->{$opt};
5376
5377     my $hotplug_options = { fstrim_cloned_disks => 1 };
5378
5379     my $old_agent = parse_guest_agent($conf);
5380     my $agent = parse_guest_agent({$opt => $value});
5381
5382     for my $option (keys %$agent) { # added/changed options
5383         next if defined($hotplug_options->{$option});
5384         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5385     }
5386
5387     for my $option (keys %$old_agent) { # removed options
5388         next if defined($hotplug_options->{$option});
5389         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5390     }
5391
5392     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5393 }
5394
5395 sub vmconfig_update_disk {
5396     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5397
5398     my $drive = parse_drive($opt, $value);
5399
5400     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5401         my $media = $drive->{media} || 'disk';
5402         my $oldmedia = $old_drive->{media} || 'disk';
5403         die "unable to change media type\n" if $media ne $oldmedia;
5404
5405         if (!drive_is_cdrom($old_drive)) {
5406
5407             if ($drive->{file} ne $old_drive->{file}) {
5408
5409                 die "skip\n" if !$hotplug;
5410
5411                 # unplug and register as unused
5412                 vm_deviceunplug($vmid, $conf, $opt);
5413                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5414
5415             } else {
5416                 # update existing disk
5417
5418                 # skip non hotpluggable value
5419                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5420                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5421                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5422                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5423                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5424                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5425                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5426                     die "skip\n";
5427                 }
5428
5429                 # apply throttle
5430                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5431                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5432                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5433                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5434                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5435                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5436                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5437                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5438                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5439                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5440                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5441                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5442                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5443                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5444                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5445                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5446                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5447                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5448
5449                     qemu_block_set_io_throttle(
5450                         $vmid,"drive-$opt",
5451                         ($drive->{mbps} || 0)*1024*1024,
5452                         ($drive->{mbps_rd} || 0)*1024*1024,
5453                         ($drive->{mbps_wr} || 0)*1024*1024,
5454                          $drive->{iops} || 0,
5455                          $drive->{iops_rd} || 0,
5456                          $drive->{iops_wr} || 0,
5457                         ($drive->{mbps_max} || 0)*1024*1024,
5458                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5459                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5460                          $drive->{iops_max} || 0,
5461                          $drive->{iops_rd_max} || 0,
5462                          $drive->{iops_wr_max} || 0,
5463                          $drive->{bps_max_length} || 1,
5464                          $drive->{bps_rd_max_length} || 1,
5465                          $drive->{bps_wr_max_length} || 1,
5466                          $drive->{iops_max_length} || 1,
5467                          $drive->{iops_rd_max_length} || 1,
5468                          $drive->{iops_wr_max_length} || 1,
5469                     );
5470
5471                 }
5472
5473                 return 1;
5474             }
5475
5476         } else { # cdrom
5477
5478             if ($drive->{file} eq 'none') {
5479                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5480                 if (drive_is_cloudinit($old_drive)) {
5481                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5482                 }
5483             } else {
5484                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5485
5486                 # force eject if locked
5487                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5488
5489                 if ($path) {
5490                     mon_cmd($vmid, "blockdev-change-medium",
5491                         id => "$opt", filename => "$path");
5492                 }
5493             }
5494
5495             return 1;
5496         }
5497     }
5498
5499     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5500     # hotplug new disks
5501     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5502     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5503 }
5504
5505 sub vmconfig_update_cloudinit_drive {
5506     my ($storecfg, $conf, $vmid) = @_;
5507
5508     my $cloudinit_ds = undef;
5509     my $cloudinit_drive = undef;
5510
5511     PVE::QemuConfig->foreach_volume($conf, sub {
5512         my ($ds, $drive) = @_;
5513         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5514             $cloudinit_ds = $ds;
5515             $cloudinit_drive = $drive;
5516         }
5517     });
5518
5519     return if !$cloudinit_drive;
5520
5521     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5522         PVE::QemuConfig->write_config($vmid, $conf);
5523     }
5524
5525     my $running = PVE::QemuServer::check_running($vmid);
5526
5527     if ($running) {
5528         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5529         if ($path) {
5530             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5531             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5532         }
5533     }
5534 }
5535
5536 # called in locked context by incoming migration
5537 sub vm_migrate_get_nbd_disks {
5538     my ($storecfg, $conf, $replicated_volumes) = @_;
5539
5540     my $local_volumes = {};
5541     PVE::QemuConfig->foreach_volume($conf, sub {
5542         my ($ds, $drive) = @_;
5543
5544         return if drive_is_cdrom($drive);
5545         return if $ds eq 'tpmstate0';
5546
5547         my $volid = $drive->{file};
5548
5549         return if !$volid;
5550
5551         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5552
5553         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5554         return if $scfg->{shared};
5555
5556         # replicated disks re-use existing state via bitmap
5557         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5558         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5559     });
5560     return $local_volumes;
5561 }
5562
5563 # called in locked context by incoming migration
5564 sub vm_migrate_alloc_nbd_disks {
5565     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5566
5567     my $nbd = {};
5568     foreach my $opt (sort keys %$source_volumes) {
5569         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5570
5571         if ($use_existing) {
5572             $nbd->{$opt}->{drivestr} = print_drive($drive);
5573             $nbd->{$opt}->{volid} = $volid;
5574             $nbd->{$opt}->{replicated} = 1;
5575             next;
5576         }
5577
5578         # storage mapping + volname = regular migration
5579         # storage mapping + format = remote migration
5580         # order of precedence, filtered by whether storage supports it:
5581         # 1. explicit requested format
5582         # 2. format of current volume
5583         # 3. default format of storage
5584         if (!$storagemap->{identity}) {
5585             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5586             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5587             if (!$format || !grep { $format eq $_ } @$validFormats) {
5588                 if ($volname) {
5589                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5590                     my $fileFormat = qemu_img_format($scfg, $volname);
5591                     $format = $fileFormat
5592                         if grep { $fileFormat eq $_ } @$validFormats;
5593                 }
5594                 $format //= $defFormat;
5595             }
5596         } else {
5597             # can't happen for remote migration, so $volname is always defined
5598             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5599             $format = qemu_img_format($scfg, $volname);
5600         }
5601
5602         my $size = $drive->{size} / 1024;
5603         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5604         my $newdrive = $drive;
5605         $newdrive->{format} = $format;
5606         $newdrive->{file} = $newvolid;
5607         my $drivestr = print_drive($newdrive);
5608         $nbd->{$opt}->{drivestr} = $drivestr;
5609         $nbd->{$opt}->{volid} = $newvolid;
5610     }
5611
5612     return $nbd;
5613 }
5614
5615 # see vm_start_nolock for parameters, additionally:
5616 # migrate_opts:
5617 #   storagemap = parsed storage map for allocating NBD disks
5618 sub vm_start {
5619     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5620
5621     return PVE::QemuConfig->lock_config($vmid, sub {
5622         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5623
5624         die "you can't start a vm if it's a template\n"
5625             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5626
5627         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5628         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5629
5630         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5631
5632         if ($has_backup_lock && $running) {
5633             # a backup is currently running, attempt to start the guest in the
5634             # existing QEMU instance
5635             return vm_resume($vmid);
5636         }
5637
5638         PVE::QemuConfig->check_lock($conf)
5639             if !($params->{skiplock} || $has_suspended_lock);
5640
5641         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5642
5643         die "VM $vmid already running\n" if $running;
5644
5645         if (my $storagemap = $migrate_opts->{storagemap}) {
5646             my $replicated = $migrate_opts->{replicated_volumes};
5647             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5648             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5649
5650             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5651                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5652             }
5653         }
5654
5655         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5656     });
5657 }
5658
5659
5660 # params:
5661 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5662 #   skiplock => 0/1, skip checking for config lock
5663 #   skiptemplate => 0/1, skip checking whether VM is template
5664 #   forcemachine => to force QEMU machine (rollback/migration)
5665 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5666 #   timeout => in seconds
5667 #   paused => start VM in paused state (backup)
5668 #   resume => resume from hibernation
5669 #   pbs-backing => {
5670 #      sata0 => {
5671 #         repository
5672 #         snapshot
5673 #         keyfile
5674 #         archive
5675 #      },
5676 #      virtio2 => ...
5677 #   }
5678 # migrate_opts:
5679 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5680 #   migratedfrom => source node
5681 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5682 #   network => CIDR of migration network
5683 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5684 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5685 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5686 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5687 #       contained in config
5688 sub vm_start_nolock {
5689     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5690
5691     my $statefile = $params->{statefile};
5692     my $resume = $params->{resume};
5693
5694     my $migratedfrom = $migrate_opts->{migratedfrom};
5695     my $migration_type = $migrate_opts->{type};
5696
5697     my $res = {};
5698
5699     # clean up leftover reboot request files
5700     eval { clear_reboot_request($vmid); };
5701     warn $@ if $@;
5702
5703     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5704         vmconfig_apply_pending($vmid, $conf, $storecfg);
5705         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5706     }
5707
5708     # don't regenerate the ISO if the VM is started as part of a live migration
5709     # this way we can reuse the old ISO with the correct config
5710     if (!$migratedfrom) {
5711         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5712             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5713             # $conf->{cloudinit}, so we could just not do this?
5714             # But we do it above, so for now let's be consistent.
5715             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5716         }
5717     }
5718
5719     # override offline migrated volumes, conf is out of date still
5720     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5721         for my $key (sort keys $offline_volumes->%*) {
5722             my $parsed = parse_drive($key, $conf->{$key});
5723             $parsed->{file} = $offline_volumes->{$key};
5724             $conf->{$key} = print_drive($parsed);
5725         }
5726     }
5727
5728     my $defaults = load_defaults();
5729
5730     # set environment variable useful inside network script
5731     # for remote migration the config is available on the target node!
5732     if (!$migrate_opts->{remote_node}) {
5733         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5734     }
5735
5736     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5737
5738     my $forcemachine = $params->{forcemachine};
5739     my $forcecpu = $params->{forcecpu};
5740     if ($resume) {
5741         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5742         $forcemachine = $conf->{runningmachine};
5743         $forcecpu = $conf->{runningcpu};
5744         print "Resuming suspended VM\n";
5745     }
5746
5747     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5748         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5749
5750     my $migration_ip;
5751     my $get_migration_ip = sub {
5752         my ($nodename) = @_;
5753
5754         return $migration_ip if defined($migration_ip);
5755
5756         my $cidr = $migrate_opts->{network};
5757
5758         if (!defined($cidr)) {
5759             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5760             $cidr = $dc_conf->{migration}->{network};
5761         }
5762
5763         if (defined($cidr)) {
5764             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5765
5766             die "could not get IP: no address configured on local " .
5767                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5768
5769             die "could not get IP: multiple addresses configured on local " .
5770                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5771
5772             $migration_ip = @$ips[0];
5773         }
5774
5775         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5776             if !defined($migration_ip);
5777
5778         return $migration_ip;
5779     };
5780
5781     if ($statefile) {
5782         if ($statefile eq 'tcp') {
5783             my $migrate = $res->{migrate} = { proto => 'tcp' };
5784             $migrate->{addr} = "localhost";
5785             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5786             my $nodename = nodename();
5787
5788             if (!defined($migration_type)) {
5789                 if (defined($datacenterconf->{migration}->{type})) {
5790                     $migration_type = $datacenterconf->{migration}->{type};
5791                 } else {
5792                     $migration_type = 'secure';
5793                 }
5794             }
5795
5796             if ($migration_type eq 'insecure') {
5797                 $migrate->{addr} = $get_migration_ip->($nodename);
5798                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5799             }
5800
5801             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5802             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5803             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5804             push @$cmd, '-incoming', $migrate->{uri};
5805             push @$cmd, '-S';
5806
5807         } elsif ($statefile eq 'unix') {
5808             # should be default for secure migrations as a ssh TCP forward
5809             # tunnel is not deterministic reliable ready and fails regurarly
5810             # to set up in time, so use UNIX socket forwards
5811             my $migrate = $res->{migrate} = { proto => 'unix' };
5812             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5813             unlink $migrate->{addr};
5814
5815             $migrate->{uri} = "unix:$migrate->{addr}";
5816             push @$cmd, '-incoming', $migrate->{uri};
5817             push @$cmd, '-S';
5818
5819         } elsif (-e $statefile) {
5820             push @$cmd, '-loadstate', $statefile;
5821         } else {
5822             my $statepath = PVE::Storage::path($storecfg, $statefile);
5823             push @$vollist, $statefile;
5824             push @$cmd, '-loadstate', $statepath;
5825         }
5826     } elsif ($params->{paused}) {
5827         push @$cmd, '-S';
5828     }
5829
5830     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5831
5832     my $pci_devices = {}; # host pci devices
5833     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5834         my $dev = $conf->{"hostpci$i"} or next;
5835         $pci_devices->{$i} = parse_hostpci($dev);
5836     }
5837
5838     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5839     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5840
5841     # map to a flat list of pci ids
5842     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5843
5844     # reserve all PCI IDs before actually doing anything with them
5845     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5846
5847     eval {
5848         my $uuid;
5849         for my $id (sort keys %$pci_devices) {
5850             my $d = $pci_devices->{$id};
5851             for my $dev ($d->{pciid}->@*) {
5852                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5853
5854                 # nvidia grid needs the uuid of the mdev as qemu parameter
5855                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5856                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5857                 }
5858             }
5859         }
5860         push @$cmd, '-uuid', $uuid if defined($uuid);
5861     };
5862     if (my $err = $@) {
5863         eval { cleanup_pci_devices($vmid, $conf) };
5864         warn $@ if $@;
5865         die $err;
5866     }
5867
5868     PVE::Storage::activate_volumes($storecfg, $vollist);
5869
5870     eval {
5871         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5872     };
5873     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5874     # timeout should be more than enough here...
5875     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5876
5877     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5878
5879     my %run_params = (
5880         timeout => $statefile ? undef : $start_timeout,
5881         umask => 0077,
5882         noerr => 1,
5883     );
5884
5885     # when migrating, prefix QEMU output so other side can pick up any
5886     # errors that might occur and show the user
5887     if ($migratedfrom) {
5888         $run_params{quiet} = 1;
5889         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5890     }
5891
5892     my %systemd_properties = (
5893         Slice => 'qemu.slice',
5894         KillMode => 'process',
5895         SendSIGKILL => 0,
5896         TimeoutStopUSec => ULONG_MAX, # infinity
5897     );
5898
5899     if (PVE::CGroup::cgroup_mode() == 2) {
5900         $systemd_properties{CPUWeight} = $cpuunits;
5901     } else {
5902         $systemd_properties{CPUShares} = $cpuunits;
5903     }
5904
5905     if (my $cpulimit = $conf->{cpulimit}) {
5906         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5907     }
5908     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5909
5910     my $run_qemu = sub {
5911         PVE::Tools::run_fork sub {
5912             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5913
5914             my $tpmpid;
5915             if (my $tpm = $conf->{tpmstate0}) {
5916                 # start the TPM emulator so QEMU can connect on start
5917                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5918             }
5919
5920             my $exitcode = run_command($cmd, %run_params);
5921             if ($exitcode) {
5922                 if ($tpmpid) {
5923                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5924                     kill 'TERM', $tpmpid;
5925                 }
5926                 die "QEMU exited with code $exitcode\n";
5927             }
5928         };
5929     };
5930
5931     if ($conf->{hugepages}) {
5932
5933         my $code = sub {
5934             my $hotplug_features =
5935                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5936             my $hugepages_topology =
5937                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5938
5939             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5940
5941             PVE::QemuServer::Memory::hugepages_mount();
5942             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5943
5944             eval { $run_qemu->() };
5945             if (my $err = $@) {
5946                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5947                     if !$conf->{keephugepages};
5948                 die $err;
5949             }
5950
5951             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5952                 if !$conf->{keephugepages};
5953         };
5954         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5955
5956     } else {
5957         eval { $run_qemu->() };
5958     }
5959
5960     if (my $err = $@) {
5961         # deactivate volumes if start fails
5962         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5963         warn $@ if $@;
5964         eval { cleanup_pci_devices($vmid, $conf) };
5965         warn $@ if $@;
5966
5967         die "start failed: $err";
5968     }
5969
5970     # re-reserve all PCI IDs now that we can know the actual VM PID
5971     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5972     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5973     warn $@ if $@;
5974
5975     if (defined($res->{migrate})) {
5976         print "migration listens on $res->{migrate}->{uri}\n";
5977     } elsif ($statefile) {
5978         eval { mon_cmd($vmid, "cont"); };
5979         warn $@ if $@;
5980     }
5981
5982     #start nbd server for storage migration
5983     if (my $nbd = $migrate_opts->{nbd}) {
5984         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5985
5986         my $migrate_storage_uri;
5987         # nbd_protocol_version > 0 for unix socket support
5988         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5989             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5990             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5991             $migrate_storage_uri = "nbd:unix:$socket_path";
5992             $res->{migrate}->{unix_sockets} = [$socket_path];
5993         } else {
5994             my $nodename = nodename();
5995             my $localip = $get_migration_ip->($nodename);
5996             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5997             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5998
5999             mon_cmd($vmid, "nbd-server-start", addr => {
6000                 type => 'inet',
6001                 data => {
6002                     host => "${localip}",
6003                     port => "${storage_migrate_port}",
6004                 },
6005             });
6006             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6007             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6008         }
6009
6010         my $block_info = mon_cmd($vmid, "query-block");
6011         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6012
6013         foreach my $opt (sort keys %$nbd) {
6014             my $drivestr = $nbd->{$opt}->{drivestr};
6015             my $volid = $nbd->{$opt}->{volid};
6016
6017             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6018
6019             mon_cmd(
6020                 $vmid,
6021                 "block-export-add",
6022                 id => "drive-$opt",
6023                 'node-name' => $block_node,
6024                 writable => JSON::true,
6025                 type => "nbd",
6026                 name => "drive-$opt", # NBD export name
6027             );
6028
6029             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6030             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6031             print "re-using replicated volume: $opt - $volid\n"
6032                 if $nbd->{$opt}->{replicated};
6033
6034             $res->{drives}->{$opt} = $nbd->{$opt};
6035             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6036         }
6037     }
6038
6039     if ($migratedfrom) {
6040         eval {
6041             set_migration_caps($vmid);
6042         };
6043         warn $@ if $@;
6044
6045         if ($spice_port) {
6046             print "spice listens on port $spice_port\n";
6047             $res->{spice_port} = $spice_port;
6048             if ($migrate_opts->{spice_ticket}) {
6049                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6050                     $migrate_opts->{spice_ticket});
6051                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6052             }
6053         }
6054
6055     } else {
6056         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6057             if !$statefile && $conf->{balloon};
6058
6059         foreach my $opt (keys %$conf) {
6060             next if $opt !~  m/^net\d+$/;
6061             my $nicconf = parse_net($conf->{$opt});
6062             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6063         }
6064         add_nets_bridge_fdb($conf, $vmid);
6065     }
6066
6067    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6068         eval {
6069             mon_cmd(
6070                 $vmid,
6071                 'qom-set',
6072                 path => "machine/peripheral/balloon0",
6073                 property => "guest-stats-polling-interval",
6074                 value => 2
6075             );
6076         };
6077         log_warn("could not set polling interval for ballooning - $@") if $@;
6078     }
6079
6080     if ($resume) {
6081         print "Resumed VM, removing state\n";
6082         if (my $vmstate = $conf->{vmstate}) {
6083             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6084             PVE::Storage::vdisk_free($storecfg, $vmstate);
6085         }
6086         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6087         PVE::QemuConfig->write_config($vmid, $conf);
6088     }
6089
6090     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6091
6092     return $res;
6093 }
6094
6095 sub vm_commandline {
6096     my ($storecfg, $vmid, $snapname) = @_;
6097
6098     my $conf = PVE::QemuConfig->load_config($vmid);
6099
6100     my ($forcemachine, $forcecpu);
6101     if ($snapname) {
6102         my $snapshot = $conf->{snapshots}->{$snapname};
6103         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6104
6105         # check for machine or CPU overrides in snapshot
6106         $forcemachine = $snapshot->{runningmachine};
6107         $forcecpu = $snapshot->{runningcpu};
6108
6109         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6110
6111         $conf = $snapshot;
6112     }
6113
6114     my $defaults = load_defaults();
6115
6116     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6117
6118     return PVE::Tools::cmd2string($cmd);
6119 }
6120
6121 sub vm_reset {
6122     my ($vmid, $skiplock) = @_;
6123
6124     PVE::QemuConfig->lock_config($vmid, sub {
6125
6126         my $conf = PVE::QemuConfig->load_config($vmid);
6127
6128         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6129
6130         mon_cmd($vmid, "system_reset");
6131     });
6132 }
6133
6134 sub get_vm_volumes {
6135     my ($conf) = @_;
6136
6137     my $vollist = [];
6138     foreach_volid($conf, sub {
6139         my ($volid, $attr) = @_;
6140
6141         return if $volid =~ m|^/|;
6142
6143         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6144         return if !$sid;
6145
6146         push @$vollist, $volid;
6147     });
6148
6149     return $vollist;
6150 }
6151
6152 sub cleanup_pci_devices {
6153     my ($vmid, $conf) = @_;
6154
6155     foreach my $key (keys %$conf) {
6156         next if $key !~ m/^hostpci(\d+)$/;
6157         my $hostpciindex = $1;
6158         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6159         my $d = parse_hostpci($conf->{$key});
6160         if ($d->{mdev}) {
6161             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6162             # don't want to break ABI just for this two liner
6163             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6164             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6165         }
6166     }
6167     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6168 }
6169
6170 sub vm_stop_cleanup {
6171     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6172
6173     eval {
6174
6175         if (!$keepActive) {
6176             my $vollist = get_vm_volumes($conf);
6177             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6178
6179             if (my $tpmdrive = $conf->{tpmstate0}) {
6180                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6181                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6182                 if ($storeid) {
6183                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6184                 }
6185             }
6186         }
6187
6188         foreach my $ext (qw(mon qmp pid vnc qga)) {
6189             unlink "/var/run/qemu-server/${vmid}.$ext";
6190         }
6191
6192         if ($conf->{ivshmem}) {
6193             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6194             # just delete it for now, VMs which have this already open do not
6195             # are affected, but new VMs will get a separated one. If this
6196             # becomes an issue we either add some sort of ref-counting or just
6197             # add a "don't delete on stop" flag to the ivshmem format.
6198             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6199         }
6200
6201         cleanup_pci_devices($vmid, $conf);
6202
6203         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6204     };
6205     warn $@ if $@; # avoid errors - just warn
6206 }
6207
6208 # call only in locked context
6209 sub _do_vm_stop {
6210     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6211
6212     my $pid = check_running($vmid, $nocheck);
6213     return if !$pid;
6214
6215     my $conf;
6216     if (!$nocheck) {
6217         $conf = PVE::QemuConfig->load_config($vmid);
6218         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6219         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6220             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6221             $timeout = $opts->{down} if $opts->{down};
6222         }
6223         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6224     }
6225
6226     eval {
6227         if ($shutdown) {
6228             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6229                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6230             } else {
6231                 mon_cmd($vmid, "system_powerdown");
6232             }
6233         } else {
6234             mon_cmd($vmid, "quit");
6235         }
6236     };
6237     my $err = $@;
6238
6239     if (!$err) {
6240         $timeout = 60 if !defined($timeout);
6241
6242         my $count = 0;
6243         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6244             $count++;
6245             sleep 1;
6246         }
6247
6248         if ($count >= $timeout) {
6249             if ($force) {
6250                 warn "VM still running - terminating now with SIGTERM\n";
6251                 kill 15, $pid;
6252             } else {
6253                 die "VM quit/powerdown failed - got timeout\n";
6254             }
6255         } else {
6256             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6257             return;
6258         }
6259     } else {
6260        if (!check_running($vmid, $nocheck)) {
6261             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6262             return;
6263        }
6264        if ($force) {
6265             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6266             kill 15, $pid;
6267         } else {
6268             die "VM quit/powerdown failed\n";
6269         }
6270     }
6271
6272     # wait again
6273     $timeout = 10;
6274
6275     my $count = 0;
6276     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6277         $count++;
6278         sleep 1;
6279     }
6280
6281     if ($count >= $timeout) {
6282         warn "VM still running - terminating now with SIGKILL\n";
6283         kill 9, $pid;
6284         sleep 1;
6285     }
6286
6287     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6288 }
6289
6290 # Note: use $nocheck to skip tests if VM configuration file exists.
6291 # We need that when migration VMs to other nodes (files already moved)
6292 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6293 sub vm_stop {
6294     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6295
6296     $force = 1 if !defined($force) && !$shutdown;
6297
6298     if ($migratedfrom){
6299         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6300         kill 15, $pid if $pid;
6301         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6302         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6303         return;
6304     }
6305
6306     PVE::QemuConfig->lock_config($vmid, sub {
6307         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6308    });
6309 }
6310
6311 sub vm_reboot {
6312     my ($vmid, $timeout) = @_;
6313
6314     PVE::QemuConfig->lock_config($vmid, sub {
6315         eval {
6316
6317             # only reboot if running, as qmeventd starts it again on a stop event
6318             return if !check_running($vmid);
6319
6320             create_reboot_request($vmid);
6321
6322             my $storecfg = PVE::Storage::config();
6323             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6324
6325         };
6326         if (my $err = $@) {
6327             # avoid that the next normal shutdown will be confused for a reboot
6328             clear_reboot_request($vmid);
6329             die $err;
6330         }
6331    });
6332 }
6333
6334 # note: if using the statestorage parameter, the caller has to check privileges
6335 sub vm_suspend {
6336     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6337
6338     my $conf;
6339     my $path;
6340     my $storecfg;
6341     my $vmstate;
6342
6343     PVE::QemuConfig->lock_config($vmid, sub {
6344
6345         $conf = PVE::QemuConfig->load_config($vmid);
6346
6347         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6348         PVE::QemuConfig->check_lock($conf)
6349             if !($skiplock || $is_backing_up);
6350
6351         die "cannot suspend to disk during backup\n"
6352             if $is_backing_up && $includestate;
6353
6354         if ($includestate) {
6355             $conf->{lock} = 'suspending';
6356             my $date = strftime("%Y-%m-%d", localtime(time()));
6357             $storecfg = PVE::Storage::config();
6358             if (!$statestorage) {
6359                 $statestorage = find_vmstate_storage($conf, $storecfg);
6360                 # check permissions for the storage
6361                 my $rpcenv = PVE::RPCEnvironment::get();
6362                 if ($rpcenv->{type} ne 'cli') {
6363                     my $authuser = $rpcenv->get_user();
6364                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6365                 }
6366             }
6367
6368
6369             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6370                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6371             $path = PVE::Storage::path($storecfg, $vmstate);
6372             PVE::QemuConfig->write_config($vmid, $conf);
6373         } else {
6374             mon_cmd($vmid, "stop");
6375         }
6376     });
6377
6378     if ($includestate) {
6379         # save vm state
6380         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6381
6382         eval {
6383             set_migration_caps($vmid, 1);
6384             mon_cmd($vmid, "savevm-start", statefile => $path);
6385             for(;;) {
6386                 my $state = mon_cmd($vmid, "query-savevm");
6387                 if (!$state->{status}) {
6388                     die "savevm not active\n";
6389                 } elsif ($state->{status} eq 'active') {
6390                     sleep(1);
6391                     next;
6392                 } elsif ($state->{status} eq 'completed') {
6393                     print "State saved, quitting\n";
6394                     last;
6395                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6396                     die "query-savevm failed with error '$state->{error}'\n"
6397                 } else {
6398                     die "query-savevm returned status '$state->{status}'\n";
6399                 }
6400             }
6401         };
6402         my $err = $@;
6403
6404         PVE::QemuConfig->lock_config($vmid, sub {
6405             $conf = PVE::QemuConfig->load_config($vmid);
6406             if ($err) {
6407                 # cleanup, but leave suspending lock, to indicate something went wrong
6408                 eval {
6409                     mon_cmd($vmid, "savevm-end");
6410                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6411                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6412                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6413                     PVE::QemuConfig->write_config($vmid, $conf);
6414                 };
6415                 warn $@ if $@;
6416                 die $err;
6417             }
6418
6419             die "lock changed unexpectedly\n"
6420                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6421
6422             mon_cmd($vmid, "quit");
6423             $conf->{lock} = 'suspended';
6424             PVE::QemuConfig->write_config($vmid, $conf);
6425         });
6426     }
6427 }
6428
6429 # $nocheck is set when called as part of a migration - in this context the
6430 # location of the config file (source or target node) is not deterministic,
6431 # since migration cannot wait for pmxcfs to process the rename
6432 sub vm_resume {
6433     my ($vmid, $skiplock, $nocheck) = @_;
6434
6435     PVE::QemuConfig->lock_config($vmid, sub {
6436         my $res = mon_cmd($vmid, 'query-status');
6437         my $resume_cmd = 'cont';
6438         my $reset = 0;
6439         my $conf;
6440         if ($nocheck) {
6441             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6442             if ($@) {
6443                 my $vmlist = PVE::Cluster::get_vmlist();
6444                 if (exists($vmlist->{ids}->{$vmid})) {
6445                     my $node = $vmlist->{ids}->{$vmid}->{node};
6446                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6447                 }
6448                 if (!$conf) {
6449                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6450                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6451                 }
6452             }
6453         } else {
6454             $conf = PVE::QemuConfig->load_config($vmid);
6455         }
6456
6457         if ($res->{status}) {
6458             return if $res->{status} eq 'running'; # job done, go home
6459             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6460             $reset = 1 if $res->{status} eq 'shutdown';
6461         }
6462
6463         if (!$nocheck) {
6464             PVE::QemuConfig->check_lock($conf)
6465                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6466         }
6467
6468         if ($reset) {
6469             # required if a VM shuts down during a backup and we get a resume
6470             # request before the backup finishes for example
6471             mon_cmd($vmid, "system_reset");
6472         }
6473
6474         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6475
6476         mon_cmd($vmid, $resume_cmd);
6477     });
6478 }
6479
6480 sub vm_sendkey {
6481     my ($vmid, $skiplock, $key) = @_;
6482
6483     PVE::QemuConfig->lock_config($vmid, sub {
6484
6485         my $conf = PVE::QemuConfig->load_config($vmid);
6486
6487         # there is no qmp command, so we use the human monitor command
6488         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6489         die $res if $res ne '';
6490     });
6491 }
6492
6493 # vzdump restore implementaion
6494
6495 sub tar_archive_read_firstfile {
6496     my $archive = shift;
6497
6498     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6499
6500     # try to detect archive type first
6501     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6502         die "unable to open file '$archive'\n";
6503     my $firstfile = <$fh>;
6504     kill 15, $pid;
6505     close $fh;
6506
6507     die "ERROR: archive contaions no data\n" if !$firstfile;
6508     chomp $firstfile;
6509
6510     return $firstfile;
6511 }
6512
6513 sub tar_restore_cleanup {
6514     my ($storecfg, $statfile) = @_;
6515
6516     print STDERR "starting cleanup\n";
6517
6518     if (my $fd = IO::File->new($statfile, "r")) {
6519         while (defined(my $line = <$fd>)) {
6520             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6521                 my $volid = $2;
6522                 eval {
6523                     if ($volid =~ m|^/|) {
6524                         unlink $volid || die 'unlink failed\n';
6525                     } else {
6526                         PVE::Storage::vdisk_free($storecfg, $volid);
6527                     }
6528                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6529                 };
6530                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6531             } else {
6532                 print STDERR "unable to parse line in statfile - $line";
6533             }
6534         }
6535         $fd->close();
6536     }
6537 }
6538
6539 sub restore_file_archive {
6540     my ($archive, $vmid, $user, $opts) = @_;
6541
6542     return restore_vma_archive($archive, $vmid, $user, $opts)
6543         if $archive eq '-';
6544
6545     my $info = PVE::Storage::archive_info($archive);
6546     my $format = $opts->{format} // $info->{format};
6547     my $comp = $info->{compression};
6548
6549     # try to detect archive format
6550     if ($format eq 'tar') {
6551         return restore_tar_archive($archive, $vmid, $user, $opts);
6552     } else {
6553         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6554     }
6555 }
6556
6557 # hepler to remove disks that will not be used after restore
6558 my $restore_cleanup_oldconf = sub {
6559     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6560
6561     my $kept_disks = {};
6562
6563     PVE::QemuConfig->foreach_volume($oldconf, sub {
6564         my ($ds, $drive) = @_;
6565
6566         return if drive_is_cdrom($drive, 1);
6567
6568         my $volid = $drive->{file};
6569         return if !$volid || $volid =~ m|^/|;
6570
6571         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6572         return if !$path || !$owner || ($owner != $vmid);
6573
6574         # Note: only delete disk we want to restore
6575         # other volumes will become unused
6576         if ($virtdev_hash->{$ds}) {
6577             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6578             if (my $err = $@) {
6579                 warn $err;
6580             }
6581         } else {
6582             $kept_disks->{$volid} = 1;
6583         }
6584     });
6585
6586     # after the restore we have no snapshots anymore
6587     for my $snapname (keys $oldconf->{snapshots}->%*) {
6588         my $snap = $oldconf->{snapshots}->{$snapname};
6589         if ($snap->{vmstate}) {
6590             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6591             if (my $err = $@) {
6592                 warn $err;
6593             }
6594         }
6595
6596         for my $volid (keys $kept_disks->%*) {
6597             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6598             warn $@ if $@;
6599         }
6600     }
6601 };
6602
6603 # Helper to parse vzdump backup device hints
6604 #
6605 # $rpcenv: Environment, used to ckeck storage permissions
6606 # $user: User ID, to check storage permissions
6607 # $storecfg: Storage configuration
6608 # $fh: the file handle for reading the configuration
6609 # $devinfo: should contain device sizes for all backu-up'ed devices
6610 # $options: backup options (pool, default storage)
6611 #
6612 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6613 my $parse_backup_hints = sub {
6614     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6615
6616     my $check_storage = sub { # assert if an image can be allocate
6617         my ($storeid, $scfg) = @_;
6618         die "Content type 'images' is not available on storage '$storeid'\n"
6619             if !$scfg->{content}->{images};
6620         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6621             if $user ne 'root@pam';
6622     };
6623
6624     my $virtdev_hash = {};
6625     while (defined(my $line = <$fh>)) {
6626         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6627             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6628             die "archive does not contain data for drive '$virtdev'\n"
6629                 if !$devinfo->{$devname};
6630
6631             if (defined($options->{storage})) {
6632                 $storeid = $options->{storage} || 'local';
6633             } elsif (!$storeid) {
6634                 $storeid = 'local';
6635             }
6636             $format = 'raw' if !$format;
6637             $devinfo->{$devname}->{devname} = $devname;
6638             $devinfo->{$devname}->{virtdev} = $virtdev;
6639             $devinfo->{$devname}->{format} = $format;
6640             $devinfo->{$devname}->{storeid} = $storeid;
6641
6642             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6643             $check_storage->($storeid, $scfg); # permission and content type check
6644
6645             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6646         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6647             my $virtdev = $1;
6648             my $drive = parse_drive($virtdev, $2);
6649
6650             if (drive_is_cloudinit($drive)) {
6651                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6652                 $storeid = $options->{storage} if defined ($options->{storage});
6653                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6654                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6655
6656                 $check_storage->($storeid, $scfg); # permission and content type check
6657
6658                 $virtdev_hash->{$virtdev} = {
6659                     format => $format,
6660                     storeid => $storeid,
6661                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6662                     is_cloudinit => 1,
6663                 };
6664             }
6665         }
6666     }
6667
6668     return $virtdev_hash;
6669 };
6670
6671 # Helper to allocate and activate all volumes required for a restore
6672 #
6673 # $storecfg: Storage configuration
6674 # $virtdev_hash: as returned by parse_backup_hints()
6675 #
6676 # Returns: { $virtdev => $volid }
6677 my $restore_allocate_devices = sub {
6678     my ($storecfg, $virtdev_hash, $vmid) = @_;
6679
6680     my $map = {};
6681     foreach my $virtdev (sort keys %$virtdev_hash) {
6682         my $d = $virtdev_hash->{$virtdev};
6683         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6684         my $storeid = $d->{storeid};
6685         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6686
6687         # test if requested format is supported
6688         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6689         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6690         $d->{format} = $defFormat if !$supported;
6691
6692         my $name;
6693         if ($d->{is_cloudinit}) {
6694             $name = "vm-$vmid-cloudinit";
6695             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6696             if ($scfg->{path}) {
6697                 $name .= ".$d->{format}";
6698             }
6699         }
6700
6701         my $volid = PVE::Storage::vdisk_alloc(
6702             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6703
6704         print STDERR "new volume ID is '$volid'\n";
6705         $d->{volid} = $volid;
6706
6707         PVE::Storage::activate_volumes($storecfg, [$volid]);
6708
6709         $map->{$virtdev} = $volid;
6710     }
6711
6712     return $map;
6713 };
6714
6715 sub restore_update_config_line {
6716     my ($cookie, $map, $line, $unique) = @_;
6717
6718     return '' if $line =~ m/^\#qmdump\#/;
6719     return '' if $line =~ m/^\#vzdump\#/;
6720     return '' if $line =~ m/^lock:/;
6721     return '' if $line =~ m/^unused\d+:/;
6722     return '' if $line =~ m/^parent:/;
6723
6724     my $res = '';
6725
6726     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6727     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6728         # try to convert old 1.X settings
6729         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6730         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6731             my ($model, $macaddr) = split(/\=/, $devconfig);
6732             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6733             my $net = {
6734                 model => $model,
6735                 bridge => "vmbr$ind",
6736                 macaddr => $macaddr,
6737             };
6738             my $netstr = print_net($net);
6739
6740             $res .= "net$cookie->{netcount}: $netstr\n";
6741             $cookie->{netcount}++;
6742         }
6743     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6744         my ($id, $netstr) = ($1, $2);
6745         my $net = parse_net($netstr);
6746         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6747         $netstr = print_net($net);
6748         $res .= "$id: $netstr\n";
6749     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6750         my $virtdev = $1;
6751         my $value = $3;
6752         my $di = parse_drive($virtdev, $value);
6753         if (defined($di->{backup}) && !$di->{backup}) {
6754             $res .= "#$line";
6755         } elsif ($map->{$virtdev}) {
6756             delete $di->{format}; # format can change on restore
6757             $di->{file} = $map->{$virtdev};
6758             $value = print_drive($di);
6759             $res .= "$virtdev: $value\n";
6760         } else {
6761             $res .= $line;
6762         }
6763     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6764         my $vmgenid = $1;
6765         if ($vmgenid ne '0') {
6766             # always generate a new vmgenid if there was a valid one setup
6767             $vmgenid = generate_uuid();
6768         }
6769         $res .= "vmgenid: $vmgenid\n";
6770     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6771         my ($uuid, $uuid_str);
6772         UUID::generate($uuid);
6773         UUID::unparse($uuid, $uuid_str);
6774         my $smbios1 = parse_smbios1($2);
6775         $smbios1->{uuid} = $uuid_str;
6776         $res .= $1.print_smbios1($smbios1)."\n";
6777     } else {
6778         $res .= $line;
6779     }
6780
6781     return $res;
6782 }
6783
6784 my $restore_deactivate_volumes = sub {
6785     my ($storecfg, $virtdev_hash) = @_;
6786
6787     my $vollist = [];
6788     for my $dev (values $virtdev_hash->%*) {
6789         push $vollist->@*, $dev->{volid} if $dev->{volid};
6790     }
6791
6792     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6793     print STDERR $@ if $@;
6794 };
6795
6796 my $restore_destroy_volumes = sub {
6797     my ($storecfg, $virtdev_hash) = @_;
6798
6799     for my $dev (values $virtdev_hash->%*) {
6800         my $volid = $dev->{volid} or next;
6801         eval {
6802             PVE::Storage::vdisk_free($storecfg, $volid);
6803             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6804         };
6805         print STDERR "unable to cleanup '$volid' - $@" if $@;
6806     }
6807 };
6808
6809 my $restore_merge_config = sub {
6810     my ($filename, $backup_conf_raw, $override_conf) = @_;
6811
6812     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6813     for my $key (keys $override_conf->%*) {
6814         $backup_conf->{$key} = $override_conf->{$key};
6815     }
6816
6817     return $backup_conf;
6818 };
6819
6820 sub scan_volids {
6821     my ($cfg, $vmid) = @_;
6822
6823     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6824
6825     my $volid_hash = {};
6826     foreach my $storeid (keys %$info) {
6827         foreach my $item (@{$info->{$storeid}}) {
6828             next if !($item->{volid} && $item->{size});
6829             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6830             $volid_hash->{$item->{volid}} = $item;
6831         }
6832     }
6833
6834     return $volid_hash;
6835 }
6836
6837 sub update_disk_config {
6838     my ($vmid, $conf, $volid_hash) = @_;
6839
6840     my $changes;
6841     my $prefix = "VM $vmid";
6842
6843     # used and unused disks
6844     my $referenced = {};
6845
6846     # Note: it is allowed to define multiple storages with same path (alias), so
6847     # we need to check both 'volid' and real 'path' (two different volid can point
6848     # to the same path).
6849
6850     my $referencedpath = {};
6851
6852     # update size info
6853     PVE::QemuConfig->foreach_volume($conf, sub {
6854         my ($opt, $drive) = @_;
6855
6856         my $volid = $drive->{file};
6857         return if !$volid;
6858         my $volume = $volid_hash->{$volid};
6859
6860         # mark volid as "in-use" for next step
6861         $referenced->{$volid} = 1;
6862         if ($volume && (my $path = $volume->{path})) {
6863             $referencedpath->{$path} = 1;
6864         }
6865
6866         return if drive_is_cdrom($drive);
6867         return if !$volume;
6868
6869         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6870         if (defined($updated)) {
6871             $changes = 1;
6872             $conf->{$opt} = print_drive($updated);
6873             print "$prefix ($opt): $msg\n";
6874         }
6875     });
6876
6877     # remove 'unusedX' entry if volume is used
6878     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6879         my ($opt, $drive) = @_;
6880
6881         my $volid = $drive->{file};
6882         return if !$volid;
6883
6884         my $path;
6885         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6886         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6887             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6888             $changes = 1;
6889             delete $conf->{$opt};
6890         }
6891
6892         $referenced->{$volid} = 1;
6893         $referencedpath->{$path} = 1 if $path;
6894     });
6895
6896     foreach my $volid (sort keys %$volid_hash) {
6897         next if $volid =~ m/vm-$vmid-state-/;
6898         next if $referenced->{$volid};
6899         my $path = $volid_hash->{$volid}->{path};
6900         next if !$path; # just to be sure
6901         next if $referencedpath->{$path};
6902         $changes = 1;
6903         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6904         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6905         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6906     }
6907
6908     return $changes;
6909 }
6910
6911 sub rescan {
6912     my ($vmid, $nolock, $dryrun) = @_;
6913
6914     my $cfg = PVE::Storage::config();
6915
6916     print "rescan volumes...\n";
6917     my $volid_hash = scan_volids($cfg, $vmid);
6918
6919     my $updatefn =  sub {
6920         my ($vmid) = @_;
6921
6922         my $conf = PVE::QemuConfig->load_config($vmid);
6923
6924         PVE::QemuConfig->check_lock($conf);
6925
6926         my $vm_volids = {};
6927         foreach my $volid (keys %$volid_hash) {
6928             my $info = $volid_hash->{$volid};
6929             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6930         }
6931
6932         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6933
6934         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6935     };
6936
6937     if (defined($vmid)) {
6938         if ($nolock) {
6939             &$updatefn($vmid);
6940         } else {
6941             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6942         }
6943     } else {
6944         my $vmlist = config_list();
6945         foreach my $vmid (keys %$vmlist) {
6946             if ($nolock) {
6947                 &$updatefn($vmid);
6948             } else {
6949                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6950             }
6951         }
6952     }
6953 }
6954
6955 sub restore_proxmox_backup_archive {
6956     my ($archive, $vmid, $user, $options) = @_;
6957
6958     my $storecfg = PVE::Storage::config();
6959
6960     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6961     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6962
6963     my $fingerprint = $scfg->{fingerprint};
6964     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6965
6966     my $repo = PVE::PBSClient::get_repository($scfg);
6967     my $namespace = $scfg->{namespace};
6968
6969     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6970     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6971     local $ENV{PBS_PASSWORD} = $password;
6972     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6973
6974     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6975         PVE::Storage::parse_volname($storecfg, $archive);
6976
6977     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6978
6979     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6980
6981     my $tmpdir = "/var/tmp/vzdumptmp$$";
6982     rmtree $tmpdir;
6983     mkpath $tmpdir;
6984
6985     my $conffile = PVE::QemuConfig->config_file($vmid);
6986      # disable interrupts (always do cleanups)
6987     local $SIG{INT} =
6988         local $SIG{TERM} =
6989         local $SIG{QUIT} =
6990         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6991
6992     # Note: $oldconf is undef if VM does not exists
6993     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6994     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6995     my $new_conf_raw = '';
6996
6997     my $rpcenv = PVE::RPCEnvironment::get();
6998     my $devinfo = {}; # info about drives included in backup
6999     my $virtdev_hash = {}; # info about allocated drives
7000
7001     eval {
7002         # enable interrupts
7003         local $SIG{INT} =
7004             local $SIG{TERM} =
7005             local $SIG{QUIT} =
7006             local $SIG{HUP} =
7007             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7008
7009         my $cfgfn = "$tmpdir/qemu-server.conf";
7010         my $firewall_config_fn = "$tmpdir/fw.conf";
7011         my $index_fn = "$tmpdir/index.json";
7012
7013         my $cmd = "restore";
7014
7015         my $param = [$pbs_backup_name, "index.json", $index_fn];
7016         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7017         my $index = PVE::Tools::file_get_contents($index_fn);
7018         $index = decode_json($index);
7019
7020         foreach my $info (@{$index->{files}}) {
7021             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7022                 my $devname = $1;
7023                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7024                     $devinfo->{$devname}->{size} = $1;
7025                 } else {
7026                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7027                 }
7028             }
7029         }
7030
7031         my $is_qemu_server_backup = scalar(
7032             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7033         );
7034         if (!$is_qemu_server_backup) {
7035             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7036         }
7037         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7038
7039         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7040         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7041
7042         if ($has_firewall_config) {
7043             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7044             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7045
7046             my $pve_firewall_dir = '/etc/pve/firewall';
7047             mkdir $pve_firewall_dir; # make sure the dir exists
7048             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7049         }
7050
7051         my $fh = IO::File->new($cfgfn, "r") ||
7052             die "unable to read qemu-server.conf - $!\n";
7053
7054         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7055
7056         # fixme: rate limit?
7057
7058         # create empty/temp config
7059         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7060
7061         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7062
7063         # allocate volumes
7064         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7065
7066         foreach my $virtdev (sort keys %$virtdev_hash) {
7067             my $d = $virtdev_hash->{$virtdev};
7068             next if $d->{is_cloudinit}; # no need to restore cloudinit
7069
7070             # this fails if storage is unavailable
7071             my $volid = $d->{volid};
7072             my $path = PVE::Storage::path($storecfg, $volid);
7073
7074             # for live-restore we only want to preload the efidisk and TPM state
7075             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7076
7077             my @ns_arg;
7078             if (defined(my $ns = $scfg->{namespace})) {
7079                 @ns_arg = ('--ns', $ns);
7080             }
7081
7082             my $pbs_restore_cmd = [
7083                 '/usr/bin/pbs-restore',
7084                 '--repository', $repo,
7085                 @ns_arg,
7086                 $pbs_backup_name,
7087                 "$d->{devname}.img.fidx",
7088                 $path,
7089                 '--verbose',
7090                 ];
7091
7092             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7093             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7094
7095             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7096                 push @$pbs_restore_cmd, '--skip-zero';
7097             }
7098
7099             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7100             print "restore proxmox backup image: $dbg_cmdstring\n";
7101             run_command($pbs_restore_cmd);
7102         }
7103
7104         $fh->seek(0, 0) || die "seek failed - $!\n";
7105
7106         my $cookie = { netcount => 0 };
7107         while (defined(my $line = <$fh>)) {
7108             $new_conf_raw .= restore_update_config_line(
7109                 $cookie,
7110                 $map,
7111                 $line,
7112                 $options->{unique},
7113             );
7114         }
7115
7116         $fh->close();
7117     };
7118     my $err = $@;
7119
7120     if ($err || !$options->{live}) {
7121         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7122     }
7123
7124     rmtree $tmpdir;
7125
7126     if ($err) {
7127         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7128         die $err;
7129     }
7130
7131     if ($options->{live}) {
7132         # keep lock during live-restore
7133         $new_conf_raw .= "\nlock: create";
7134     }
7135
7136     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7137     PVE::QemuConfig->write_config($vmid, $new_conf);
7138
7139     eval { rescan($vmid, 1); };
7140     warn $@ if $@;
7141
7142     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7143
7144     if ($options->{live}) {
7145         # enable interrupts
7146         local $SIG{INT} =
7147             local $SIG{TERM} =
7148             local $SIG{QUIT} =
7149             local $SIG{HUP} =
7150             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7151
7152         my $conf = PVE::QemuConfig->load_config($vmid);
7153         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7154
7155         # these special drives are already restored before start
7156         delete $devinfo->{'drive-efidisk0'};
7157         delete $devinfo->{'drive-tpmstate0-backup'};
7158
7159         my $pbs_opts = {
7160             repo => $repo,
7161             keyfile => $keyfile,
7162             snapshot => $pbs_backup_name,
7163             namespace => $namespace,
7164         };
7165         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7166
7167         PVE::QemuConfig->remove_lock($vmid, "create");
7168     }
7169 }
7170
7171 sub pbs_live_restore {
7172     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7173
7174     print "starting VM for live-restore\n";
7175     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7176
7177     my $pbs_backing = {};
7178     for my $ds (keys %$restored_disks) {
7179         $ds =~ m/^drive-(.*)$/;
7180         my $confname = $1;
7181         $pbs_backing->{$confname} = {
7182             repository => $opts->{repo},
7183             snapshot => $opts->{snapshot},
7184             archive => "$ds.img.fidx",
7185         };
7186         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7187         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7188
7189         my $drive = parse_drive($confname, $conf->{$confname});
7190         print "restoring '$ds' to '$drive->{file}'\n";
7191     }
7192
7193     my $drives_streamed = 0;
7194     eval {
7195         # make sure HA doesn't interrupt our restore by stopping the VM
7196         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7197             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7198         }
7199
7200         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7201         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7202         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7203
7204         my $qmeventd_fd = register_qmeventd_handle($vmid);
7205
7206         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7207         # this will effectively collapse the backing image chain consisting of
7208         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7209         # removes itself once all backing images vanish with 'auto-remove=on')
7210         my $jobs = {};
7211         for my $ds (sort keys %$restored_disks) {
7212             my $job_id = "restore-$ds";
7213             mon_cmd($vmid, 'block-stream',
7214                 'job-id' => $job_id,
7215                 device => "$ds",
7216             );
7217             $jobs->{$job_id} = {};
7218         }
7219
7220         mon_cmd($vmid, 'cont');
7221         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7222
7223         print "restore-drive jobs finished successfully, removing all tracking block devices"
7224             ." to disconnect from Proxmox Backup Server\n";
7225
7226         for my $ds (sort keys %$restored_disks) {
7227             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7228         }
7229
7230         close($qmeventd_fd);
7231     };
7232
7233     my $err = $@;
7234
7235     if ($err) {
7236         warn "An error occurred during live-restore: $err\n";
7237         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7238         die "live-restore failed\n";
7239     }
7240 }
7241
7242 sub restore_vma_archive {
7243     my ($archive, $vmid, $user, $opts, $comp) = @_;
7244
7245     my $readfrom = $archive;
7246
7247     my $cfg = PVE::Storage::config();
7248     my $commands = [];
7249     my $bwlimit = $opts->{bwlimit};
7250
7251     my $dbg_cmdstring = '';
7252     my $add_pipe = sub {
7253         my ($cmd) = @_;
7254         push @$commands, $cmd;
7255         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7256         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7257         $readfrom = '-';
7258     };
7259
7260     my $input = undef;
7261     if ($archive eq '-') {
7262         $input = '<&STDIN';
7263     } else {
7264         # If we use a backup from a PVE defined storage we also consider that
7265         # storage's rate limit:
7266         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7267         if (defined($volid)) {
7268             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7269             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7270             if ($readlimit) {
7271                 print STDERR "applying read rate limit: $readlimit\n";
7272                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7273                 $add_pipe->($cstream);
7274             }
7275         }
7276     }
7277
7278     if ($comp) {
7279         my $info = PVE::Storage::decompressor_info('vma', $comp);
7280         my $cmd = $info->{decompressor};
7281         push @$cmd, $readfrom;
7282         $add_pipe->($cmd);
7283     }
7284
7285     my $tmpdir = "/var/tmp/vzdumptmp$$";
7286     rmtree $tmpdir;
7287
7288     # disable interrupts (always do cleanups)
7289     local $SIG{INT} =
7290         local $SIG{TERM} =
7291         local $SIG{QUIT} =
7292         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7293
7294     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7295     POSIX::mkfifo($mapfifo, 0600);
7296     my $fifofh;
7297     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7298
7299     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7300
7301     my $oldtimeout;
7302     my $timeout = 5;
7303
7304     my $devinfo = {}; # info about drives included in backup
7305     my $virtdev_hash = {}; # info about allocated drives
7306
7307     my $rpcenv = PVE::RPCEnvironment::get();
7308
7309     my $conffile = PVE::QemuConfig->config_file($vmid);
7310
7311     # Note: $oldconf is undef if VM does not exist
7312     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7313     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7314     my $new_conf_raw = '';
7315
7316     my %storage_limits;
7317
7318     my $print_devmap = sub {
7319         my $cfgfn = "$tmpdir/qemu-server.conf";
7320
7321         # we can read the config - that is already extracted
7322         my $fh = IO::File->new($cfgfn, "r") ||
7323             die "unable to read qemu-server.conf - $!\n";
7324
7325         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7326         if (-f $fwcfgfn) {
7327             my $pve_firewall_dir = '/etc/pve/firewall';
7328             mkdir $pve_firewall_dir; # make sure the dir exists
7329             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7330         }
7331
7332         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7333
7334         foreach my $info (values %{$virtdev_hash}) {
7335             my $storeid = $info->{storeid};
7336             next if defined($storage_limits{$storeid});
7337
7338             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7339             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7340             $storage_limits{$storeid} = $limit * 1024;
7341         }
7342
7343         foreach my $devname (keys %$devinfo) {
7344             die "found no device mapping information for device '$devname'\n"
7345                 if !$devinfo->{$devname}->{virtdev};
7346         }
7347
7348         # create empty/temp config
7349         if ($oldconf) {
7350             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7351             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7352         }
7353
7354         # allocate volumes
7355         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7356
7357         # print restore information to $fifofh
7358         foreach my $virtdev (sort keys %$virtdev_hash) {
7359             my $d = $virtdev_hash->{$virtdev};
7360             next if $d->{is_cloudinit}; # no need to restore cloudinit
7361
7362             my $storeid = $d->{storeid};
7363             my $volid = $d->{volid};
7364
7365             my $map_opts = '';
7366             if (my $limit = $storage_limits{$storeid}) {
7367                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7368             }
7369
7370             my $write_zeros = 1;
7371             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7372                 $write_zeros = 0;
7373             }
7374
7375             my $path = PVE::Storage::path($cfg, $volid);
7376
7377             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7378
7379             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7380         }
7381
7382         $fh->seek(0, 0) || die "seek failed - $!\n";
7383
7384         my $cookie = { netcount => 0 };
7385         while (defined(my $line = <$fh>)) {
7386             $new_conf_raw .= restore_update_config_line(
7387                 $cookie,
7388                 $map,
7389                 $line,
7390                 $opts->{unique},
7391             );
7392         }
7393
7394         $fh->close();
7395     };
7396
7397     eval {
7398         # enable interrupts
7399         local $SIG{INT} =
7400             local $SIG{TERM} =
7401             local $SIG{QUIT} =
7402             local $SIG{HUP} =
7403             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7404         local $SIG{ALRM} = sub { die "got timeout\n"; };
7405
7406         $oldtimeout = alarm($timeout);
7407
7408         my $parser = sub {
7409             my $line = shift;
7410
7411             print "$line\n";
7412
7413             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7414                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7415                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7416             } elsif ($line =~ m/^CTIME: /) {
7417                 # we correctly received the vma config, so we can disable
7418                 # the timeout now for disk allocation (set to 10 minutes, so
7419                 # that we always timeout if something goes wrong)
7420                 alarm(600);
7421                 &$print_devmap();
7422                 print $fifofh "done\n";
7423                 my $tmp = $oldtimeout || 0;
7424                 $oldtimeout = undef;
7425                 alarm($tmp);
7426                 close($fifofh);
7427                 $fifofh = undef;
7428             }
7429         };
7430
7431         print "restore vma archive: $dbg_cmdstring\n";
7432         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7433     };
7434     my $err = $@;
7435
7436     alarm($oldtimeout) if $oldtimeout;
7437
7438     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7439
7440     close($fifofh) if $fifofh;
7441     unlink $mapfifo;
7442     rmtree $tmpdir;
7443
7444     if ($err) {
7445         $restore_destroy_volumes->($cfg, $virtdev_hash);
7446         die $err;
7447     }
7448
7449     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7450     PVE::QemuConfig->write_config($vmid, $new_conf);
7451
7452     eval { rescan($vmid, 1); };
7453     warn $@ if $@;
7454
7455     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7456 }
7457
7458 sub restore_tar_archive {
7459     my ($archive, $vmid, $user, $opts) = @_;
7460
7461     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7462         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7463         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7464     }
7465
7466     if ($archive ne '-') {
7467         my $firstfile = tar_archive_read_firstfile($archive);
7468         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7469             if $firstfile ne 'qemu-server.conf';
7470     }
7471
7472     my $storecfg = PVE::Storage::config();
7473
7474     # avoid zombie disks when restoring over an existing VM -> cleanup first
7475     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7476     # skiplock=1 because qmrestore has set the 'create' lock itself already
7477     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7478     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7479
7480     my $tocmd = "/usr/lib/qemu-server/qmextract";
7481
7482     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7483     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7484     $tocmd .= ' --prealloc' if $opts->{prealloc};
7485     $tocmd .= ' --info' if $opts->{info};
7486
7487     # tar option "xf" does not autodetect compression when read from STDIN,
7488     # so we pipe to zcat
7489     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7490         PVE::Tools::shellquote("--to-command=$tocmd");
7491
7492     my $tmpdir = "/var/tmp/vzdumptmp$$";
7493     mkpath $tmpdir;
7494
7495     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7496     local $ENV{VZDUMP_VMID} = $vmid;
7497     local $ENV{VZDUMP_USER} = $user;
7498
7499     my $conffile = PVE::QemuConfig->config_file($vmid);
7500     my $new_conf_raw = '';
7501
7502     # disable interrupts (always do cleanups)
7503     local $SIG{INT} =
7504         local $SIG{TERM} =
7505         local $SIG{QUIT} =
7506         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7507
7508     eval {
7509         # enable interrupts
7510         local $SIG{INT} =
7511             local $SIG{TERM} =
7512             local $SIG{QUIT} =
7513             local $SIG{HUP} =
7514             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7515
7516         if ($archive eq '-') {
7517             print "extracting archive from STDIN\n";
7518             run_command($cmd, input => "<&STDIN");
7519         } else {
7520             print "extracting archive '$archive'\n";
7521             run_command($cmd);
7522         }
7523
7524         return if $opts->{info};
7525
7526         # read new mapping
7527         my $map = {};
7528         my $statfile = "$tmpdir/qmrestore.stat";
7529         if (my $fd = IO::File->new($statfile, "r")) {
7530             while (defined (my $line = <$fd>)) {
7531                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7532                     $map->{$1} = $2 if $1;
7533                 } else {
7534                     print STDERR "unable to parse line in statfile - $line\n";
7535                 }
7536             }
7537             $fd->close();
7538         }
7539
7540         my $confsrc = "$tmpdir/qemu-server.conf";
7541
7542         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7543
7544         my $cookie = { netcount => 0 };
7545         while (defined (my $line = <$srcfd>)) {
7546             $new_conf_raw .= restore_update_config_line(
7547                 $cookie,
7548                 $map,
7549                 $line,
7550                 $opts->{unique},
7551             );
7552         }
7553
7554         $srcfd->close();
7555     };
7556     if (my $err = $@) {
7557         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7558         die $err;
7559     }
7560
7561     rmtree $tmpdir;
7562
7563     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7564
7565     PVE::Cluster::cfs_update(); # make sure we read new file
7566
7567     eval { rescan($vmid, 1); };
7568     warn $@ if $@;
7569 };
7570
7571 sub foreach_storage_used_by_vm {
7572     my ($conf, $func) = @_;
7573
7574     my $sidhash = {};
7575
7576     PVE::QemuConfig->foreach_volume($conf, sub {
7577         my ($ds, $drive) = @_;
7578         return if drive_is_cdrom($drive);
7579
7580         my $volid = $drive->{file};
7581
7582         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7583         $sidhash->{$sid} = $sid if $sid;
7584     });
7585
7586     foreach my $sid (sort keys %$sidhash) {
7587         &$func($sid);
7588     }
7589 }
7590
7591 my $qemu_snap_storage = {
7592     rbd => 1,
7593 };
7594 sub do_snapshots_with_qemu {
7595     my ($storecfg, $volid, $deviceid) = @_;
7596
7597     return if $deviceid =~ m/tpmstate0/;
7598
7599     my $storage_name = PVE::Storage::parse_volume_id($volid);
7600     my $scfg = $storecfg->{ids}->{$storage_name};
7601     die "could not find storage '$storage_name'\n" if !defined($scfg);
7602
7603     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7604         return 1;
7605     }
7606
7607     if ($volid =~ m/\.(qcow2|qed)$/){
7608         return 1;
7609     }
7610
7611     return;
7612 }
7613
7614 sub qga_check_running {
7615     my ($vmid, $nowarn) = @_;
7616
7617     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7618     if ($@) {
7619         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7620         return 0;
7621     }
7622     return 1;
7623 }
7624
7625 sub template_create {
7626     my ($vmid, $conf, $disk) = @_;
7627
7628     my $storecfg = PVE::Storage::config();
7629
7630     PVE::QemuConfig->foreach_volume($conf, sub {
7631         my ($ds, $drive) = @_;
7632
7633         return if drive_is_cdrom($drive);
7634         return if $disk && $ds ne $disk;
7635
7636         my $volid = $drive->{file};
7637         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7638
7639         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7640         $drive->{file} = $voliddst;
7641         $conf->{$ds} = print_drive($drive);
7642         PVE::QemuConfig->write_config($vmid, $conf);
7643     });
7644 }
7645
7646 sub convert_iscsi_path {
7647     my ($path) = @_;
7648
7649     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7650         my $portal = $1;
7651         my $target = $2;
7652         my $lun = $3;
7653
7654         my $initiator_name = get_initiator_name();
7655
7656         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7657                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7658     }
7659
7660     die "cannot convert iscsi path '$path', unkown format\n";
7661 }
7662
7663 sub qemu_img_convert {
7664     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7665
7666     my $storecfg = PVE::Storage::config();
7667     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7668     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7669
7670     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7671
7672     my $cachemode;
7673     my $src_path;
7674     my $src_is_iscsi = 0;
7675     my $src_format;
7676
7677     if ($src_storeid) {
7678         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7679         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7680         $src_format = qemu_img_format($src_scfg, $src_volname);
7681         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7682         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7683         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7684     } elsif (-f $src_volid || -b $src_volid) {
7685         $src_path = $src_volid;
7686         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7687             $src_format = $1;
7688         }
7689     }
7690
7691     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7692
7693     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7694     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7695     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7696     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7697
7698     my $cmd = [];
7699     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7700     push @$cmd, '-l', "snapshot.name=$snapname"
7701         if $snapname && $src_format && $src_format eq "qcow2";
7702     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7703     push @$cmd, '-T', $cachemode if defined($cachemode);
7704     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7705
7706     if ($src_is_iscsi) {
7707         push @$cmd, '--image-opts';
7708         $src_path = convert_iscsi_path($src_path);
7709     } elsif ($src_format) {
7710         push @$cmd, '-f', $src_format;
7711     }
7712
7713     if ($dst_is_iscsi) {
7714         push @$cmd, '--target-image-opts';
7715         $dst_path = convert_iscsi_path($dst_path);
7716     } else {
7717         push @$cmd, '-O', $dst_format;
7718     }
7719
7720     push @$cmd, $src_path;
7721
7722     if (!$dst_is_iscsi && $is_zero_initialized) {
7723         push @$cmd, "zeroinit:$dst_path";
7724     } else {
7725         push @$cmd, $dst_path;
7726     }
7727
7728     my $parser = sub {
7729         my $line = shift;
7730         if($line =~ m/\((\S+)\/100\%\)/){
7731             my $percent = $1;
7732             my $transferred = int($size * $percent / 100);
7733             my $total_h = render_bytes($size, 1);
7734             my $transferred_h = render_bytes($transferred, 1);
7735
7736             print "transferred $transferred_h of $total_h ($percent%)\n";
7737         }
7738
7739     };
7740
7741     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7742     my $err = $@;
7743     die "copy failed: $err" if $err;
7744 }
7745
7746 sub qemu_img_format {
7747     my ($scfg, $volname) = @_;
7748
7749     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7750         return $1;
7751     } else {
7752         return "raw";
7753     }
7754 }
7755
7756 sub qemu_drive_mirror {
7757     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7758
7759     $jobs = {} if !$jobs;
7760
7761     my $qemu_target;
7762     my $format;
7763     $jobs->{"drive-$drive"} = {};
7764
7765     if ($dst_volid =~ /^nbd:/) {
7766         $qemu_target = $dst_volid;
7767         $format = "nbd";
7768     } else {
7769         my $storecfg = PVE::Storage::config();
7770         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7771
7772         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7773
7774         $format = qemu_img_format($dst_scfg, $dst_volname);
7775
7776         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7777
7778         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7779     }
7780
7781     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7782     $opts->{format} = $format if $format;
7783
7784     if (defined($src_bitmap)) {
7785         $opts->{sync} = 'incremental';
7786         $opts->{bitmap} = $src_bitmap;
7787         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7788     }
7789
7790     if (defined($bwlimit)) {
7791         $opts->{speed} = $bwlimit * 1024;
7792         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7793     } else {
7794         print "drive mirror is starting for drive-$drive\n";
7795     }
7796
7797     # if a job already runs for this device we get an error, catch it for cleanup
7798     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7799     if (my $err = $@) {
7800         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7801         warn "$@\n" if $@;
7802         die "mirroring error: $err\n";
7803     }
7804
7805     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7806 }
7807
7808 # $completion can be either
7809 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7810 # 'cancel': wait until all jobs are ready, block-job-cancel them
7811 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7812 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7813 sub qemu_drive_mirror_monitor {
7814     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7815
7816     $completion //= 'complete';
7817     $op //= "mirror";
7818
7819     eval {
7820         my $err_complete = 0;
7821
7822         my $starttime = time ();
7823         while (1) {
7824             die "block job ('$op') timed out\n" if $err_complete > 300;
7825
7826             my $stats = mon_cmd($vmid, "query-block-jobs");
7827             my $ctime = time();
7828
7829             my $running_jobs = {};
7830             for my $stat (@$stats) {
7831                 next if $stat->{type} ne $op;
7832                 $running_jobs->{$stat->{device}} = $stat;
7833             }
7834
7835             my $readycounter = 0;
7836
7837             for my $job_id (sort keys %$jobs) {
7838                 my $job = $running_jobs->{$job_id};
7839
7840                 my $vanished = !defined($job);
7841                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7842                 if($complete || ($vanished && $completion eq 'auto')) {
7843                     print "$job_id: $op-job finished\n";
7844                     delete $jobs->{$job_id};
7845                     next;
7846                 }
7847
7848                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7849
7850                 my $busy = $job->{busy};
7851                 my $ready = $job->{ready};
7852                 if (my $total = $job->{len}) {
7853                     my $transferred = $job->{offset} || 0;
7854                     my $remaining = $total - $transferred;
7855                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7856
7857                     my $duration = $ctime - $starttime;
7858                     my $total_h = render_bytes($total, 1);
7859                     my $transferred_h = render_bytes($transferred, 1);
7860
7861                     my $status = sprintf(
7862                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7863                         render_duration($duration),
7864                     );
7865
7866                     if ($ready) {
7867                         if ($busy) {
7868                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7869                         } else {
7870                             $status .= ", ready";
7871                         }
7872                     }
7873                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7874                     $jobs->{$job_id}->{ready} = $ready;
7875                 }
7876
7877                 $readycounter++ if $job->{ready};
7878             }
7879
7880             last if scalar(keys %$jobs) == 0;
7881
7882             if ($readycounter == scalar(keys %$jobs)) {
7883                 print "all '$op' jobs are ready\n";
7884
7885                 # do the complete later (or has already been done)
7886                 last if $completion eq 'skip' || $completion eq 'auto';
7887
7888                 if ($vmiddst && $vmiddst != $vmid) {
7889                     my $agent_running = $qga && qga_check_running($vmid);
7890                     if ($agent_running) {
7891                         print "freeze filesystem\n";
7892                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7893                         warn $@ if $@;
7894                     } else {
7895                         print "suspend vm\n";
7896                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7897                         warn $@ if $@;
7898                     }
7899
7900                     # if we clone a disk for a new target vm, we don't switch the disk
7901                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7902
7903                     if ($agent_running) {
7904                         print "unfreeze filesystem\n";
7905                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7906                         warn $@ if $@;
7907                     } else {
7908                         print "resume vm\n";
7909                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7910                         warn $@ if $@;
7911                     }
7912
7913                     last;
7914                 } else {
7915
7916                     for my $job_id (sort keys %$jobs) {
7917                         # try to switch the disk if source and destination are on the same guest
7918                         print "$job_id: Completing block job_id...\n";
7919
7920                         my $op;
7921                         if ($completion eq 'complete') {
7922                             $op = 'block-job-complete';
7923                         } elsif ($completion eq 'cancel') {
7924                             $op = 'block-job-cancel';
7925                         } else {
7926                             die "invalid completion value: $completion\n";
7927                         }
7928                         eval { mon_cmd($vmid, $op, device => $job_id) };
7929                         if ($@ =~ m/cannot be completed/) {
7930                             print "$job_id: block job cannot be completed, trying again.\n";
7931                             $err_complete++;
7932                         }else {
7933                             print "$job_id: Completed successfully.\n";
7934                             $jobs->{$job_id}->{complete} = 1;
7935                         }
7936                     }
7937                 }
7938             }
7939             sleep 1;
7940         }
7941     };
7942     my $err = $@;
7943
7944     if ($err) {
7945         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7946         die "block job ($op) error: $err";
7947     }
7948 }
7949
7950 sub qemu_blockjobs_cancel {
7951     my ($vmid, $jobs) = @_;
7952
7953     foreach my $job (keys %$jobs) {
7954         print "$job: Cancelling block job\n";
7955         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7956         $jobs->{$job}->{cancel} = 1;
7957     }
7958
7959     while (1) {
7960         my $stats = mon_cmd($vmid, "query-block-jobs");
7961
7962         my $running_jobs = {};
7963         foreach my $stat (@$stats) {
7964             $running_jobs->{$stat->{device}} = $stat;
7965         }
7966
7967         foreach my $job (keys %$jobs) {
7968
7969             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7970                 print "$job: Done.\n";
7971                 delete $jobs->{$job};
7972             }
7973         }
7974
7975         last if scalar(keys %$jobs) == 0;
7976
7977         sleep 1;
7978     }
7979 }
7980
7981 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7982 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7983 my sub clone_disk_check_io_uring {
7984     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7985
7986     return if !$use_drive_mirror;
7987
7988     # Don't complain when not changing storage.
7989     # Assume if it works for the source, it'll work for the target too.
7990     return if $src_storeid eq $dst_storeid;
7991
7992     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7993     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7994
7995     my $cache_direct = drive_uses_cache_direct($src_drive);
7996
7997     my $src_uses_io_uring;
7998     if ($src_drive->{aio}) {
7999         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8000     } else {
8001         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8002     }
8003
8004     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8005         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8006 }
8007
8008 sub clone_disk {
8009     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8010
8011     my ($vmid, $running) = $source->@{qw(vmid running)};
8012     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8013
8014     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8015     my ($storage, $format) = $dest->@{qw(storage format)};
8016
8017     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8018
8019     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8020         die "cloning from/to EFI disk requires EFI disk\n"
8021             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8022         die "cloning from/to TPM state requires TPM state\n"
8023             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8024
8025         # This would lead to two device nodes in QEMU pointing to the same backing image!
8026         die "cannot change drive name when cloning disk from/to the same VM\n"
8027             if $use_drive_mirror && $vmid == $newvmid;
8028     }
8029
8030     die "cannot move TPM state while VM is running\n"
8031         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8032
8033     my $newvolid;
8034
8035     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8036     print "$src_drivename " if $src_drivename;
8037     print "($drive->{file})\n";
8038
8039     if (!$full) {
8040         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8041         push @$newvollist, $newvolid;
8042     } else {
8043         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8044         my $storeid = $storage || $src_storeid;
8045
8046         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8047
8048         my $name = undef;
8049         my $size = undef;
8050         if (drive_is_cloudinit($drive)) {
8051             $name = "vm-$newvmid-cloudinit";
8052             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8053             if ($scfg->{path}) {
8054                 $name .= ".$dst_format";
8055             }
8056             $snapname = undef;
8057             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8058         } elsif ($dst_drivename eq 'efidisk0') {
8059             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8060         } elsif ($dst_drivename eq 'tpmstate0') {
8061             $dst_format = 'raw';
8062             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8063         } else {
8064             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8065
8066             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8067         }
8068         $newvolid = PVE::Storage::vdisk_alloc(
8069             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8070         );
8071         push @$newvollist, $newvolid;
8072
8073         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8074
8075         if (drive_is_cloudinit($drive)) {
8076             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8077             # if this is the case, we have to complete any block-jobs still there from
8078             # previous drive-mirrors
8079             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8080                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8081             }
8082             goto no_data_clone;
8083         }
8084
8085         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8086         if ($use_drive_mirror) {
8087             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8088                 $completion, $qga, $bwlimit);
8089         } else {
8090             if ($dst_drivename eq 'efidisk0') {
8091                 # the relevant data on the efidisk may be smaller than the source
8092                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8093                 # that is given by the OVMF_VARS.fd
8094                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8095                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8096
8097                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8098
8099                 # better for Ceph if block size is not too small, see bug #3324
8100                 my $bs = 1024*1024;
8101
8102                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8103
8104                 if ($src_format eq 'qcow2' && $snapname) {
8105                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8106                         if !min_version(kvm_user_version(), 6, 2);
8107                     push $cmd->@*, '-l', $snapname;
8108                 }
8109                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8110                 run_command($cmd);
8111             } else {
8112                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8113             }
8114         }
8115     }
8116
8117 no_data_clone:
8118     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8119
8120     my $disk = dclone($drive);
8121     delete $disk->{format};
8122     $disk->{file} = $newvolid;
8123     $disk->{size} = $size if defined($size);
8124
8125     return $disk;
8126 }
8127
8128 sub get_running_qemu_version {
8129     my ($vmid) = @_;
8130     my $res = mon_cmd($vmid, "query-version");
8131     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8132 }
8133
8134 sub qemu_use_old_bios_files {
8135     my ($machine_type) = @_;
8136
8137     return if !$machine_type;
8138
8139     my $use_old_bios_files = undef;
8140
8141     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8142         $machine_type = $1;
8143         $use_old_bios_files = 1;
8144     } else {
8145         my $version = extract_version($machine_type, kvm_user_version());
8146         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8147         # load new efi bios files on migration. So this hack is required to allow
8148         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8149         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8150         $use_old_bios_files = !min_version($version, 2, 4);
8151     }
8152
8153     return ($use_old_bios_files, $machine_type);
8154 }
8155
8156 sub get_efivars_size {
8157     my ($conf, $efidisk) = @_;
8158
8159     my $arch = get_vm_arch($conf);
8160     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8161     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8162     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8163     return -s $ovmf_vars;
8164 }
8165
8166 sub update_efidisk_size {
8167     my ($conf) = @_;
8168
8169     return if !defined($conf->{efidisk0});
8170
8171     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8172     $disk->{size} = get_efivars_size($conf);
8173     $conf->{efidisk0} = print_drive($disk);
8174
8175     return;
8176 }
8177
8178 sub update_tpmstate_size {
8179     my ($conf) = @_;
8180
8181     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8182     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8183     $conf->{tpmstate0} = print_drive($disk);
8184 }
8185
8186 sub create_efidisk($$$$$$$) {
8187     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8188
8189     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8190
8191     my $vars_size_b = -s $ovmf_vars;
8192     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8193     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8194     PVE::Storage::activate_volumes($storecfg, [$volid]);
8195
8196     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8197     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8198
8199     return ($volid, $size/1024);
8200 }
8201
8202 sub vm_iothreads_list {
8203     my ($vmid) = @_;
8204
8205     my $res = mon_cmd($vmid, 'query-iothreads');
8206
8207     my $iothreads = {};
8208     foreach my $iothread (@$res) {
8209         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8210     }
8211
8212     return $iothreads;
8213 }
8214
8215 sub scsihw_infos {
8216     my ($conf, $drive) = @_;
8217
8218     my $maxdev = 0;
8219
8220     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8221         $maxdev = 7;
8222     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8223         $maxdev = 1;
8224     } else {
8225         $maxdev = 256;
8226     }
8227
8228     my $controller = int($drive->{index} / $maxdev);
8229     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8230         ? "virtioscsi"
8231         : "scsihw";
8232
8233     return ($maxdev, $controller, $controller_prefix);
8234 }
8235
8236 sub resolve_dst_disk_format {
8237         my ($storecfg, $storeid, $src_volname, $format) = @_;
8238         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8239
8240         if (!$format) {
8241             # if no target format is specified, use the source disk format as hint
8242             if ($src_volname) {
8243                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8244                 $format = qemu_img_format($scfg, $src_volname);
8245             } else {
8246                 return $defFormat;
8247             }
8248         }
8249
8250         # test if requested format is supported - else use default
8251         my $supported = grep { $_ eq $format } @$validFormats;
8252         $format = $defFormat if !$supported;
8253         return $format;
8254 }
8255
8256 # NOTE: if this logic changes, please update docs & possibly gui logic
8257 sub find_vmstate_storage {
8258     my ($conf, $storecfg) = @_;
8259
8260     # first, return storage from conf if set
8261     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8262
8263     my ($target, $shared, $local);
8264
8265     foreach_storage_used_by_vm($conf, sub {
8266         my ($sid) = @_;
8267         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8268         my $dst = $scfg->{shared} ? \$shared : \$local;
8269         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8270     });
8271
8272     # second, use shared storage where VM has at least one disk
8273     # third, use local storage where VM has at least one disk
8274     # fall back to local storage
8275     $target = $shared // $local // 'local';
8276
8277     return $target;
8278 }
8279
8280 sub generate_uuid {
8281     my ($uuid, $uuid_str);
8282     UUID::generate($uuid);
8283     UUID::unparse($uuid, $uuid_str);
8284     return $uuid_str;
8285 }
8286
8287 sub generate_smbios1_uuid {
8288     return "uuid=".generate_uuid();
8289 }
8290
8291 sub nbd_stop {
8292     my ($vmid) = @_;
8293
8294     mon_cmd($vmid, 'nbd-server-stop');
8295 }
8296
8297 sub create_reboot_request {
8298     my ($vmid) = @_;
8299     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8300         or die "failed to create reboot trigger file: $!\n";
8301     close($fh);
8302 }
8303
8304 sub clear_reboot_request {
8305     my ($vmid) = @_;
8306     my $path = "/run/qemu-server/$vmid.reboot";
8307     my $res = 0;
8308
8309     $res = unlink($path);
8310     die "could not remove reboot request for $vmid: $!"
8311         if !$res && $! != POSIX::ENOENT;
8312
8313     return $res;
8314 }
8315
8316 sub bootorder_from_legacy {
8317     my ($conf, $bootcfg) = @_;
8318
8319     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8320     my $bootindex_hash = {};
8321     my $i = 1;
8322     foreach my $o (split(//, $boot)) {
8323         $bootindex_hash->{$o} = $i*100;
8324         $i++;
8325     }
8326
8327     my $bootorder = {};
8328
8329     PVE::QemuConfig->foreach_volume($conf, sub {
8330         my ($ds, $drive) = @_;
8331
8332         if (drive_is_cdrom ($drive, 1)) {
8333             if ($bootindex_hash->{d}) {
8334                 $bootorder->{$ds} = $bootindex_hash->{d};
8335                 $bootindex_hash->{d} += 1;
8336             }
8337         } elsif ($bootindex_hash->{c}) {
8338             $bootorder->{$ds} = $bootindex_hash->{c}
8339                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8340             $bootindex_hash->{c} += 1;
8341         }
8342     });
8343
8344     if ($bootindex_hash->{n}) {
8345         for (my $i = 0; $i < $MAX_NETS; $i++) {
8346             my $netname = "net$i";
8347             next if !$conf->{$netname};
8348             $bootorder->{$netname} = $bootindex_hash->{n};
8349             $bootindex_hash->{n} += 1;
8350         }
8351     }
8352
8353     return $bootorder;
8354 }
8355
8356 # Generate default device list for 'boot: order=' property. Matches legacy
8357 # default boot order, but with explicit device names. This is important, since
8358 # the fallback for when neither 'order' nor the old format is specified relies
8359 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8360 sub get_default_bootdevices {
8361     my ($conf) = @_;
8362
8363     my @ret = ();
8364
8365     # harddisk
8366     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8367     push @ret, $first if $first;
8368
8369     # cdrom
8370     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8371     push @ret, $first if $first;
8372
8373     # network
8374     for (my $i = 0; $i < $MAX_NETS; $i++) {
8375         my $netname = "net$i";
8376         next if !$conf->{$netname};
8377         push @ret, $netname;
8378         last;
8379     }
8380
8381     return \@ret;
8382 }
8383
8384 sub device_bootorder {
8385     my ($conf) = @_;
8386
8387     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8388
8389     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8390
8391     my $bootorder = {};
8392     if (!defined($boot) || $boot->{legacy}) {
8393         $bootorder = bootorder_from_legacy($conf, $boot);
8394     } elsif ($boot->{order}) {
8395         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8396         for my $dev (PVE::Tools::split_list($boot->{order})) {
8397             $bootorder->{$dev} = $i++;
8398         }
8399     }
8400
8401     return $bootorder;
8402 }
8403
8404 sub register_qmeventd_handle {
8405     my ($vmid) = @_;
8406
8407     my $fh;
8408     my $peer = "/var/run/qmeventd.sock";
8409     my $count = 0;
8410
8411     for (;;) {
8412         $count++;
8413         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8414         last if $fh;
8415         if ($! != EINTR && $! != EAGAIN) {
8416             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8417         }
8418         if ($count > 4) {
8419             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8420               . "after $count retries\n";
8421         }
8422         usleep(25000);
8423     }
8424
8425     # send handshake to mark VM as backing up
8426     print $fh to_json({vzdump => {vmid => "$vmid"}});
8427
8428     # return handle to be closed later when inhibit is no longer required
8429     return $fh;
8430 }
8431
8432 # bash completion helper
8433
8434 sub complete_backup_archives {
8435     my ($cmdname, $pname, $cvalue) = @_;
8436
8437     my $cfg = PVE::Storage::config();
8438
8439     my $storeid;
8440
8441     if ($cvalue =~ m/^([^:]+):/) {
8442         $storeid = $1;
8443     }
8444
8445     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8446
8447     my $res = [];
8448     foreach my $id (keys %$data) {
8449         foreach my $item (@{$data->{$id}}) {
8450             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8451             push @$res, $item->{volid} if defined($item->{volid});
8452         }
8453     }
8454
8455     return $res;
8456 }
8457
8458 my $complete_vmid_full = sub {
8459     my ($running) = @_;
8460
8461     my $idlist = vmstatus();
8462
8463     my $res = [];
8464
8465     foreach my $id (keys %$idlist) {
8466         my $d = $idlist->{$id};
8467         if (defined($running)) {
8468             next if $d->{template};
8469             next if $running && $d->{status} ne 'running';
8470             next if !$running && $d->{status} eq 'running';
8471         }
8472         push @$res, $id;
8473
8474     }
8475     return $res;
8476 };
8477
8478 sub complete_vmid {
8479     return &$complete_vmid_full();
8480 }
8481
8482 sub complete_vmid_stopped {
8483     return &$complete_vmid_full(0);
8484 }
8485
8486 sub complete_vmid_running {
8487     return &$complete_vmid_full(1);
8488 }
8489
8490 sub complete_storage {
8491
8492     my $cfg = PVE::Storage::config();
8493     my $ids = $cfg->{ids};
8494
8495     my $res = [];
8496     foreach my $sid (keys %$ids) {
8497         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8498         next if !$ids->{$sid}->{content}->{images};
8499         push @$res, $sid;
8500     }
8501
8502     return $res;
8503 }
8504
8505 sub complete_migration_storage {
8506     my ($cmd, $param, $current_value, $all_args) = @_;
8507
8508     my $targetnode = @$all_args[1];
8509
8510     my $cfg = PVE::Storage::config();
8511     my $ids = $cfg->{ids};
8512
8513     my $res = [];
8514     foreach my $sid (keys %$ids) {
8515         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8516         next if !$ids->{$sid}->{content}->{images};
8517         push @$res, $sid;
8518     }
8519
8520     return $res;
8521 }
8522
8523 sub vm_is_paused {
8524     my ($vmid) = @_;
8525     my $qmpstatus = eval {
8526         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8527         mon_cmd($vmid, "query-status");
8528     };
8529     warn "$@\n" if $@;
8530     return $qmpstatus && $qmpstatus->{status} eq "paused";
8531 }
8532
8533 sub check_volume_storage_type {
8534     my ($storecfg, $vol) = @_;
8535
8536     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8537     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8538     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8539
8540     die "storage '$storeid' does not support content-type '$vtype'\n"
8541         if !$scfg->{content}->{$vtype};
8542
8543     return 1;
8544 }
8545
8546 sub add_nets_bridge_fdb {
8547     my ($conf, $vmid) = @_;
8548
8549     for my $opt (keys %$conf) {
8550         next if $opt !~ m/^net(\d+)$/;
8551         my $iface = "tap${vmid}i$1";
8552         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8553         my $net = parse_net($conf->{$opt}, 1) or next;
8554
8555         my $mac = $net->{macaddr};
8556         if (!$mac) {
8557             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8558                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8559             next;
8560         }
8561
8562         my $bridge = $net->{bridge};
8563         if ($have_sdn) {
8564             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8565         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8566             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8567         }
8568     }
8569 }
8570
8571 sub del_nets_bridge_fdb {
8572     my ($conf, $vmid) = @_;
8573
8574     for my $opt (keys %$conf) {
8575         next if $opt !~ m/^net(\d+)$/;
8576         my $iface = "tap${vmid}i$1";
8577
8578         my $net = parse_net($conf->{$opt}) or next;
8579         my $mac = $net->{macaddr} or next;
8580
8581         my $bridge = $net->{bridge};
8582         if ($have_sdn) {
8583             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8584         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8585             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8586         }
8587     }
8588 }
8589
8590 1;