PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB;
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196 };
 197
 198 my $ivshmem_fmt = {
 199     size => {
 200         type => 'integer',
 201         minimum => 1,
 202         description => "The size of the file in MB.",
 203     },
 204     name => {
 205         type => 'string',
 206         pattern => '[a-zA-Z0-9\-]+',
 207         optional => 1,
 208         format_description => 'string',
 209         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 210     },
 211 };
 212
 213 my $audio_fmt = {
 214     device => {
 215         type => 'string',
 216         enum => [qw(ich9-intel-hda intel-hda AC97)],
 217         description =>  "Configure an audio device."
 218     },
 219     driver =>  {
 220         type => 'string',
 221         enum => ['spice', 'none'],
 222         default => 'spice',
 223         optional => 1,
 224         description => "Driver backend for the audio device."
 225     },
 226 };
 227
 228 my $spice_enhancements_fmt = {
 229     foldersharing => {
 230         type => 'boolean',
 231         optional => 1,
 232         default => '0',
 233         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 234     },
 235     videostreaming =>  {
 236         type => 'string',
 237         enum => ['off', 'all', 'filter'],
 238         default => 'off',
 239         optional => 1,
 240         description => "Enable video streaming. Uses compression for detected video streams."
 241     },
 242 };
 243
 244 my $rng_fmt = {
 245     source => {
 246         type => 'string',
 247         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 248         default_key => 1,
 249         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 250             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 251             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 252             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 253             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 254             ." a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 259             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 260             ." `0` to disable limiting (potentially dangerous!).",
 261         optional => 1,
 262
 263         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 264         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 265         # reading from /dev/urandom
 266         default => 1024,
 267     },
 268     period => {
 269         type => 'integer',
 270         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 271             ." the guest to retrieve another 'max_bytes' of entropy.",
 272         optional => 1,
 273         default => 1000,
 274     },
 275 };
 276
 277 my $meta_info_fmt = {
 278     'ctime' => {
 279         type => 'integer',
 280         description => "The guest creation timestamp as UNIX epoch time",
 281         minimum => 0,
 282         optional => 1,
 283     },
 284     'creation-qemu' => {
 285         type => 'string',
 286         description => "The QEMU (machine) version from the time this VM was created.",
 287         pattern => '\d+(\.\d+)+',
 288         optional => 1,
 289     },
 290 };
 291
 292 my $confdesc = {
 293     onboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Specifies whether a VM will be started during system bootup.",
 297         default => 0,
 298     },
 299     autostart => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Automatic restart after crash (currently ignored).",
 303         default => 0,
 304     },
 305     hotplug => {
 306         optional => 1,
 307         type => 'string', format => 'pve-hotplug-features',
 308         description => "Selectively enable hotplug features. This is a comma separated list of"
 309             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 310             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 311             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 312             ." windows > 7.",
 313         default => 'network,disk,usb',
 314     },
 315     reboot => {
 316         optional => 1,
 317         type => 'boolean',
 318         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 319         default => 1,
 320     },
 321     lock => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Lock/unlock the VM.",
 325         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 326     },
 327     cpulimit => {
 328         optional => 1,
 329         type => 'number',
 330         description => "Limit of CPU usage.",
 331         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 332             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 333         minimum => 0,
 334         maximum => 128,
 335         default => 0,
 336     },
 337     cpuunits => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 341         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 342             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 343             ." weights of all the other running VMs.",
 344         minimum => 1,
 345         maximum => 262144,
 346         default => 'cgroup v1: 1024, cgroup v2: 100',
 347     },
 348     memory => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 352             ." you use the balloon device.",
 353         minimum => 16,
 354         default => 512,
 355     },
 356     balloon => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 360         minimum => 0,
 361     },
 362     shares => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 366             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 367             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 368         minimum => 0,
 369         maximum => 50000,
 370         default => 1000,
 371     },
 372     keyboard => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Keyboard layout for VNC server. This option is generally not required and"
 376         ." is often better handled from within the guest OS.",
 377         enum => PVE::Tools::kvmkeymaplist(),
 378         default => undef,
 379     },
 380     name => {
 381         optional => 1,
 382         type => 'string', format => 'dns-name',
 383         description => "Set a name for the VM. Only used on the configuration web interface.",
 384     },
 385     scsihw => {
 386         optional => 1,
 387         type => 'string',
 388         description => "SCSI controller model",
 389         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 390         default => 'lsi',
 391     },
 392     description => {
 393         optional => 1,
 394         type => 'string',
 395         description => "Description for the VM. Shown in the web-interface VM's summary."
 396             ." This is saved as comment inside the configuration file.",
 397         maxLength => 1024 * 8,
 398     },
 399     ostype => {
 400         optional => 1,
 401         type => 'string',
 402         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 403         description => "Specify guest operating system.",
 404         verbose_description => <<EODESC,
 405 Specify guest operating system. This is used to enable special
 406 optimization/features for specific operating systems:
 407
 408 [horizontal]
 409 other;; unspecified OS
 410 wxp;; Microsoft Windows XP
 411 w2k;; Microsoft Windows 2000
 412 w2k3;; Microsoft Windows 2003
 413 w2k8;; Microsoft Windows 2008
 414 wvista;; Microsoft Windows Vista
 415 win7;; Microsoft Windows 7
 416 win8;; Microsoft Windows 8/2012/2012r2
 417 win10;; Microsoft Windows 10/2016/2019
 418 win11;; Microsoft Windows 11/2022
 419 l24;; Linux 2.4 Kernel
 420 l26;; Linux 2.6 - 6.X Kernel
 421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 422 EODESC
 423     },
 424     boot => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-boot',
 427         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 428             ." key or 'legacy=' is deprecated.",
 429     },
 430     bootdisk => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-bootdisk',
 433         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 434         pattern => '(ide|sata|scsi|virtio)\d+',
 435     },
 436     smp => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPUs. Please use option -sockets instead.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     sockets => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of CPU sockets.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     cores => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of cores per socket.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     numa => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable NUMA.",
 461         default => 0,
 462     },
 463     hugepages => {
 464         optional => 1,
 465         type => 'string',
 466         description => "Enable/disable hugepages memory.",
 467         enum => [qw(any 2 1024)],
 468     },
 469     keephugepages => {
 470         optional => 1,
 471         type => 'boolean',
 472         default => 0,
 473         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 474             ." after VM shutdown and can be used for subsequent starts.",
 475     },
 476     vcpus => {
 477         optional => 1,
 478         type => 'integer',
 479         description => "Number of hotplugged vcpus.",
 480         minimum => 1,
 481         default => 0,
 482     },
 483     acpi => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Enable/disable ACPI.",
 487         default => 1,
 488     },
 489     agent => {
 490         optional => 1,
 491         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 492         type => 'string',
 493         format => $agent_fmt,
 494     },
 495     kvm => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable KVM hardware virtualization.",
 499         default => 1,
 500     },
 501     tdf => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable time drift fix.",
 505         default => 0,
 506     },
 507     localtime => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 511             ." the `ostype` indicates a Microsoft Windows OS.",
 512     },
 513     freeze => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 517     },
 518     vga => {
 519         optional => 1,
 520         type => 'string', format => $vga_fmt,
 521         description => "Configure the VGA hardware.",
 522         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 523             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 524             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 525             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 526             ." display server. For win* OS you can select how many independent displays you want,"
 527             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 528             ." using a serial device as terminal.",
 529     },
 530     watchdog => {
 531         optional => 1,
 532         type => 'string', format => 'pve-qm-watchdog',
 533         description => "Create a virtual hardware watchdog device.",
 534         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 535             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 536             ." else the watchdog will reset the guest (or execute the respective action specified)",
 537     },
 538     startdate => {
 539         optional => 1,
 540         type => 'string',
 541         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 542         description => "Set the initial date of the real time clock. Valid format for date are:"
 543             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 544         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 545         default => 'now',
 546     },
 547     startup =>  get_standard_option('pve-startup-order'),
 548     template => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Enable/disable Template.",
 552         default => 0,
 553     },
 554     args => {
 555         optional => 1,
 556         type => 'string',
 557         description => "Arbitrary arguments passed to kvm.",
 558         verbose_description => <<EODESCR,
 559 Arbitrary arguments passed to kvm, for example:
 560
 561 args: -no-reboot -smbios 'type=0,vendor=FOO'
 562
 563 NOTE: this option is for experts only.
 564 EODESCR
 565     },
 566     tablet => {
 567         optional => 1,
 568         type => 'boolean',
 569         default => 1,
 570         description => "Enable/disable the USB tablet device.",
 571         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 572             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 573             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 574             ." may consider disabling this to save some context switches. This is turned off by"
 575             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 576     },
 577     migrate_speed => {
 578         optional => 1,
 579         type => 'integer',
 580         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 581         minimum => 0,
 582         default => 0,
 583     },
 584     migrate_downtime => {
 585         optional => 1,
 586         type => 'number',
 587         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 588         minimum => 0,
 589         default => 0.1,
 590     },
 591     cdrom => {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-ide',
 594         typetext => '<volume>',
 595         description => "This is an alias for option -ide2",
 596     },
 597     cpu => {
 598         optional => 1,
 599         description => "Emulated CPU type.",
 600         type => 'string',
 601         format => 'pve-vm-cpu-conf',
 602     },
 603     parent => get_standard_option('pve-snapshot-name', {
 604         optional => 1,
 605         description => "Parent snapshot name. This is used internally, and should not be modified.",
 606     }),
 607     snaptime => {
 608         optional => 1,
 609         description => "Timestamp for snapshots.",
 610         type => 'integer',
 611         minimum => 0,
 612     },
 613     vmstate => {
 614         optional => 1,
 615         type => 'string', format => 'pve-volume-id',
 616         description => "Reference to a volume which stores the VM state. This is used internally"
 617             ." for snapshots.",
 618     },
 619     vmstatestorage => get_standard_option('pve-storage-id', {
 620         description => "Default storage for VM state volumes/files.",
 621         optional => 1,
 622     }),
 623     runningmachine => get_standard_option('pve-qemu-machine', {
 624         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 625             ." for snapshots.",
 626     }),
 627     runningcpu => {
 628         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 629             ." internally for snapshots.",
 630         optional => 1,
 631         type => 'string',
 632         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 633         format_description => 'QEMU -cpu parameter'
 634     },
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 652             ." remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 667             ." to disable explicitly.",
 668         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 669             ." value identifier to the guest OS. This allows to notify the guest operating system"
 670             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 671             ." execution or creation from a template). The guest operating system notices the"
 672             ." change, and is then able to react as appropriate by marking its copies of"
 673             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 674             ."Note that auto-creation only works when done through API/CLI create or update methods"
 675             .", but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 689             ." the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704     tags => {
 705         type => 'string', format => 'pve-tag-list',
 706         description => 'Tags of the VM. This is only meta information.',
 707         optional => 1,
 708     },
 709     rng0 => {
 710         type => 'string',
 711         format => $rng_fmt,
 712         description => "Configure a VirtIO-based Random Number Generator.",
 713         optional => 1,
 714     },
 715     meta => {
 716         type => 'string',
 717         format => $meta_info_fmt,
 718         description => "Some (read-only) meta-information about this guest.",
 719         optional => 1,
 720     },
 721     affinity => {
 722         type => 'string', format => 'pve-cpuset',
 723         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 724         optional => 1,
 725     },
 726 };
 727
 728 my $cicustom_fmt = {
 729     meta => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all meta data passed to the VM via"
 733             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     network => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     user => {
 745         type => 'string',
 746         optional => 1,
 747         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 748         format => 'pve-volume-id',
 749         format_description => 'volume',
 750     },
 751     vendor => {
 752         type => 'string',
 753         optional => 1,
 754         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 760
 761 my $confdesc_cloudinit = {
 762     citype => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'Specifies the cloud-init configuration format. The default depends on the'
 766             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 767             .' and `configdrive2` for windows.',
 768         enum => ['configdrive2', 'nocloud', 'opennebula'],
 769     },
 770     ciuser => {
 771         optional => 1,
 772         type => 'string',
 773         description => "cloud-init: User name to change ssh keys and password for instead of the"
 774             ." image's configured default user.",
 775     },
 776     cipassword => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Password to assign the user. Using this is generally not'
 780             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 781             .' support hashed passwords.',
 782     },
 783     ciupgrade => {
 784         optional => 1,
 785         type => 'boolean',
 786         description => 'cloud-init: do an automatic package upgrade after the first boot.'
 787     },
 788     cicustom => {
 789         optional => 1,
 790         type => 'string',
 791         description => 'cloud-init: Specify custom files to replace the automatically generated'
 792             .' ones at start.',
 793         format => 'pve-qm-cicustom',
 794     },
 795     searchdomain => {
 796         optional => 1,
 797         type => 'string',
 798         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 799             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 800             .' are set.',
 801     },
 802     nameserver => {
 803         optional => 1,
 804         type => 'string', format => 'address-list',
 805         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 806             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 807             .' are set.',
 808     },
 809     sshkeys => {
 810         optional => 1,
 811         type => 'string',
 812         format => 'urlencoded',
 813         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 814     },
 815 };
 816
 817 # what about other qemu settings ?
 818 #cpu => 'string',
 819 #machine => 'string',
 820 #fda => 'file',
 821 #fdb => 'file',
 822 #mtdblock => 'file',
 823 #sd => 'file',
 824 #pflash => 'file',
 825 #snapshot => 'bool',
 826 #bootp => 'file',
 827 ##tftp => 'dir',
 828 ##smb => 'dir',
 829 #kernel => 'file',
 830 #append => 'string',
 831 #initrd => 'file',
 832 ##soundhw => 'string',
 833
 834 while (my ($k, $v) = each %$confdesc) {
 835     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 836 }
 837
 838 my $MAX_NETS = 32;
 839 my $MAX_SERIAL_PORTS = 4;
 840 my $MAX_PARALLEL_PORTS = 3;
 841 my $MAX_NUMA = 8;
 842
 843 my $numa_fmt = {
 844     cpus => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "CPUs accessing this NUMA node.",
 848         format_description => "id[-id];...",
 849     },
 850     memory => {
 851         type => "number",
 852         description => "Amount of memory this NUMA node provides.",
 853         optional => 1,
 854     },
 855     hostnodes => {
 856         type => "string",
 857         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 858         description => "Host NUMA nodes to use.",
 859         format_description => "id[-id];...",
 860         optional => 1,
 861     },
 862     policy => {
 863         type => 'string',
 864         enum => [qw(preferred bind interleave)],
 865         description => "NUMA allocation policy.",
 866         optional => 1,
 867     },
 868 };
 869 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 870 my $numadesc = {
 871     optional => 1,
 872     type => 'string', format => $numa_fmt,
 873     description => "NUMA topology.",
 874 };
 875 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 876
 877 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 878     $confdesc->{"numa$i"} = $numadesc;
 879 }
 880
 881 my $nic_model_list = [
 882     'e1000',
 883     'e1000-82540em',
 884     'e1000-82544gc',
 885     'e1000-82545em',
 886     'e1000e',
 887     'i82551',
 888     'i82557b',
 889     'i82559er',
 890     'ne2k_isa',
 891     'ne2k_pci',
 892     'pcnet',
 893     'rtl8139',
 894     'virtio',
 895     'vmxnet3',
 896 ];
 897 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 898
 899 my $net_fmt_bridge_descr = <<__EOD__;
 900 Bridge to attach the network device to. The Proxmox VE standard bridge
 901 is called 'vmbr0'.
 902
 903 If you do not specify a bridge, we create a kvm user (NATed) network
 904 device, which provides DHCP and DNS services. The following addresses
 905 are used:
 906
 907  10.0.2.2   Gateway
 908  10.0.2.3   DNS Server
 909  10.0.2.4   SMB Server
 910
 911 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 912 __EOD__
 913
 914 my $net_fmt = {
 915     macaddr  => get_standard_option('mac-addr', {
 916         description => "MAC address. That address must be unique withing your network. This is"
 917             ." automatically generated if not specified.",
 918     }),
 919     model => {
 920         type => 'string',
 921         description => "Network Card Model. The 'virtio' model provides the best performance with"
 922             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 923             ." best to use 'e1000'.",
 924         enum => $nic_model_list,
 925         default_key => 1,
 926     },
 927     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 928     bridge => get_standard_option('pve-bridge-id', {
 929         description => $net_fmt_bridge_descr,
 930         optional => 1,
 931     }),
 932     queues => {
 933         type => 'integer',
 934         minimum => 0, maximum => 64,
 935         description => 'Number of packet queues to be used on the device.',
 936         optional => 1,
 937     },
 938     rate => {
 939         type => 'number',
 940         minimum => 0,
 941         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 942         optional => 1,
 943     },
 944     tag => {
 945         type => 'integer',
 946         minimum => 1, maximum => 4094,
 947         description => 'VLAN tag to apply to packets on this interface.',
 948         optional => 1,
 949     },
 950     trunks => {
 951         type => 'string',
 952         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 953         description => 'VLAN trunks to pass through this interface.',
 954         format_description => 'vlanid[;vlanid...]',
 955         optional => 1,
 956     },
 957     firewall => {
 958         type => 'boolean',
 959         description => 'Whether this interface should be protected by the firewall.',
 960         optional => 1,
 961     },
 962     link_down => {
 963         type => 'boolean',
 964         description => 'Whether this interface should be disconnected (like pulling the plug).',
 965         optional => 1,
 966     },
 967     mtu => {
 968         type => 'integer',
 969         minimum => 1, maximum => 65520,
 970         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 971         optional => 1,
 972     },
 973 };
 974
 975 my $netdesc = {
 976     optional => 1,
 977     type => 'string', format => $net_fmt,
 978     description => "Specify network devices.",
 979 };
 980
 981 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 982
 983 my $ipconfig_fmt = {
 984     ip => {
 985         type => 'string',
 986         format => 'pve-ipv4-config',
 987         format_description => 'IPv4Format/CIDR',
 988         description => 'IPv4 address in CIDR format.',
 989         optional => 1,
 990         default => 'dhcp',
 991     },
 992     gw => {
 993         type => 'string',
 994         format => 'ipv4',
 995         format_description => 'GatewayIPv4',
 996         description => 'Default gateway for IPv4 traffic.',
 997         optional => 1,
 998         requires => 'ip',
 999     },
1000     ip6 => {
1001         type => 'string',
1002         format => 'pve-ipv6-config',
1003         format_description => 'IPv6Format/CIDR',
1004         description => 'IPv6 address in CIDR format.',
1005         optional => 1,
1006         default => 'dhcp',
1007     },
1008     gw6 => {
1009         type => 'string',
1010         format => 'ipv6',
1011         format_description => 'GatewayIPv6',
1012         description => 'Default gateway for IPv6 traffic.',
1013         optional => 1,
1014         requires => 'ip6',
1015     },
1016 };
1017 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1018 my $ipconfigdesc = {
1019     optional => 1,
1020     type => 'string', format => 'pve-qm-ipconfig',
1021     description => <<'EODESCR',
1022 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1023
1024 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1025
1026 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1027 gateway should be provided.
1028 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1029 cloud-init 19.4 or newer.
1030
1031 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1032 dhcp on IPv4.
1033 EODESCR
1034 };
1035 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1036
1037 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1038     $confdesc->{"net$i"} = $netdesc;
1039     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1040 }
1041
1042 foreach my $key (keys %$confdesc_cloudinit) {
1043     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1044 }
1045
1046 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1047 sub pve_verify_cpuset {
1048     my ($set_text, $noerr) = @_;
1049
1050     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1051
1052     if ($@) {
1053         return if $noerr;
1054         die "unable to parse cpuset option\n";
1055     }
1056
1057     return PVE::CpuSet->new($members)->short_string();
1058 }
1059
1060 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1061 sub verify_volume_id_or_qm_path {
1062     my ($volid, $noerr) = @_;
1063
1064     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1065
1066     return verify_volume_id_or_absolute_path($volid, $noerr);
1067 }
1068
1069 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1070 sub verify_volume_id_or_absolute_path {
1071     my ($volid, $noerr) = @_;
1072
1073     return $volid if $volid =~ m|^/|;
1074
1075     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1076     if ($@) {
1077         return if $noerr;
1078         die $@;
1079     }
1080     return $volid;
1081 }
1082
1083 my $serialdesc = {
1084         optional => 1,
1085         type => 'string',
1086         pattern => '(/dev/.+|socket)',
1087         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1088         verbose_description =>  <<EODESCR,
1089 Create a serial device inside the VM (n is 0 to 3), and pass through a
1090 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1091 host side (use 'qm terminal' to open a terminal connection).
1092
1093 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1094 use with special care.
1095
1096 CAUTION: Experimental! User reported problems with this option.
1097 EODESCR
1098 };
1099
1100 my $paralleldesc= {
1101         optional => 1,
1102         type => 'string',
1103         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1104         description =>  "Map host parallel devices (n is 0 to 2).",
1105         verbose_description =>  <<EODESCR,
1106 Map host parallel devices (n is 0 to 2).
1107
1108 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1109 machines - use with special care.
1110
1111 CAUTION: Experimental! User reported problems with this option.
1112 EODESCR
1113 };
1114
1115 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1116     $confdesc->{"parallel$i"} = $paralleldesc;
1117 }
1118
1119 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1120     $confdesc->{"serial$i"} = $serialdesc;
1121 }
1122
1123 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1124     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1125 }
1126
1127 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1128     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1129 }
1130
1131 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++)  {
1132     $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1133 }
1134
1135 my $boot_fmt = {
1136     legacy => {
1137         optional => 1,
1138         default_key => 1,
1139         type => 'string',
1140         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1141                      . " Deprecated, use 'order=' instead.",
1142         pattern => '[acdn]{1,4}',
1143         format_description => "[acdn]{1,4}",
1144
1145         # note: this is also the fallback if boot: is not given at all
1146         default => 'cdn',
1147     },
1148     order => {
1149         optional => 1,
1150         type => 'string',
1151         format => 'pve-qm-bootdev-list',
1152         format_description => "device[;device...]",
1153         description => <<EODESC,
1154 The guest will attempt to boot from devices in the order they appear here.
1155
1156 Disks, optical drives and passed-through storage USB devices will be directly
1157 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1158 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1159
1160 Note that only devices in this list will be marked as bootable and thus loaded
1161 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1162 (e.g. software-raid), you need to specify all of them here.
1163
1164 Overrides the deprecated 'legacy=[acdn]*' value when given.
1165 EODESC
1166     },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1169
1170 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1171 sub verify_bootdev {
1172     my ($dev, $noerr) = @_;
1173
1174     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1175     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1176
1177     my $check = sub {
1178         my ($base) = @_;
1179         return 0 if $dev !~ m/^$base\d+$/;
1180         return 0 if !$confdesc->{$dev};
1181         return 1;
1182     };
1183
1184     return $dev if $check->("net");
1185     return $dev if $check->("usb");
1186     return $dev if $check->("hostpci");
1187
1188     return if $noerr;
1189     die "invalid boot device '$dev'\n";
1190 }
1191
1192 sub print_bootorder {
1193     my ($devs) = @_;
1194     return "" if !@$devs;
1195     my $data = { order => join(';', @$devs) };
1196     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1197 }
1198
1199 my $kvm_api_version = 0;
1200
1201 sub kvm_version {
1202     return $kvm_api_version if $kvm_api_version;
1203
1204     open my $fh, '<', '/dev/kvm' or return;
1205
1206     # 0xae00 => KVM_GET_API_VERSION
1207     $kvm_api_version = ioctl($fh, 0xae00, 0);
1208     close($fh);
1209
1210     return $kvm_api_version;
1211 }
1212
1213 my $kvm_user_version = {};
1214 my $kvm_mtime = {};
1215
1216 sub kvm_user_version {
1217     my ($binary) = @_;
1218
1219     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1220     my $st = stat($binary);
1221
1222     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1223     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1224         $cachedmtime == $st->mtime;
1225
1226     $kvm_user_version->{$binary} = 'unknown';
1227     $kvm_mtime->{$binary} = $st->mtime;
1228
1229     my $code = sub {
1230         my $line = shift;
1231         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1232             $kvm_user_version->{$binary} = $2;
1233         }
1234     };
1235
1236     eval { run_command([$binary, '--version'], outfunc => $code); };
1237     warn $@ if $@;
1238
1239     return $kvm_user_version->{$binary};
1240
1241 }
1242 my sub extract_version {
1243     my ($machine_type, $version) = @_;
1244     $version = kvm_user_version() if !defined($version);
1245     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1246 }
1247
1248 sub kernel_has_vhost_net {
1249     return -c '/dev/vhost-net';
1250 }
1251
1252 sub option_exists {
1253     my $key = shift;
1254     return defined($confdesc->{$key});
1255 }
1256
1257 my $cdrom_path;
1258 sub get_cdrom_path {
1259
1260     return $cdrom_path if defined($cdrom_path);
1261
1262     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1263
1264     if (!defined($cdrom_path)) {
1265         log_warn("no physical CD-ROM available, ignoring");
1266         $cdrom_path = '';
1267     }
1268
1269     return $cdrom_path;
1270 }
1271
1272 sub get_iso_path {
1273     my ($storecfg, $vmid, $cdrom) = @_;
1274
1275     if ($cdrom eq 'cdrom') {
1276         return get_cdrom_path();
1277     } elsif ($cdrom eq 'none') {
1278         return '';
1279     } elsif ($cdrom =~ m|^/|) {
1280         return $cdrom;
1281     } else {
1282         return PVE::Storage::path($storecfg, $cdrom);
1283     }
1284 }
1285
1286 # try to convert old style file names to volume IDs
1287 sub filename_to_volume_id {
1288     my ($vmid, $file, $media) = @_;
1289
1290      if (!($file eq 'none' || $file eq 'cdrom' ||
1291           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1292
1293         return if $file =~ m|/|;
1294
1295         if ($media && $media eq 'cdrom') {
1296             $file = "local:iso/$file";
1297         } else {
1298             $file = "local:$vmid/$file";
1299         }
1300     }
1301
1302     return $file;
1303 }
1304
1305 sub verify_media_type {
1306     my ($opt, $vtype, $media) = @_;
1307
1308     return if !$media;
1309
1310     my $etype;
1311     if ($media eq 'disk') {
1312         $etype = 'images';
1313     } elsif ($media eq 'cdrom') {
1314         $etype = 'iso';
1315     } else {
1316         die "internal error";
1317     }
1318
1319     return if ($vtype eq $etype);
1320
1321     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1322 }
1323
1324 sub cleanup_drive_path {
1325     my ($opt, $storecfg, $drive) = @_;
1326
1327     # try to convert filesystem paths to volume IDs
1328
1329     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1330         ($drive->{file} !~ m|^/dev/.+|) &&
1331         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1332         ($drive->{file} !~ m/^\d+$/)) {
1333         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1334         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1335             if !$vtype;
1336         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1337         verify_media_type($opt, $vtype, $drive->{media});
1338         $drive->{file} = $volid;
1339     }
1340
1341     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1342 }
1343
1344 sub parse_hotplug_features {
1345     my ($data) = @_;
1346
1347     my $res = {};
1348
1349     return $res if $data eq '0';
1350
1351     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1352
1353     foreach my $feature (PVE::Tools::split_list($data)) {
1354         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1355             $res->{$1} = 1;
1356         } else {
1357             die "invalid hotplug feature '$feature'\n";
1358         }
1359     }
1360     return $res;
1361 }
1362
1363 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1364 sub pve_verify_hotplug_features {
1365     my ($value, $noerr) = @_;
1366
1367     return $value if parse_hotplug_features($value);
1368
1369     return if $noerr;
1370
1371     die "unable to parse hotplug option\n";
1372 }
1373
1374 sub scsi_inquiry {
1375     my($fh, $noerr) = @_;
1376
1377     my $SG_IO = 0x2285;
1378     my $SG_GET_VERSION_NUM = 0x2282;
1379
1380     my $versionbuf = "\x00" x 8;
1381     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1382     if (!$ret) {
1383         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1384         return;
1385     }
1386     my $version = unpack("I", $versionbuf);
1387     if ($version < 30000) {
1388         die "scsi generic interface too old\n"  if !$noerr;
1389         return;
1390     }
1391
1392     my $buf = "\x00" x 36;
1393     my $sensebuf = "\x00" x 8;
1394     my $cmd = pack("C x3 C x1", 0x12, 36);
1395
1396     # see /usr/include/scsi/sg.h
1397     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1398
1399     my $packet = pack(
1400         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1401     );
1402
1403     $ret = ioctl($fh, $SG_IO, $packet);
1404     if (!$ret) {
1405         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1406         return;
1407     }
1408
1409     my @res = unpack($sg_io_hdr_t, $packet);
1410     if ($res[17] || $res[18]) {
1411         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1412         return;
1413     }
1414
1415     my $res = {};
1416     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1417
1418     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1419     $res->{type} &= 0x1F;
1420
1421     return $res;
1422 }
1423
1424 sub path_is_scsi {
1425     my ($path) = @_;
1426
1427     my $fh = IO::File->new("+<$path") || return;
1428     my $res = scsi_inquiry($fh, 1);
1429     close($fh);
1430
1431     return $res;
1432 }
1433
1434 sub print_tabletdevice_full {
1435     my ($conf, $arch) = @_;
1436
1437     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1438
1439     # we use uhci for old VMs because tablet driver was buggy in older qemu
1440     my $usbbus;
1441     if ($q35 || $arch eq 'aarch64') {
1442         $usbbus = 'ehci';
1443     } else {
1444         $usbbus = 'uhci';
1445     }
1446
1447     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1448 }
1449
1450 sub print_keyboarddevice_full {
1451     my ($conf, $arch) = @_;
1452
1453     return if $arch ne 'aarch64';
1454
1455     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1456 }
1457
1458 my sub get_drive_id {
1459     my ($drive) = @_;
1460     return "$drive->{interface}$drive->{index}";
1461 }
1462
1463 sub print_drivedevice_full {
1464     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1465
1466     my $device = '';
1467     my $maxdev = 0;
1468
1469     my $drive_id = get_drive_id($drive);
1470     if ($drive->{interface} eq 'virtio') {
1471         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1472         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1473         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1474     } elsif ($drive->{interface} eq 'scsi') {
1475
1476         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1477         my $unit = $drive->{index} % $maxdev;
1478         my $devicetype = 'hd';
1479         my $path = '';
1480         if (drive_is_cdrom($drive)) {
1481             $devicetype = 'cd';
1482         } else {
1483             if ($drive->{file} =~ m|^/|) {
1484                 $path = $drive->{file};
1485                 if (my $info = path_is_scsi($path)) {
1486                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1487                         $devicetype = 'block';
1488                     } elsif ($info->{type} == 1) { # tape
1489                         $devicetype = 'generic';
1490                     }
1491                 }
1492             } else {
1493                  $path = PVE::Storage::path($storecfg, $drive->{file});
1494             }
1495
1496             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1497             my $version = extract_version($machine_type, kvm_user_version());
1498             if ($path =~ m/^iscsi\:\/\// &&
1499                !min_version($version, 4, 1)) {
1500                 $devicetype = 'generic';
1501             }
1502         }
1503
1504         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1505             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1506         } else {
1507             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1508                 .",lun=$drive->{index}";
1509         }
1510         $device .= ",drive=drive-$drive_id,id=$drive_id";
1511
1512         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1513             $device .= ",rotation_rate=1";
1514         }
1515         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1516
1517     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1518         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1519         my $controller = int($drive->{index} / $maxdev);
1520         my $unit = $drive->{index} % $maxdev;
1521         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1522
1523         $device = "ide-$devicetype";
1524         if ($drive->{interface} eq 'ide') {
1525             $device .= ",bus=ide.$controller,unit=$unit";
1526         } else {
1527             $device .= ",bus=ahci$controller.$unit";
1528         }
1529         $device .= ",drive=drive-$drive_id,id=$drive_id";
1530
1531         if ($devicetype eq 'hd') {
1532             if (my $model = $drive->{model}) {
1533                 $model = URI::Escape::uri_unescape($model);
1534                 $device .= ",model=$model";
1535             }
1536             if ($drive->{ssd}) {
1537                 $device .= ",rotation_rate=1";
1538             }
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541     } elsif ($drive->{interface} eq 'usb') {
1542         die "implement me";
1543         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1544     } else {
1545         die "unsupported interface type";
1546     }
1547
1548     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1549
1550     if (my $serial = $drive->{serial}) {
1551         $serial = URI::Escape::uri_unescape($serial);
1552         $device .= ",serial=$serial";
1553     }
1554
1555
1556     return $device;
1557 }
1558
1559 sub get_initiator_name {
1560     my $initiator;
1561
1562     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1563     while (defined(my $line = <$fh>)) {
1564         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1565         $initiator = $1;
1566         last;
1567     }
1568     $fh->close();
1569
1570     return $initiator;
1571 }
1572
1573 my sub storage_allows_io_uring_default {
1574     my ($scfg, $cache_direct) = @_;
1575
1576     # io_uring with cache mode writeback or writethrough on krbd will hang...
1577     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1578
1579     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1580     # sometimes, just plain disable...
1581     return if $scfg && $scfg->{type} eq 'lvm';
1582
1583     # io_uring causes problems when used with CIFS since kernel 5.15
1584     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1585     return if $scfg && $scfg->{type} eq 'cifs';
1586
1587     return 1;
1588 }
1589
1590 my sub drive_uses_cache_direct {
1591     my ($drive, $scfg) = @_;
1592
1593     my $cache_direct = 0;
1594
1595     if (my $cache = $drive->{cache}) {
1596         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1598         $cache_direct = 1;
1599     }
1600
1601     return $cache_direct;
1602 }
1603
1604 sub print_drive_commandline_full {
1605     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1606
1607     my $path;
1608     my $volid = $drive->{file};
1609     my $format = $drive->{format};
1610     my $drive_id = get_drive_id($drive);
1611
1612     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1613     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1614
1615     if (drive_is_cdrom($drive)) {
1616         $path = get_iso_path($storecfg, $vmid, $volid);
1617         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1618     } else {
1619         if ($storeid) {
1620             $path = PVE::Storage::path($storecfg, $volid);
1621             $format //= qemu_img_format($scfg, $volname);
1622         } else {
1623             $path = $volid;
1624             $format //= "raw";
1625         }
1626    }
1627
1628    my $is_rbd = $path =~ m/^rbd:/;
1629
1630     my $opts = '';
1631     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1632     foreach my $o (@qemu_drive_options) {
1633         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1634     }
1635
1636     # snapshot only accepts on|off
1637     if (defined($drive->{snapshot})) {
1638         my $v = $drive->{snapshot} ? 'on' : 'off';
1639         $opts .= ",snapshot=$v";
1640     }
1641
1642     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1643         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1644     }
1645
1646     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1647         my ($dir, $qmpname) = @$type;
1648         if (my $v = $drive->{"mbps$dir"}) {
1649             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1650         }
1651         if (my $v = $drive->{"mbps${dir}_max"}) {
1652             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1653         }
1654         if (my $v = $drive->{"bps${dir}_max_length"}) {
1655             $opts .= ",throttling.bps$qmpname-max-length=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}"}) {
1658             $opts .= ",throttling.iops$qmpname=$v";
1659         }
1660         if (my $v = $drive->{"iops${dir}_max"}) {
1661             $opts .= ",throttling.iops$qmpname-max=$v";
1662         }
1663         if (my $v = $drive->{"iops${dir}_max_length"}) {
1664             $opts .= ",throttling.iops$qmpname-max-length=$v";
1665         }
1666     }
1667
1668     if ($pbs_name) {
1669         $format = "rbd" if $is_rbd;
1670         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1671             if !$format;
1672         $opts .= ",format=alloc-track,file.driver=$format";
1673     } elsif ($format) {
1674         $opts .= ",format=$format";
1675     }
1676
1677     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1678
1679     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1680
1681     if (!$drive->{aio}) {
1682         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1683             # io_uring supports all cache modes
1684             $opts .= ",aio=io_uring";
1685         } else {
1686             # aio native works only with O_DIRECT
1687             if($cache_direct) {
1688                 $opts .= ",aio=native";
1689             } else {
1690                 $opts .= ",aio=threads";
1691             }
1692         }
1693     }
1694
1695     if (!drive_is_cdrom($drive)) {
1696         my $detectzeroes;
1697         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1698             $detectzeroes = 'off';
1699         } elsif ($drive->{discard}) {
1700             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1701         } else {
1702             # This used to be our default with discard not being specified:
1703             $detectzeroes = 'on';
1704         }
1705
1706         # note: 'detect-zeroes' works per blockdev and we want it to persist
1707         # after the alloc-track is removed, so put it on 'file' directly
1708         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1709         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1710     }
1711
1712     if ($pbs_name) {
1713         $opts .= ",backing=$pbs_name";
1714         $opts .= ",auto-remove=on";
1715     }
1716
1717     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1718     my $file_param = "file";
1719     if ($pbs_name) {
1720         # non-rbd drivers require the underlying file to be a seperate block
1721         # node, so add a second .file indirection
1722         $file_param .= ".file" if !$is_rbd;
1723         $file_param .= ".filename";
1724     }
1725     my $pathinfo = $path ? "$file_param=$path," : '';
1726
1727     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1728 }
1729
1730 sub print_pbs_blockdev {
1731     my ($pbs_conf, $pbs_name) = @_;
1732     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1733     $blockdev .= ",repository=$pbs_conf->{repository}";
1734     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1735     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1736     $blockdev .= ",archive=$pbs_conf->{archive}";
1737     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1738     return $blockdev;
1739 }
1740
1741 sub print_netdevice_full {
1742     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1743
1744     my $device = $net->{model};
1745     if ($net->{model} eq 'virtio') {
1746          $device = 'virtio-net-pci';
1747      };
1748
1749     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1750     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1751     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1752         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1753         # and out of each queue plus one config interrupt and control vector queue
1754         my $vectors = $net->{queues} * 2 + 2;
1755         $tmpstr .= ",vectors=$vectors,mq=on";
1756         if (min_version($machine_version, 7, 1)) {
1757             $tmpstr .= ",packed=on";
1758         }
1759     }
1760
1761     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1762         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1763     }
1764
1765     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1766
1767     if (my $mtu = $net->{mtu}) {
1768         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1769             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1770             if ($mtu == 1) {
1771                  $mtu = $bridge_mtu;
1772             } elsif ($mtu < 576) {
1773                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1774             } elsif ($mtu > $bridge_mtu) {
1775                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1776             }
1777             $tmpstr .= ",host_mtu=$mtu";
1778         } else {
1779             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1780         }
1781     }
1782
1783     if ($use_old_bios_files) {
1784         my $romfile;
1785         if ($device eq 'virtio-net-pci') {
1786             $romfile = 'pxe-virtio.rom';
1787         } elsif ($device eq 'e1000') {
1788             $romfile = 'pxe-e1000.rom';
1789         } elsif ($device eq 'e1000e') {
1790             $romfile = 'pxe-e1000e.rom';
1791         } elsif ($device eq 'ne2k') {
1792             $romfile = 'pxe-ne2k_pci.rom';
1793         } elsif ($device eq 'pcnet') {
1794             $romfile = 'pxe-pcnet.rom';
1795         } elsif ($device eq 'rtl8139') {
1796             $romfile = 'pxe-rtl8139.rom';
1797         }
1798         $tmpstr .= ",romfile=$romfile" if $romfile;
1799     }
1800
1801     return $tmpstr;
1802 }
1803
1804 sub print_netdev_full {
1805     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1806
1807     my $i = '';
1808     if ($netid =~ m/^net(\d+)$/) {
1809         $i = int($1);
1810     }
1811
1812     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1813
1814     my $ifname = "tap${vmid}i$i";
1815
1816     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1817     die "interface name '$ifname' is too long (max 15 character)\n"
1818         if length($ifname) >= 16;
1819
1820     my $vhostparam = '';
1821     if (is_native($arch)) {
1822         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1823     }
1824
1825     my $vmname = $conf->{name} || "vm$vmid";
1826
1827     my $netdev = "";
1828     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1829
1830     if ($net->{bridge}) {
1831         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1832             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1833     } else {
1834         $netdev = "type=user,id=$netid,hostname=$vmname";
1835     }
1836
1837     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1838
1839     return $netdev;
1840 }
1841
1842 my $vga_map = {
1843     'cirrus' => 'cirrus-vga',
1844     'std' => 'VGA',
1845     'vmware' => 'vmware-svga',
1846     'virtio' => 'virtio-vga',
1847     'virtio-gl' => 'virtio-vga-gl',
1848 };
1849
1850 sub print_vga_device {
1851     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1852
1853     my $type = $vga_map->{$vga->{type}};
1854     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1855         $type = 'virtio-gpu';
1856     }
1857     my $vgamem_mb = $vga->{memory};
1858
1859     my $max_outputs = '';
1860     if ($qxlnum) {
1861         $type = $id ? 'qxl' : 'qxl-vga';
1862
1863         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1864             # set max outputs so linux can have up to 4 qxl displays with one device
1865             if (min_version($machine_version, 4, 1)) {
1866                 $max_outputs = ",max_outputs=4";
1867             }
1868         }
1869     }
1870
1871     die "no devicetype for $vga->{type}\n" if !$type;
1872
1873     my $memory = "";
1874     if ($vgamem_mb) {
1875         if ($vga->{type} =~ /^virtio/) {
1876             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1877             $memory = ",max_hostmem=$bytes";
1878         } elsif ($qxlnum) {
1879             # from https://www.spice-space.org/multiple-monitors.html
1880             $memory = ",vgamem_mb=$vga->{memory}";
1881             my $ram = $vgamem_mb * 4;
1882             my $vram = $vgamem_mb * 2;
1883             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1884         } else {
1885             $memory = ",vgamem_mb=$vga->{memory}";
1886         }
1887     } elsif ($qxlnum && $id) {
1888         $memory = ",ram_size=67108864,vram_size=33554432";
1889     }
1890
1891     my $edidoff = "";
1892     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1893         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1894     }
1895
1896     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1897     my $vgaid = "vga" . ($id // '');
1898     my $pciaddr;
1899     if ($q35 && $vgaid eq 'vga') {
1900         # the first display uses pcie.0 bus on q35 machines
1901         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1902     } else {
1903         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1904     }
1905
1906     if ($vga->{type} eq 'virtio-gl') {
1907         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1908         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1909             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1910
1911         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1912             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1913     }
1914
1915     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1916 }
1917
1918 sub parse_number_sets {
1919     my ($set) = @_;
1920     my $res = [];
1921     foreach my $part (split(/;/, $set)) {
1922         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1923             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1924             push @$res, [ $1, $2 ];
1925         } else {
1926             die "invalid range: $part\n";
1927         }
1928     }
1929     return $res;
1930 }
1931
1932 sub parse_numa {
1933     my ($data) = @_;
1934
1935     my $res = parse_property_string($numa_fmt, $data);
1936     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1937     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1938     return $res;
1939 }
1940
1941 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1942 sub parse_net {
1943     my ($data, $disable_mac_autogen) = @_;
1944
1945     my $res = eval { parse_property_string($net_fmt, $data) };
1946     if ($@) {
1947         warn $@;
1948         return;
1949     }
1950     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1951         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1952         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1953     }
1954     return $res;
1955 }
1956
1957 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1958 sub parse_ipconfig {
1959     my ($data) = @_;
1960
1961     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1962     if ($@) {
1963         warn $@;
1964         return;
1965     }
1966
1967     if ($res->{gw} && !$res->{ip}) {
1968         warn 'gateway specified without specifying an IP address';
1969         return;
1970     }
1971     if ($res->{gw6} && !$res->{ip6}) {
1972         warn 'IPv6 gateway specified without specifying an IPv6 address';
1973         return;
1974     }
1975     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1976         warn 'gateway specified together with DHCP';
1977         return;
1978     }
1979     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1980         # gw6 + auto/dhcp
1981         warn "IPv6 gateway specified together with $res->{ip6} address";
1982         return;
1983     }
1984
1985     if (!$res->{ip} && !$res->{ip6}) {
1986         return { ip => 'dhcp', ip6 => 'dhcp' };
1987     }
1988
1989     return $res;
1990 }
1991
1992 sub print_net {
1993     my $net = shift;
1994
1995     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1996 }
1997
1998 sub add_random_macs {
1999     my ($settings) = @_;
2000
2001     foreach my $opt (keys %$settings) {
2002         next if $opt !~ m/^net(\d+)$/;
2003         my $net = parse_net($settings->{$opt});
2004         next if !$net;
2005         $settings->{$opt} = print_net($net);
2006     }
2007 }
2008
2009 sub vm_is_volid_owner {
2010     my ($storecfg, $vmid, $volid) = @_;
2011
2012     if ($volid !~  m|^/|) {
2013         my ($path, $owner);
2014         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2015         if ($owner && ($owner == $vmid)) {
2016             return 1;
2017         }
2018     }
2019
2020     return;
2021 }
2022
2023 sub vmconfig_register_unused_drive {
2024     my ($storecfg, $vmid, $conf, $drive) = @_;
2025
2026     if (drive_is_cloudinit($drive)) {
2027         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2028         warn $@ if $@;
2029         delete $conf->{cloudinit};
2030     } elsif (!drive_is_cdrom($drive)) {
2031         my $volid = $drive->{file};
2032         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2033             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2034         }
2035     }
2036 }
2037
2038 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2039 my $smbios1_fmt = {
2040     uuid => {
2041         type => 'string',
2042         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2043         format_description => 'UUID',
2044         description => "Set SMBIOS1 UUID.",
2045         optional => 1,
2046     },
2047     version => {
2048         type => 'string',
2049         pattern => '[A-Za-z0-9+\/]+={0,2}',
2050         format_description => 'Base64 encoded string',
2051         description => "Set SMBIOS1 version.",
2052         optional => 1,
2053     },
2054     serial => {
2055         type => 'string',
2056         pattern => '[A-Za-z0-9+\/]+={0,2}',
2057         format_description => 'Base64 encoded string',
2058         description => "Set SMBIOS1 serial number.",
2059         optional => 1,
2060     },
2061     manufacturer => {
2062         type => 'string',
2063         pattern => '[A-Za-z0-9+\/]+={0,2}',
2064         format_description => 'Base64 encoded string',
2065         description => "Set SMBIOS1 manufacturer.",
2066         optional => 1,
2067     },
2068     product => {
2069         type => 'string',
2070         pattern => '[A-Za-z0-9+\/]+={0,2}',
2071         format_description => 'Base64 encoded string',
2072         description => "Set SMBIOS1 product ID.",
2073         optional => 1,
2074     },
2075     sku => {
2076         type => 'string',
2077         pattern => '[A-Za-z0-9+\/]+={0,2}',
2078         format_description => 'Base64 encoded string',
2079         description => "Set SMBIOS1 SKU string.",
2080         optional => 1,
2081     },
2082     family => {
2083         type => 'string',
2084         pattern => '[A-Za-z0-9+\/]+={0,2}',
2085         format_description => 'Base64 encoded string',
2086         description => "Set SMBIOS1 family string.",
2087         optional => 1,
2088     },
2089     base64 => {
2090         type => 'boolean',
2091         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2092         optional => 1,
2093     },
2094 };
2095
2096 sub parse_smbios1 {
2097     my ($data) = @_;
2098
2099     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2100     warn $@ if $@;
2101     return $res;
2102 }
2103
2104 sub print_smbios1 {
2105     my ($smbios1) = @_;
2106     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2107 }
2108
2109 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2110
2111 sub parse_watchdog {
2112     my ($value) = @_;
2113
2114     return if !$value;
2115
2116     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2117     warn $@ if $@;
2118     return $res;
2119 }
2120
2121 sub parse_guest_agent {
2122     my ($conf) = @_;
2123
2124     return {} if !defined($conf->{agent});
2125
2126     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2127     warn $@ if $@;
2128
2129     # if the agent is disabled ignore the other potentially set properties
2130     return {} if !$res->{enabled};
2131     return $res;
2132 }
2133
2134 sub get_qga_key {
2135     my ($conf, $key) = @_;
2136     return undef if !defined($conf->{agent});
2137
2138     my $agent = parse_guest_agent($conf);
2139     return $agent->{$key};
2140 }
2141
2142 sub parse_vga {
2143     my ($value) = @_;
2144
2145     return {} if !$value;
2146     my $res = eval { parse_property_string($vga_fmt, $value) };
2147     warn $@ if $@;
2148     return $res;
2149 }
2150
2151 sub parse_rng {
2152     my ($value) = @_;
2153
2154     return if !$value;
2155
2156     my $res = eval { parse_property_string($rng_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_meta_info {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub new_meta_info_string {
2172     my () = @_; # for now do not allow to override any value
2173
2174     return PVE::JSONSchema::print_property_string(
2175         {
2176             'creation-qemu' => kvm_user_version(),
2177             ctime => "". int(time()),
2178         },
2179         $meta_info_fmt
2180     );
2181 }
2182
2183 sub qemu_created_version_fixups {
2184     my ($conf, $forcemachine, $kvmver) = @_;
2185
2186     my $meta = parse_meta_info($conf->{meta}) // {};
2187     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2188
2189     # check if we need to apply some handling for VMs that always use the latest machine version but
2190     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2191     # would be required (we do not want to pin machine version for non-windows OS type)
2192     if (
2193         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2194         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2195         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2196         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2197     ) {
2198         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2199         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2200             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2201             # and thus with the predictable interface naming of systemd
2202             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2203         }
2204     }
2205     return;
2206 }
2207
2208 # add JSON properties for create and set function
2209 sub json_config_properties {
2210     my ($prop, $with_disk_alloc) = @_;
2211
2212     my $skip_json_config_opts = {
2213         parent => 1,
2214         snaptime => 1,
2215         vmstate => 1,
2216         runningmachine => 1,
2217         runningcpu => 1,
2218         meta => 1,
2219     };
2220
2221     foreach my $opt (keys %$confdesc) {
2222         next if $skip_json_config_opts->{$opt};
2223
2224         if ($with_disk_alloc && is_valid_drivename($opt)) {
2225             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2226         } else {
2227             $prop->{$opt} = $confdesc->{$opt};
2228         }
2229     }
2230
2231     return $prop;
2232 }
2233
2234 # Properties that we can read from an OVF file
2235 sub json_ovf_properties {
2236     my $prop = {};
2237
2238     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2239         $prop->{$device} = {
2240             type => 'string',
2241             format => 'pve-volume-id-or-absolute-path',
2242             description => "Disk image that gets imported to $device",
2243             optional => 1,
2244         };
2245     }
2246
2247     $prop->{cores} = {
2248         type => 'integer',
2249         description => "The number of CPU cores.",
2250         optional => 1,
2251     };
2252     $prop->{memory} = {
2253         type => 'integer',
2254         description => "Amount of RAM for the VM in MB.",
2255         optional => 1,
2256     };
2257     $prop->{name} = {
2258         type => 'string',
2259         description => "Name of the VM.",
2260         optional => 1,
2261     };
2262
2263     return $prop;
2264 }
2265
2266 # return copy of $confdesc_cloudinit to generate documentation
2267 sub cloudinit_config_properties {
2268
2269     return dclone($confdesc_cloudinit);
2270 }
2271
2272 sub cloudinit_pending_properties {
2273     my $p = {
2274         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2275         name => 1,
2276     };
2277     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2278     return $p;
2279 }
2280
2281 sub check_type {
2282     my ($key, $value) = @_;
2283
2284     die "unknown setting '$key'\n" if !$confdesc->{$key};
2285
2286     my $type = $confdesc->{$key}->{type};
2287
2288     if (!defined($value)) {
2289         die "got undefined value\n";
2290     }
2291
2292     if ($value =~ m/[\n\r]/) {
2293         die "property contains a line feed\n";
2294     }
2295
2296     if ($type eq 'boolean') {
2297         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2298         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2299         die "type check ('boolean') failed - got '$value'\n";
2300     } elsif ($type eq 'integer') {
2301         return int($1) if $value =~ m/^(\d+)$/;
2302         die "type check ('integer') failed - got '$value'\n";
2303     } elsif ($type eq 'number') {
2304         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2305         die "type check ('number') failed - got '$value'\n";
2306     } elsif ($type eq 'string') {
2307         if (my $fmt = $confdesc->{$key}->{format}) {
2308             PVE::JSONSchema::check_format($fmt, $value);
2309             return $value;
2310         }
2311         $value =~ s/^\"(.*)\"$/$1/;
2312         return $value;
2313     } else {
2314         die "internal error"
2315     }
2316 }
2317
2318 sub destroy_vm {
2319     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2320
2321     my $conf = PVE::QemuConfig->load_config($vmid);
2322
2323     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2324         PVE::QemuConfig->check_lock($conf);
2325     }
2326
2327     if ($conf->{template}) {
2328         # check if any base image is still used by a linked clone
2329         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2330                 my ($ds, $drive) = @_;
2331                 return if drive_is_cdrom($drive);
2332
2333                 my $volid = $drive->{file};
2334                 return if !$volid || $volid =~ m|^/|;
2335
2336                 die "base volume '$volid' is still in use by linked cloned\n"
2337                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2338
2339         });
2340     }
2341
2342     my $volids = {};
2343     my $remove_owned_drive = sub {
2344         my ($ds, $drive) = @_;
2345         return if drive_is_cdrom($drive, 1);
2346
2347         my $volid = $drive->{file};
2348         return if !$volid || $volid =~ m|^/|;
2349         return if $volids->{$volid};
2350
2351         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2352         return if !$path || !$owner || ($owner != $vmid);
2353
2354         $volids->{$volid} = 1;
2355         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2356         warn "Could not remove disk '$volid', check manually: $@" if $@;
2357     };
2358
2359     # only remove disks owned by this VM (referenced in the config)
2360     my $include_opts = {
2361         include_unused => 1,
2362         extra_keys => ['vmstate'],
2363     };
2364     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2365
2366     for my $snap (values %{$conf->{snapshots}}) {
2367         next if !defined($snap->{vmstate});
2368         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2369         next if !defined($drive);
2370         $remove_owned_drive->('vmstate', $drive);
2371     }
2372
2373     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2374
2375     if ($purge_unreferenced) { # also remove unreferenced disk
2376         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2377         PVE::Storage::foreach_volid($vmdisks, sub {
2378             my ($volid, $sid, $volname, $d) = @_;
2379             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2380             warn $@ if $@;
2381         });
2382     }
2383
2384     if (defined $replacement_conf) {
2385         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2386     } else {
2387         PVE::QemuConfig->destroy_config($vmid);
2388     }
2389 }
2390
2391 sub parse_vm_config {
2392     my ($filename, $raw, $strict) = @_;
2393
2394     return if !defined($raw);
2395
2396     my $res = {
2397         digest => Digest::SHA::sha1_hex($raw),
2398         snapshots => {},
2399         pending => {},
2400         cloudinit => {},
2401     };
2402
2403     my $handle_error = sub {
2404         my ($msg) = @_;
2405
2406         if ($strict) {
2407             die $msg;
2408         } else {
2409             warn $msg;
2410         }
2411     };
2412
2413     $filename =~ m|/qemu-server/(\d+)\.conf$|
2414         || die "got strange filename '$filename'";
2415
2416     my $vmid = $1;
2417
2418     my $conf = $res;
2419     my $descr;
2420     my $finish_description = sub {
2421         if (defined($descr)) {
2422             $descr =~ s/\s+$//;
2423             $conf->{description} = $descr;
2424         }
2425         $descr = undef;
2426     };
2427     my $section = '';
2428
2429     my @lines = split(/\n/, $raw);
2430     foreach my $line (@lines) {
2431         next if $line =~ m/^\s*$/;
2432
2433         if ($line =~ m/^\[PENDING\]\s*$/i) {
2434             $section = 'pending';
2435             $finish_description->();
2436             $conf = $res->{$section} = {};
2437             next;
2438         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2439             $section = 'cloudinit';
2440             $finish_description->();
2441             $conf = $res->{$section} = {};
2442             next;
2443
2444         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2445             $section = $1;
2446             $finish_description->();
2447             $conf = $res->{snapshots}->{$section} = {};
2448             next;
2449         }
2450
2451         if ($line =~ m/^\#(.*)$/) {
2452             $descr = '' if !defined($descr);
2453             $descr .= PVE::Tools::decode_text($1) . "\n";
2454             next;
2455         }
2456
2457         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2458             $descr = '' if !defined($descr);
2459             $descr .= PVE::Tools::decode_text($2);
2460         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2461             $conf->{snapstate} = $1;
2462         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2463             my $key = $1;
2464             my $value = $2;
2465             $conf->{$key} = $value;
2466         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2467             my $value = $1;
2468             if ($section eq 'pending') {
2469                 $conf->{delete} = $value; # we parse this later
2470             } else {
2471                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2472             }
2473         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2474             my $key = $1;
2475             my $value = $2;
2476             if ($section eq 'cloudinit') {
2477                 # ignore validation only used for informative purpose
2478                 $conf->{$key} = $value;
2479                 next;
2480             }
2481             eval { $value = check_type($key, $value); };
2482             if ($@) {
2483                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2484             } else {
2485                 $key = 'ide2' if $key eq 'cdrom';
2486                 my $fmt = $confdesc->{$key}->{format};
2487                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2488                     my $v = parse_drive($key, $value);
2489                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2490                         $v->{file} = $volid;
2491                         $value = print_drive($v);
2492                     } else {
2493                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2494                         next;
2495                     }
2496                 }
2497
2498                 $conf->{$key} = $value;
2499             }
2500         } else {
2501             $handle_error->("vm $vmid - unable to parse config: $line\n");
2502         }
2503     }
2504
2505     $finish_description->();
2506     delete $res->{snapstate}; # just to be sure
2507
2508     return $res;
2509 }
2510
2511 sub write_vm_config {
2512     my ($filename, $conf) = @_;
2513
2514     delete $conf->{snapstate}; # just to be sure
2515
2516     if ($conf->{cdrom}) {
2517         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2518         $conf->{ide2} = $conf->{cdrom};
2519         delete $conf->{cdrom};
2520     }
2521
2522     # we do not use 'smp' any longer
2523     if ($conf->{sockets}) {
2524         delete $conf->{smp};
2525     } elsif ($conf->{smp}) {
2526         $conf->{sockets} = $conf->{smp};
2527         delete $conf->{cores};
2528         delete $conf->{smp};
2529     }
2530
2531     my $used_volids = {};
2532
2533     my $cleanup_config = sub {
2534         my ($cref, $pending, $snapname) = @_;
2535
2536         foreach my $key (keys %$cref) {
2537             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2538                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2539             my $value = $cref->{$key};
2540             if ($key eq 'delete') {
2541                 die "propertry 'delete' is only allowed in [PENDING]\n"
2542                     if !$pending;
2543                 # fixme: check syntax?
2544                 next;
2545             }
2546             eval { $value = check_type($key, $value); };
2547             die "unable to parse value of '$key' - $@" if $@;
2548
2549             $cref->{$key} = $value;
2550
2551             if (!$snapname && is_valid_drivename($key)) {
2552                 my $drive = parse_drive($key, $value);
2553                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2554             }
2555         }
2556     };
2557
2558     &$cleanup_config($conf);
2559
2560     &$cleanup_config($conf->{pending}, 1);
2561
2562     foreach my $snapname (keys %{$conf->{snapshots}}) {
2563         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2564         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2565     }
2566
2567     # remove 'unusedX' settings if we re-add a volume
2568     foreach my $key (keys %$conf) {
2569         my $value = $conf->{$key};
2570         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2571             delete $conf->{$key};
2572         }
2573     }
2574
2575     my $generate_raw_config = sub {
2576         my ($conf, $pending) = @_;
2577
2578         my $raw = '';
2579
2580         # add description as comment to top of file
2581         if (defined(my $descr = $conf->{description})) {
2582             if ($descr) {
2583                 foreach my $cl (split(/\n/, $descr)) {
2584                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2585                 }
2586             } else {
2587                 $raw .= "#\n" if $pending;
2588             }
2589         }
2590
2591         foreach my $key (sort keys %$conf) {
2592             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2593             $raw .= "$key: $conf->{$key}\n";
2594         }
2595         return $raw;
2596     };
2597
2598     my $raw = &$generate_raw_config($conf);
2599
2600     if (scalar(keys %{$conf->{pending}})){
2601         $raw .= "\n[PENDING]\n";
2602         $raw .= &$generate_raw_config($conf->{pending}, 1);
2603     }
2604
2605     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2606         $raw .= "\n[special:cloudinit]\n";
2607         $raw .= &$generate_raw_config($conf->{cloudinit});
2608     }
2609
2610     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2611         $raw .= "\n[$snapname]\n";
2612         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2613     }
2614
2615     return $raw;
2616 }
2617
2618 sub load_defaults {
2619
2620     my $res = {};
2621
2622     # we use static defaults from our JSON schema configuration
2623     foreach my $key (keys %$confdesc) {
2624         if (defined(my $default = $confdesc->{$key}->{default})) {
2625             $res->{$key} = $default;
2626         }
2627     }
2628
2629     return $res;
2630 }
2631
2632 sub config_list {
2633     my $vmlist = PVE::Cluster::get_vmlist();
2634     my $res = {};
2635     return $res if !$vmlist || !$vmlist->{ids};
2636     my $ids = $vmlist->{ids};
2637     my $nodename = nodename();
2638
2639     foreach my $vmid (keys %$ids) {
2640         my $d = $ids->{$vmid};
2641         next if !$d->{node} || $d->{node} ne $nodename;
2642         next if !$d->{type} || $d->{type} ne 'qemu';
2643         $res->{$vmid}->{exists} = 1;
2644     }
2645     return $res;
2646 }
2647
2648 # test if VM uses local resources (to prevent migration)
2649 sub check_local_resources {
2650     my ($conf, $noerr) = @_;
2651
2652     my @loc_res = ();
2653
2654     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2655     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2656
2657     push @loc_res, "ivshmem" if $conf->{ivshmem};
2658
2659     foreach my $k (keys %$conf) {
2660         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2661         # sockets are safe: they will recreated be on the target side post-migrate
2662         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2663         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2664     }
2665
2666     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2667
2668     return \@loc_res;
2669 }
2670
2671 # check if used storages are available on all nodes (use by migrate)
2672 sub check_storage_availability {
2673     my ($storecfg, $conf, $node) = @_;
2674
2675     PVE::QemuConfig->foreach_volume($conf, sub {
2676         my ($ds, $drive) = @_;
2677
2678         my $volid = $drive->{file};
2679         return if !$volid;
2680
2681         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2682         return if !$sid;
2683
2684         # check if storage is available on both nodes
2685         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2686         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2687
2688         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2689
2690         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2691             if !$scfg->{content}->{$vtype};
2692    });
2693 }
2694
2695 # list nodes where all VM images are available (used by has_feature API)
2696 sub shared_nodes {
2697     my ($conf, $storecfg) = @_;
2698
2699     my $nodelist = PVE::Cluster::get_nodelist();
2700     my $nodehash = { map { $_ => 1 } @$nodelist };
2701     my $nodename = nodename();
2702
2703     PVE::QemuConfig->foreach_volume($conf, sub {
2704         my ($ds, $drive) = @_;
2705
2706         my $volid = $drive->{file};
2707         return if !$volid;
2708
2709         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2710         if ($storeid) {
2711             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2712             if ($scfg->{disable}) {
2713                 $nodehash = {};
2714             } elsif (my $avail = $scfg->{nodes}) {
2715                 foreach my $node (keys %$nodehash) {
2716                     delete $nodehash->{$node} if !$avail->{$node};
2717                 }
2718             } elsif (!$scfg->{shared}) {
2719                 foreach my $node (keys %$nodehash) {
2720                     delete $nodehash->{$node} if $node ne $nodename
2721                 }
2722             }
2723         }
2724     });
2725
2726     return $nodehash
2727 }
2728
2729 sub check_local_storage_availability {
2730     my ($conf, $storecfg) = @_;
2731
2732     my $nodelist = PVE::Cluster::get_nodelist();
2733     my $nodehash = { map { $_ => {} } @$nodelist };
2734
2735     PVE::QemuConfig->foreach_volume($conf, sub {
2736         my ($ds, $drive) = @_;
2737
2738         my $volid = $drive->{file};
2739         return if !$volid;
2740
2741         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2742         if ($storeid) {
2743             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2744
2745             if ($scfg->{disable}) {
2746                 foreach my $node (keys %$nodehash) {
2747                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2748                 }
2749             } elsif (my $avail = $scfg->{nodes}) {
2750                 foreach my $node (keys %$nodehash) {
2751                     if (!$avail->{$node}) {
2752                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2753                     }
2754                 }
2755             }
2756         }
2757     });
2758
2759     foreach my $node (values %$nodehash) {
2760         if (my $unavail = $node->{unavailable_storages}) {
2761             $node->{unavailable_storages} = [ sort keys %$unavail ];
2762         }
2763     }
2764
2765     return $nodehash
2766 }
2767
2768 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2769 sub check_running {
2770     my ($vmid, $nocheck, $node) = @_;
2771
2772     # $nocheck is set when called during a migration, in which case the config
2773     # file might still or already reside on the *other* node
2774     # - because rename has already happened, and current node is source
2775     # - because rename hasn't happened yet, and current node is target
2776     # - because rename has happened, current node is target, but hasn't yet
2777     # processed it yet
2778     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2779     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2780 }
2781
2782 sub vzlist {
2783
2784     my $vzlist = config_list();
2785
2786     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2787
2788     while (defined(my $de = $fd->read)) {
2789         next if $de !~ m/^(\d+)\.pid$/;
2790         my $vmid = $1;
2791         next if !defined($vzlist->{$vmid});
2792         if (my $pid = check_running($vmid)) {
2793             $vzlist->{$vmid}->{pid} = $pid;
2794         }
2795     }
2796
2797     return $vzlist;
2798 }
2799
2800 our $vmstatus_return_properties = {
2801     vmid => get_standard_option('pve-vmid'),
2802     status => {
2803         description => "QEMU process status.",
2804         type => 'string',
2805         enum => ['stopped', 'running'],
2806     },
2807     maxmem => {
2808         description => "Maximum memory in bytes.",
2809         type => 'integer',
2810         optional => 1,
2811         renderer => 'bytes',
2812     },
2813     maxdisk => {
2814         description => "Root disk size in bytes.",
2815         type => 'integer',
2816         optional => 1,
2817         renderer => 'bytes',
2818     },
2819     name => {
2820         description => "VM name.",
2821         type => 'string',
2822         optional => 1,
2823     },
2824     qmpstatus => {
2825         description => "VM run state from the 'query-status' QMP monitor command.",
2826         type => 'string',
2827         optional => 1,
2828     },
2829     pid => {
2830         description => "PID of running qemu process.",
2831         type => 'integer',
2832         optional => 1,
2833     },
2834     uptime => {
2835         description => "Uptime.",
2836         type => 'integer',
2837         optional => 1,
2838         renderer => 'duration',
2839     },
2840     cpus => {
2841         description => "Maximum usable CPUs.",
2842         type => 'number',
2843         optional => 1,
2844     },
2845     lock => {
2846         description => "The current config lock, if any.",
2847         type => 'string',
2848         optional => 1,
2849     },
2850     tags => {
2851         description  => "The current configured tags, if any",
2852         type => 'string',
2853         optional => 1,
2854     },
2855     'running-machine' => {
2856         description => "The currently running machine type (if running).",
2857         type => 'string',
2858         optional => 1,
2859     },
2860     'running-qemu' => {
2861         description => "The currently running QEMU version (if running).",
2862         type => 'string',
2863         optional => 1,
2864     },
2865 };
2866
2867 my $last_proc_pid_stat;
2868
2869 # get VM status information
2870 # This must be fast and should not block ($full == false)
2871 # We only query KVM using QMP if $full == true (this can be slow)
2872 sub vmstatus {
2873     my ($opt_vmid, $full) = @_;
2874
2875     my $res = {};
2876
2877     my $storecfg = PVE::Storage::config();
2878
2879     my $list = vzlist();
2880     my $defaults = load_defaults();
2881
2882     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2883
2884     my $cpucount = $cpuinfo->{cpus} || 1;
2885
2886     foreach my $vmid (keys %$list) {
2887         next if $opt_vmid && ($vmid ne $opt_vmid);
2888
2889         my $conf = PVE::QemuConfig->load_config($vmid);
2890
2891         my $d = { vmid => int($vmid) };
2892         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2893
2894         # fixme: better status?
2895         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2896
2897         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2898         if (defined($size)) {
2899             $d->{disk} = 0; # no info available
2900             $d->{maxdisk} = $size;
2901         } else {
2902             $d->{disk} = 0;
2903             $d->{maxdisk} = 0;
2904         }
2905
2906         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2907             * ($conf->{cores} || $defaults->{cores});
2908         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2909         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2910
2911         $d->{name} = $conf->{name} || "VM $vmid";
2912         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2913             : $defaults->{memory}*(1024*1024);
2914
2915         if ($conf->{balloon}) {
2916             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2917             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2918                 : $defaults->{shares};
2919         }
2920
2921         $d->{uptime} = 0;
2922         $d->{cpu} = 0;
2923         $d->{mem} = 0;
2924
2925         $d->{netout} = 0;
2926         $d->{netin} = 0;
2927
2928         $d->{diskread} = 0;
2929         $d->{diskwrite} = 0;
2930
2931         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2932
2933         $d->{serial} = 1 if conf_has_serial($conf);
2934         $d->{lock} = $conf->{lock} if $conf->{lock};
2935         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2936
2937         $res->{$vmid} = $d;
2938     }
2939
2940     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2941     foreach my $dev (keys %$netdev) {
2942         next if $dev !~ m/^tap([1-9]\d*)i/;
2943         my $vmid = $1;
2944         my $d = $res->{$vmid};
2945         next if !$d;
2946
2947         $d->{netout} += $netdev->{$dev}->{receive};
2948         $d->{netin} += $netdev->{$dev}->{transmit};
2949
2950         if ($full) {
2951             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2952             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2953         }
2954
2955     }
2956
2957     my $ctime = gettimeofday;
2958
2959     foreach my $vmid (keys %$list) {
2960
2961         my $d = $res->{$vmid};
2962         my $pid = $d->{pid};
2963         next if !$pid;
2964
2965         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2966         next if !$pstat; # not running
2967
2968         my $used = $pstat->{utime} + $pstat->{stime};
2969
2970         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2971
2972         if ($pstat->{vsize}) {
2973             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2974         }
2975
2976         my $old = $last_proc_pid_stat->{$pid};
2977         if (!$old) {
2978             $last_proc_pid_stat->{$pid} = {
2979                 time => $ctime,
2980                 used => $used,
2981                 cpu => 0,
2982             };
2983             next;
2984         }
2985
2986         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2987
2988         if ($dtime > 1000) {
2989             my $dutime = $used -  $old->{used};
2990
2991             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2992             $last_proc_pid_stat->{$pid} = {
2993                 time => $ctime,
2994                 used => $used,
2995                 cpu => $d->{cpu},
2996             };
2997         } else {
2998             $d->{cpu} = $old->{cpu};
2999         }
3000     }
3001
3002     return $res if !$full;
3003
3004     my $qmpclient = PVE::QMPClient->new();
3005
3006     my $ballooncb = sub {
3007         my ($vmid, $resp) = @_;
3008
3009         my $info = $resp->{'return'};
3010         return if !$info->{max_mem};
3011
3012         my $d = $res->{$vmid};
3013
3014         # use memory assigned to VM
3015         $d->{maxmem} = $info->{max_mem};
3016         $d->{balloon} = $info->{actual};
3017
3018         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3019             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3020             $d->{freemem} = $info->{free_mem};
3021         }
3022
3023         $d->{ballooninfo} = $info;
3024     };
3025
3026     my $blockstatscb = sub {
3027         my ($vmid, $resp) = @_;
3028         my $data = $resp->{'return'} || [];
3029         my $totalrdbytes = 0;
3030         my $totalwrbytes = 0;
3031
3032         for my $blockstat (@$data) {
3033             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3034             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3035
3036             $blockstat->{device} =~ s/drive-//;
3037             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3038         }
3039         $res->{$vmid}->{diskread} = $totalrdbytes;
3040         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3041     };
3042
3043     my $machinecb = sub {
3044         my ($vmid, $resp) = @_;
3045         my $data = $resp->{'return'} || [];
3046
3047         $res->{$vmid}->{'running-machine'} =
3048             PVE::QemuServer::Machine::current_from_query_machines($data);
3049     };
3050
3051     my $versioncb = sub {
3052         my ($vmid, $resp) = @_;
3053         my $data = $resp->{'return'} // {};
3054         my $version = 'unknown';
3055
3056         if (my $v = $data->{qemu}) {
3057             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3058         }
3059
3060         $res->{$vmid}->{'running-qemu'} = $version;
3061     };
3062
3063     my $statuscb = sub {
3064         my ($vmid, $resp) = @_;
3065
3066         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3067         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3068         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3069         # this fails if ballon driver is not loaded, so this must be
3070         # the last commnand (following command are aborted if this fails).
3071         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3072
3073         my $status = 'unknown';
3074         if (!defined($status = $resp->{'return'}->{status})) {
3075             warn "unable to get VM status\n";
3076             return;
3077         }
3078
3079         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3080     };
3081
3082     foreach my $vmid (keys %$list) {
3083         next if $opt_vmid && ($vmid ne $opt_vmid);
3084         next if !$res->{$vmid}->{pid}; # not running
3085         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3086     }
3087
3088     $qmpclient->queue_execute(undef, 2);
3089
3090     foreach my $vmid (keys %$list) {
3091         next if $opt_vmid && ($vmid ne $opt_vmid);
3092         next if !$res->{$vmid}->{pid}; #not running
3093
3094         # we can't use the $qmpclient since it might have already aborted on
3095         # 'query-balloon', but this might also fail for older versions...
3096         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3097         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3098     }
3099
3100     foreach my $vmid (keys %$list) {
3101         next if $opt_vmid && ($vmid ne $opt_vmid);
3102         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3103     }
3104
3105     return $res;
3106 }
3107
3108 sub conf_has_serial {
3109     my ($conf) = @_;
3110
3111     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3112         if ($conf->{"serial$i"}) {
3113             return 1;
3114         }
3115     }
3116
3117     return 0;
3118 }
3119
3120 sub conf_has_audio {
3121     my ($conf, $id) = @_;
3122
3123     $id //= 0;
3124     my $audio = $conf->{"audio$id"};
3125     return if !defined($audio);
3126
3127     my $audioproperties = parse_property_string($audio_fmt, $audio);
3128     my $audiodriver = $audioproperties->{driver} // 'spice';
3129
3130     return {
3131         dev => $audioproperties->{device},
3132         dev_id => "audiodev$id",
3133         backend => $audiodriver,
3134         backend_id => "$audiodriver-backend${id}",
3135     };
3136 }
3137
3138 sub audio_devs {
3139     my ($audio, $audiopciaddr, $machine_version) = @_;
3140
3141     my $devs = [];
3142
3143     my $id = $audio->{dev_id};
3144     my $audiodev = "";
3145     if (min_version($machine_version, 4, 2)) {
3146         $audiodev = ",audiodev=$audio->{backend_id}";
3147     }
3148
3149     if ($audio->{dev} eq 'AC97') {
3150         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3151     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3152         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3153         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3154         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3155     } else {
3156         die "unkown audio device '$audio->{dev}', implement me!";
3157     }
3158
3159     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3160
3161     return $devs;
3162 }
3163
3164 sub get_tpm_paths {
3165     my ($vmid) = @_;
3166     return {
3167         socket => "/var/run/qemu-server/$vmid.swtpm",
3168         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3169     };
3170 }
3171
3172 sub add_tpm_device {
3173     my ($vmid, $devices, $conf) = @_;
3174
3175     return if !$conf->{tpmstate0};
3176
3177     my $paths = get_tpm_paths($vmid);
3178
3179     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3180     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3181     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3182 }
3183
3184 sub start_swtpm {
3185     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3186
3187     return if !$tpmdrive;
3188
3189     my $state;
3190     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3191     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3192     if ($storeid) {
3193         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3194     } else {
3195         $state = $tpm->{file};
3196     }
3197
3198     my $paths = get_tpm_paths($vmid);
3199
3200     # during migration, we will get state from remote
3201     #
3202     if (!$migration) {
3203         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3204         my $setup_cmd = [
3205             "swtpm_setup",
3206             "--tpmstate",
3207             "file://$state",
3208             "--createek",
3209             "--create-ek-cert",
3210             "--create-platform-cert",
3211             "--lock-nvram",
3212             "--config",
3213             "/etc/swtpm_setup.conf", # do not use XDG configs
3214             "--runas",
3215             "0", # force creation as root, error if not possible
3216             "--not-overwrite", # ignore existing state, do not modify
3217         ];
3218
3219         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3220         # TPM 2.0 supports ECC crypto, use if possible
3221         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3222
3223         run_command($setup_cmd, outfunc => sub {
3224             print "swtpm_setup: $1\n";
3225         });
3226     }
3227
3228     # Used to distinguish different invocations in the log.
3229     my $log_prefix = "[id=" . int(time()) . "] ";
3230
3231     my $emulator_cmd = [
3232         "swtpm",
3233         "socket",
3234         "--tpmstate",
3235         "backend-uri=file://$state,mode=0600",
3236         "--ctrl",
3237         "type=unixio,path=$paths->{socket},mode=0600",
3238         "--pid",
3239         "file=$paths->{pid}",
3240         "--terminate", # terminate on QEMU disconnect
3241         "--daemon",
3242         "--log",
3243         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3244     ];
3245     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3246     run_command($emulator_cmd, outfunc => sub { print $1; });
3247
3248     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3249     while (! -e $paths->{pid}) {
3250         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3251         usleep(50_000);
3252     }
3253
3254     # return untainted PID of swtpm daemon so it can be killed on error
3255     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3256     return $1;
3257 }
3258
3259 sub vga_conf_has_spice {
3260     my ($vga) = @_;
3261
3262     my $vgaconf = parse_vga($vga);
3263     my $vgatype = $vgaconf->{type};
3264     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3265
3266     return $1 || 1;
3267 }
3268
3269 sub is_native($) {
3270     my ($arch) = @_;
3271     return get_host_arch() eq $arch;
3272 }
3273
3274 sub get_vm_arch {
3275     my ($conf) = @_;
3276     return $conf->{arch} // get_host_arch();
3277 }
3278
3279 my $default_machines = {
3280     x86_64 => 'pc',
3281     aarch64 => 'virt',
3282 };
3283
3284 sub get_installed_machine_version {
3285     my ($kvmversion) = @_;
3286     $kvmversion = kvm_user_version() if !defined($kvmversion);
3287     $kvmversion =~ m/^(\d+\.\d+)/;
3288     return $1;
3289 }
3290
3291 sub windows_get_pinned_machine_version {
3292     my ($machine, $base_version, $kvmversion) = @_;
3293
3294     my $pin_version = $base_version;
3295     if (!defined($base_version) ||
3296         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3297     ) {
3298         $pin_version = get_installed_machine_version($kvmversion);
3299     }
3300     if (!$machine || $machine eq 'pc') {
3301         $machine = "pc-i440fx-$pin_version";
3302     } elsif ($machine eq 'q35') {
3303         $machine = "pc-q35-$pin_version";
3304     } elsif ($machine eq 'virt') {
3305         $machine = "virt-$pin_version";
3306     } else {
3307         warn "unknown machine type '$machine', not touching that!\n";
3308     }
3309
3310     return $machine;
3311 }
3312
3313 sub get_vm_machine {
3314     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3315
3316     my $machine = $forcemachine || $conf->{machine};
3317
3318     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3319         $kvmversion //= kvm_user_version();
3320         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3321         # layout which confuses windows quite a bit and may result in various regressions..
3322         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3323         if (windows_version($conf->{ostype})) {
3324             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3325         }
3326         $arch //= 'x86_64';
3327         $machine ||= $default_machines->{$arch};
3328         if ($add_pve_version) {
3329             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3330             $machine .= "+pve$pvever";
3331         }
3332     }
3333
3334     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3335         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3336         $machine = $1 if $is_pxe;
3337
3338         # for version-pinned machines that do not include a pve-version (e.g.
3339         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3340         $machine .= '+pve0';
3341
3342         $machine .= '.pxe' if $is_pxe;
3343     }
3344
3345     return $machine;
3346 }
3347
3348 sub get_ovmf_files($$$) {
3349     my ($arch, $efidisk, $smm) = @_;
3350
3351     my $types = $OVMF->{$arch}
3352         or die "no OVMF images known for architecture '$arch'\n";
3353
3354     my $type = 'default';
3355     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3356         $type = $smm ? "4m" : "4m-no-smm";
3357         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3358     }
3359
3360     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3361     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3362     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3363
3364     return ($ovmf_code, $ovmf_vars);
3365 }
3366
3367 my $Arch2Qemu = {
3368     aarch64 => '/usr/bin/qemu-system-aarch64',
3369     x86_64 => '/usr/bin/qemu-system-x86_64',
3370 };
3371 sub get_command_for_arch($) {
3372     my ($arch) = @_;
3373     return '/usr/bin/kvm' if is_native($arch);
3374
3375     my $cmd = $Arch2Qemu->{$arch}
3376         or die "don't know how to emulate architecture '$arch'\n";
3377     return $cmd;
3378 }
3379
3380 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3381 # to use in a QEMU command line (-cpu element), first array_intersect the result
3382 # of query_supported_ with query_understood_. This is necessary because:
3383 #
3384 # a) query_understood_ returns flags the host cannot use and
3385 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3386 #    flags, but CPU settings - with most of them being flags. Those settings
3387 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3388 #
3389 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3390 # expensive. If you need the value returned from this, you can get it much
3391 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3392 # $accel being 'kvm' or 'tcg'.
3393 #
3394 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3395 # changes, automatically populating pmxcfs.
3396 #
3397 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3398 # since kvm and tcg machines support different flags
3399 #
3400 sub query_supported_cpu_flags {
3401     my ($arch) = @_;
3402
3403     $arch //= get_host_arch();
3404     my $default_machine = $default_machines->{$arch};
3405
3406     my $flags = {};
3407
3408     # FIXME: Once this is merged, the code below should work for ARM as well:
3409     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3410     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3411         $arch eq "aarch64";
3412
3413     my $kvm_supported = defined(kvm_version());
3414     my $qemu_cmd = get_command_for_arch($arch);
3415     my $fakevmid = -1;
3416     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3417
3418     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3419     my $query_supported_run_qemu = sub {
3420         my ($kvm) = @_;
3421
3422         my $flags = {};
3423         my $cmd = [
3424             $qemu_cmd,
3425             '-machine', $default_machine,
3426             '-display', 'none',
3427             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3428             '-mon', 'chardev=qmp,mode=control',
3429             '-pidfile', $pidfile,
3430             '-S', '-daemonize'
3431         ];
3432
3433         if (!$kvm) {
3434             push @$cmd, '-accel', 'tcg';
3435         }
3436
3437         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3438         die "QEMU flag querying VM exited with code " . $rc if $rc;
3439
3440         eval {
3441             my $cmd_result = mon_cmd(
3442                 $fakevmid,
3443                 'query-cpu-model-expansion',
3444                 type => 'full',
3445                 model => { name => 'host' }
3446             );
3447
3448             my $props = $cmd_result->{model}->{props};
3449             foreach my $prop (keys %$props) {
3450                 next if $props->{$prop} ne '1';
3451                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3452                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3453                 # We only keep those with underscores, to match /proc/cpuinfo
3454                 $prop =~ s/\.|-/_/g;
3455                 $flags->{$prop} = 1;
3456             }
3457         };
3458         my $err = $@;
3459
3460         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3461         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3462
3463         die $err if $err;
3464
3465         return [ sort keys %$flags ];
3466     };
3467
3468     # We need to query QEMU twice, since KVM and TCG have different supported flags
3469     PVE::QemuConfig->lock_config($fakevmid, sub {
3470         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3471         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3472
3473         if ($kvm_supported) {
3474             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3475             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3476         }
3477     });
3478
3479     return $flags;
3480 }
3481
3482 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3483 my $understood_cpu_flag_dir = "/usr/share/kvm";
3484 sub query_understood_cpu_flags {
3485     my $arch = get_host_arch();
3486     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3487
3488     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3489         if ! -e $filepath;
3490
3491     my $raw = file_get_contents($filepath);
3492     $raw =~ s/^\s+|\s+$//g;
3493     my @flags = split(/\s+/, $raw);
3494
3495     return \@flags;
3496 }
3497
3498 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3499 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3500 my sub should_disable_smm {
3501     my ($conf, $vga, $machine) = @_;
3502
3503     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3504
3505     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3506         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3507 }
3508
3509 my sub print_ovmf_drive_commandlines {
3510     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3511
3512     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3513
3514     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3515
3516     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3517     if ($d) {
3518         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3519         my ($path, $format) = $d->@{'file', 'format'};
3520         if ($storeid) {
3521             $path = PVE::Storage::path($storecfg, $d->{file});
3522             if (!defined($format)) {
3523                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3524                 $format = qemu_img_format($scfg, $volname);
3525             }
3526         } elsif (!defined($format)) {
3527             die "efidisk format must be specified\n";
3528         }
3529         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3530         if ($path =~ m/^rbd:/) {
3531             $var_drive_str .= ',cache=writeback';
3532             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3533         }
3534         $var_drive_str .= ",format=$format,file=$path";
3535
3536         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3537         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3538     } else {
3539         log_warn("no efidisk configured! Using temporary efivars disk.");
3540         my $path = "/tmp/$vmid-ovmf.fd";
3541         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3542         $var_drive_str .= ",format=raw,file=$path";
3543         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3544     }
3545
3546     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3547 }
3548
3549 sub config_to_command {
3550     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3551         $pbs_backing) = @_;
3552
3553     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3554     my $devices = [];
3555     my $bridges = {};
3556     my $ostype = $conf->{ostype};
3557     my $winversion = windows_version($ostype);
3558     my $kvm = $conf->{kvm};
3559     my $nodename = nodename();
3560
3561     my $arch = get_vm_arch($conf);
3562     my $kvm_binary = get_command_for_arch($arch);
3563     my $kvmver = kvm_user_version($kvm_binary);
3564
3565     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3566         $kvmver //= "undefined";
3567         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3568     }
3569
3570     my $add_pve_version = min_version($kvmver, 4, 1);
3571
3572     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3573     my $machine_version = extract_version($machine_type, $kvmver);
3574     $kvm //= 1 if is_native($arch);
3575
3576     $machine_version =~ m/(\d+)\.(\d+)/;
3577     my ($machine_major, $machine_minor) = ($1, $2);
3578
3579     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3580         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3581     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3582         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3583             ." please upgrade node '$nodename'\n"
3584     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3585         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3586         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3587             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3588             ." node '$nodename'\n";
3589     }
3590
3591     # if a specific +pve version is required for a feature, use $version_guard
3592     # instead of min_version to allow machines to be run with the minimum
3593     # required version
3594     my $required_pve_version = 0;
3595     my $version_guard = sub {
3596         my ($major, $minor, $pve) = @_;
3597         return 0 if !min_version($machine_version, $major, $minor, $pve);
3598         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3599         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3600         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3601         return 1;
3602     };
3603
3604     if ($kvm && !defined kvm_version()) {
3605         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3606             ." or enable in BIOS.\n";
3607     }
3608
3609     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3610     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3611     my $use_old_bios_files = undef;
3612     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3613
3614     my $cmd = [];
3615     if ($conf->{affinity}) {
3616         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3617     }
3618
3619     push @$cmd, $kvm_binary;
3620
3621     push @$cmd, '-id', $vmid;
3622
3623     my $vmname = $conf->{name} || "vm$vmid";
3624
3625     push @$cmd, '-name', "$vmname,debug-threads=on";
3626
3627     push @$cmd, '-no-shutdown';
3628
3629     my $use_virtio = 0;
3630
3631     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3632     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3633     push @$cmd, '-mon', "chardev=qmp,mode=control";
3634
3635     if (min_version($machine_version, 2, 12)) {
3636         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3637         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3638     }
3639
3640     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3641
3642     push @$cmd, '-daemonize';
3643
3644     if ($conf->{smbios1}) {
3645         my $smbios_conf = parse_smbios1($conf->{smbios1});
3646         if ($smbios_conf->{base64}) {
3647             # Do not pass base64 flag to qemu
3648             delete $smbios_conf->{base64};
3649             my $smbios_string = "";
3650             foreach my $key (keys %$smbios_conf) {
3651                 my $value;
3652                 if ($key eq "uuid") {
3653                     $value = $smbios_conf->{uuid}
3654                 } else {
3655                     $value = decode_base64($smbios_conf->{$key});
3656                 }
3657                 # qemu accepts any binary data, only commas need escaping by double comma
3658                 $value =~ s/,/,,/g;
3659                 $smbios_string .= "," . $key . "=" . $value if $value;
3660             }
3661             push @$cmd, '-smbios', "type=1" . $smbios_string;
3662         } else {
3663             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3664         }
3665     }
3666
3667     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3668         my ($code_drive_str, $var_drive_str) =
3669             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3670         push $cmd->@*, '-drive', $code_drive_str;
3671         push $cmd->@*, '-drive', $var_drive_str;
3672     }
3673
3674     if ($q35) { # tell QEMU to load q35 config early
3675         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3676         if (min_version($machine_version, 4, 0)) {
3677             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3678         } else {
3679             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3680         }
3681     }
3682
3683     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3684         push @$cmd, $fixups->@*;
3685     }
3686
3687     if ($conf->{vmgenid}) {
3688         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3689     }
3690
3691     # add usb controllers
3692     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3693         $conf, $bridges, $arch, $machine_type, $machine_version);
3694     push @$devices, @usbcontrollers if @usbcontrollers;
3695     my $vga = parse_vga($conf->{vga});
3696
3697     my $qxlnum = vga_conf_has_spice($conf->{vga});
3698     $vga->{type} = 'qxl' if $qxlnum;
3699
3700     if (!$vga->{type}) {
3701         if ($arch eq 'aarch64') {
3702             $vga->{type} = 'virtio';
3703         } elsif (min_version($machine_version, 2, 9)) {
3704             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3705         } else {
3706             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3707         }
3708     }
3709
3710     # enable absolute mouse coordinates (needed by vnc)
3711     my $tablet = $conf->{tablet};
3712     if (!defined($tablet)) {
3713         $tablet = $defaults->{tablet};
3714         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3715         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3716     }
3717
3718     if ($tablet) {
3719         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3720         my $kbd = print_keyboarddevice_full($conf, $arch);
3721         push @$devices, '-device', $kbd if defined($kbd);
3722     }
3723
3724     my $bootorder = device_bootorder($conf);
3725
3726     # host pci device passthrough
3727     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3728         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3729
3730     # usb devices
3731     my $usb_dev_features = {};
3732     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3733
3734     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3735         $conf, $usb_dev_features, $bootorder, $machine_version);
3736     push @$devices, @usbdevices if @usbdevices;
3737
3738     # serial devices
3739     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3740         my $path = $conf->{"serial$i"} or next;
3741         if ($path eq 'socket') {
3742             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3743             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3744             # On aarch64, serial0 is the UART device. QEMU only allows
3745             # connecting UART devices via the '-serial' command line, as
3746             # the device has a fixed slot on the hardware...
3747             if ($arch eq 'aarch64' && $i == 0) {
3748                 push @$devices, '-serial', "chardev:serial$i";
3749             } else {
3750                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3751             }
3752         } else {
3753             die "no such serial device\n" if ! -c $path;
3754             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3755             push @$devices, '-device', "isa-serial,chardev=serial$i";
3756         }
3757     }
3758
3759     # parallel devices
3760     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3761         if (my $path = $conf->{"parallel$i"}) {
3762             die "no such parallel device\n" if ! -c $path;
3763             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3764             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3765             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3766         }
3767     }
3768
3769     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3770         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3771         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3772         push @$devices, @$audio_devs;
3773     }
3774
3775     add_tpm_device($vmid, $devices, $conf);
3776
3777     my $sockets = 1;
3778     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3779     $sockets = $conf->{sockets} if  $conf->{sockets};
3780
3781     my $cores = $conf->{cores} || 1;
3782
3783     my $maxcpus = $sockets * $cores;
3784
3785     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3786
3787     my $allowed_vcpus = $cpuinfo->{cpus};
3788
3789     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3790
3791     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3792         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3793         for (my $i = 2; $i <= $vcpus; $i++)  {
3794             my $cpustr = print_cpu_device($conf,$i);
3795             push @$cmd, '-device', $cpustr;
3796         }
3797
3798     } else {
3799
3800         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3801     }
3802     push @$cmd, '-nodefaults';
3803
3804     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3805
3806     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3807
3808     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3809
3810     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3811         push @$devices, '-device', print_vga_device(
3812             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3813
3814         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3815
3816         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3817         push @$cmd,  '-vnc', "unix:$socket,password=on";
3818     } else {
3819         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3820         push @$cmd, '-nographic';
3821     }
3822
3823     # time drift fix
3824     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3825     my $useLocaltime = $conf->{localtime};
3826
3827     if ($winversion >= 5) { # windows
3828         $useLocaltime = 1 if !defined($conf->{localtime});
3829
3830         # use time drift fix when acpi is enabled
3831         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3832             $tdf = 1 if !defined($conf->{tdf});
3833         }
3834     }
3835
3836     if ($winversion >= 6) {
3837         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3838         push @$machineFlags, 'hpet=off';
3839     }
3840
3841     push @$rtcFlags, 'driftfix=slew' if $tdf;
3842
3843     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3844         push @$rtcFlags, "base=$conf->{startdate}";
3845     } elsif ($useLocaltime) {
3846         push @$rtcFlags, 'base=localtime';
3847     }
3848
3849     if ($forcecpu) {
3850         push @$cmd, '-cpu', $forcecpu;
3851     } else {
3852         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3853     }
3854
3855     PVE::QemuServer::Memory::config(
3856         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3857
3858     push @$cmd, '-S' if $conf->{freeze};
3859
3860     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3861
3862     my $guest_agent = parse_guest_agent($conf);
3863
3864     if ($guest_agent->{enabled}) {
3865         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3866         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3867
3868         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3869             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3870             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3871             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3872         } elsif ($guest_agent->{type} eq 'isa') {
3873             push @$devices, '-device', "isa-serial,chardev=qga0";
3874         }
3875     }
3876
3877     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3878     if ($rng && $version_guard->(4, 1, 2)) {
3879         check_rng_source($rng->{source});
3880
3881         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3882         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3883         my $limiter_str = "";
3884         if ($max_bytes) {
3885             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3886         }
3887
3888         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3889         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3890         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3891     }
3892
3893     my $spice_port;
3894
3895     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3896         if ($qxlnum > 1) {
3897             if ($winversion){
3898                 for (my $i = 1; $i < $qxlnum; $i++){
3899                     push @$devices, '-device', print_vga_device(
3900                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3901                 }
3902             } else {
3903                 # assume other OS works like Linux
3904                 my ($ram, $vram) = ("134217728", "67108864");
3905                 if ($vga->{memory}) {
3906                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3907                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3908                 }
3909                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3910                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3911             }
3912         }
3913
3914         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3915
3916         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3917         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3918         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3919
3920         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3921         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3922         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3923
3924         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3925         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3926
3927         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3928         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3929         if ($spice_enhancement->{foldersharing}) {
3930             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3931             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3932         }
3933
3934         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3935         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3936             if $spice_enhancement->{videostreaming};
3937
3938         push @$devices, '-spice', "$spice_opts";
3939     }
3940
3941     # enable balloon by default, unless explicitly disabled
3942     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3943         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3944         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3945         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3946         push @$devices, '-device', $ballooncmd;
3947     }
3948
3949     if ($conf->{watchdog}) {
3950         my $wdopts = parse_watchdog($conf->{watchdog});
3951         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3952         my $watchdog = $wdopts->{model} || 'i6300esb';
3953         push @$devices, '-device', "$watchdog$pciaddr";
3954         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3955     }
3956
3957     my $vollist = [];
3958     my $scsicontroller = {};
3959     my $ahcicontroller = {};
3960     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3961
3962     # Add iscsi initiator name if available
3963     if (my $initiator = get_initiator_name()) {
3964         push @$devices, '-iscsi', "initiator-name=$initiator";
3965     }
3966
3967     PVE::QemuConfig->foreach_volume($conf, sub {
3968         my ($ds, $drive) = @_;
3969
3970         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3971             check_volume_storage_type($storecfg, $drive->{file});
3972             push @$vollist, $drive->{file};
3973         }
3974
3975         # ignore efidisk here, already added in bios/fw handling code above
3976         return if $drive->{interface} eq 'efidisk';
3977         # similar for TPM
3978         return if $drive->{interface} eq 'tpmstate';
3979
3980         $use_virtio = 1 if $ds =~ m/^virtio/;
3981
3982         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3983
3984         if ($drive->{interface} eq 'virtio'){
3985            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3986         }
3987
3988         if ($drive->{interface} eq 'scsi') {
3989
3990             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3991
3992             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3993                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3994
3995             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3996             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3997
3998             my $iothread = '';
3999             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4000                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4001                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4002             } elsif ($drive->{iothread}) {
4003                 log_warn(
4004                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4005                 );
4006             }
4007
4008             my $queues = '';
4009             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4010                 $queues = ",num_queues=$drive->{queues}";
4011             }
4012
4013             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4014                 if !$scsicontroller->{$controller};
4015             $scsicontroller->{$controller}=1;
4016         }
4017
4018         if ($drive->{interface} eq 'sata') {
4019             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4020             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4021             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4022                 if !$ahcicontroller->{$controller};
4023             $ahcicontroller->{$controller}=1;
4024         }
4025
4026         my $pbs_conf = $pbs_backing->{$ds};
4027         my $pbs_name = undef;
4028         if ($pbs_conf) {
4029             $pbs_name = "drive-$ds-pbs";
4030             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4031         }
4032
4033         my $drive_cmd = print_drive_commandline_full(
4034             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4035
4036         # extra protection for templates, but SATA and IDE don't support it..
4037         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4038
4039         push @$devices, '-drive',$drive_cmd;
4040         push @$devices, '-device', print_drivedevice_full(
4041             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4042     });
4043
4044     for (my $i = 0; $i < $MAX_NETS; $i++) {
4045         my $netname = "net$i";
4046
4047         next if !$conf->{$netname};
4048         my $d = parse_net($conf->{$netname});
4049         next if !$d;
4050         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4051
4052         $use_virtio = 1 if $d->{model} eq 'virtio';
4053
4054         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4055
4056         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4057         push @$devices, '-netdev', $netdevfull;
4058
4059         my $netdevicefull = print_netdevice_full(
4060             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4061
4062         push @$devices, '-device', $netdevicefull;
4063     }
4064
4065     if ($conf->{ivshmem}) {
4066         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4067
4068         my $bus;
4069         if ($q35) {
4070             $bus = print_pcie_addr("ivshmem");
4071         } else {
4072             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4073         }
4074
4075         my $ivshmem_name = $ivshmem->{name} // $vmid;
4076         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4077
4078         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4079         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4080             .",size=$ivshmem->{size}M";
4081     }
4082
4083     # pci.4 is nested in pci.1
4084     $bridges->{1} = 1 if $bridges->{4};
4085
4086     if (!$q35) { # add pci bridges
4087         if (min_version($machine_version, 2, 3)) {
4088            $bridges->{1} = 1;
4089            $bridges->{2} = 1;
4090         }
4091         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4092     }
4093
4094     for my $k (sort {$b cmp $a} keys %$bridges) {
4095         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4096
4097         my $k_name = $k;
4098         if ($k == 2 && $legacy_igd) {
4099             $k_name = "$k-igd";
4100         }
4101         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4102         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4103
4104         if ($q35) { # add after -readconfig pve-q35.cfg
4105             splice @$devices, 2, 0, '-device', $devstr;
4106         } else {
4107             unshift @$devices, '-device', $devstr if $k > 0;
4108         }
4109     }
4110
4111     if (!$kvm) {
4112         push @$machineFlags, 'accel=tcg';
4113     }
4114
4115     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4116
4117     my $machine_type_min = $machine_type;
4118     if ($add_pve_version) {
4119         $machine_type_min =~ s/\+pve\d+$//;
4120         $machine_type_min .= "+pve$required_pve_version";
4121     }
4122     push @$machineFlags, "type=${machine_type_min}";
4123
4124     push @$cmd, @$devices;
4125     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4126     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4127     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4128
4129     if (my $vmstate = $conf->{vmstate}) {
4130         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4131         push @$vollist, $vmstate;
4132         push @$cmd, '-loadstate', $statepath;
4133         print "activating and using '$vmstate' as vmstate\n";
4134     }
4135
4136     if (PVE::QemuConfig->is_template($conf)) {
4137         # needed to workaround base volumes being read-only
4138         push @$cmd, '-snapshot';
4139     }
4140
4141     # add custom args
4142     if ($conf->{args}) {
4143         my $aa = PVE::Tools::split_args($conf->{args});
4144         push @$cmd, @$aa;
4145     }
4146
4147     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4148 }
4149
4150 sub check_rng_source {
4151     my ($source) = @_;
4152
4153     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4154     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4155         if ! -e $source;
4156
4157     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4158     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4159         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4160         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4161         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4162             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4163             ." to the host.\n";
4164     }
4165 }
4166
4167 sub spice_port {
4168     my ($vmid) = @_;
4169
4170     my $res = mon_cmd($vmid, 'query-spice');
4171
4172     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4173 }
4174
4175 sub vm_devices_list {
4176     my ($vmid) = @_;
4177
4178     my $res = mon_cmd($vmid, 'query-pci');
4179     my $devices_to_check = [];
4180     my $devices = {};
4181     foreach my $pcibus (@$res) {
4182         push @$devices_to_check, @{$pcibus->{devices}},
4183     }
4184
4185     while (@$devices_to_check) {
4186         my $to_check = [];
4187         for my $d (@$devices_to_check) {
4188             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4189             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4190
4191             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4192             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4193         }
4194         $devices_to_check = $to_check;
4195     }
4196
4197     my $resblock = mon_cmd($vmid, 'query-block');
4198     foreach my $block (@$resblock) {
4199         if($block->{device} =~ m/^drive-(\S+)/){
4200                 $devices->{$1} = 1;
4201         }
4202     }
4203
4204     my $resmice = mon_cmd($vmid, 'query-mice');
4205     foreach my $mice (@$resmice) {
4206         if ($mice->{name} eq 'QEMU HID Tablet') {
4207             $devices->{tablet} = 1;
4208             last;
4209         }
4210     }
4211
4212     # for usb devices there is no query-usb
4213     # but we can iterate over the entries in
4214     # qom-list path=/machine/peripheral
4215     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4216     foreach my $per (@$resperipheral) {
4217         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4218             $devices->{$per->{name}} = 1;
4219         }
4220     }
4221
4222     return $devices;
4223 }
4224
4225 sub vm_deviceplug {
4226     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4227
4228     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4229
4230     my $devices_list = vm_devices_list($vmid);
4231     return 1 if defined($devices_list->{$deviceid});
4232
4233     # add PCI bridge if we need it for the device
4234     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4235
4236     if ($deviceid eq 'tablet') {
4237         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4238     } elsif ($deviceid eq 'keyboard') {
4239         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4240     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4241         my $id = $1;
4242         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4243         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4244     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4245         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4246     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4247         qemu_iothread_add($vmid, $deviceid, $device);
4248
4249         qemu_driveadd($storecfg, $vmid, $device);
4250         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4251
4252         qemu_deviceadd($vmid, $devicefull);
4253         eval { qemu_deviceaddverify($vmid, $deviceid); };
4254         if (my $err = $@) {
4255             eval { qemu_drivedel($vmid, $deviceid); };
4256             warn $@ if $@;
4257             die $err;
4258         }
4259     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4260         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4261         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4262         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4263
4264         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4265
4266         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4267             qemu_iothread_add($vmid, $deviceid, $device);
4268             $devicefull .= ",iothread=iothread-$deviceid";
4269         }
4270
4271         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4272             $devicefull .= ",num_queues=$device->{queues}";
4273         }
4274
4275         qemu_deviceadd($vmid, $devicefull);
4276         qemu_deviceaddverify($vmid, $deviceid);
4277     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4278         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4279         qemu_driveadd($storecfg, $vmid, $device);
4280
4281         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4282         eval { qemu_deviceadd($vmid, $devicefull); };
4283         if (my $err = $@) {
4284             eval { qemu_drivedel($vmid, $deviceid); };
4285             warn $@ if $@;
4286             die $err;
4287         }
4288     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4289         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4290
4291         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4292         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4293         my $use_old_bios_files = undef;
4294         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4295
4296         my $netdevicefull = print_netdevice_full(
4297             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4298         qemu_deviceadd($vmid, $netdevicefull);
4299         eval {
4300             qemu_deviceaddverify($vmid, $deviceid);
4301             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4302         };
4303         if (my $err = $@) {
4304             eval { qemu_netdevdel($vmid, $deviceid); };
4305             warn $@ if $@;
4306             die $err;
4307         }
4308     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4309         my $bridgeid = $2;
4310         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4311         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4312
4313         qemu_deviceadd($vmid, $devicefull);
4314         qemu_deviceaddverify($vmid, $deviceid);
4315     } else {
4316         die "can't hotplug device '$deviceid'\n";
4317     }
4318
4319     return 1;
4320 }
4321
4322 # fixme: this should raise exceptions on error!
4323 sub vm_deviceunplug {
4324     my ($vmid, $conf, $deviceid) = @_;
4325
4326     my $devices_list = vm_devices_list($vmid);
4327     return 1 if !defined($devices_list->{$deviceid});
4328
4329     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4330     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4331
4332     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4333         qemu_devicedel($vmid, $deviceid);
4334     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4335         qemu_devicedel($vmid, $deviceid);
4336         qemu_devicedelverify($vmid, $deviceid);
4337     } elsif ($deviceid =~ m/^usb\d+$/) {
4338         qemu_devicedel($vmid, $deviceid);
4339         qemu_devicedelverify($vmid, $deviceid);
4340     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4341         my $device = parse_drive($deviceid, $conf->{$deviceid});
4342
4343         qemu_devicedel($vmid, $deviceid);
4344         qemu_devicedelverify($vmid, $deviceid);
4345         qemu_drivedel($vmid, $deviceid);
4346         qemu_iothread_del($vmid, $deviceid, $device);
4347     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4348         qemu_devicedel($vmid, $deviceid);
4349         qemu_devicedelverify($vmid, $deviceid);
4350     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4351         my $device = parse_drive($deviceid, $conf->{$deviceid});
4352
4353         qemu_devicedel($vmid, $deviceid);
4354         qemu_devicedelverify($vmid, $deviceid);
4355         qemu_drivedel($vmid, $deviceid);
4356         qemu_deletescsihw($conf, $vmid, $deviceid);
4357
4358         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4359             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4360     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4361         qemu_devicedel($vmid, $deviceid);
4362         qemu_devicedelverify($vmid, $deviceid);
4363         qemu_netdevdel($vmid, $deviceid);
4364     } else {
4365         die "can't unplug device '$deviceid'\n";
4366     }
4367
4368     return 1;
4369 }
4370
4371 sub qemu_spice_usbredir_chardev_add {
4372     my ($vmid, $id) = @_;
4373
4374     mon_cmd($vmid, "chardev-add" , (
4375         id => $id,
4376         backend => {
4377             type => 'spicevmc',
4378             data => {
4379                 type => "usbredir",
4380             },
4381         },
4382     ));
4383 }
4384
4385 sub qemu_deviceadd {
4386     my ($vmid, $devicefull) = @_;
4387
4388     $devicefull = "driver=".$devicefull;
4389     my %options =  split(/[=,]/, $devicefull);
4390
4391     mon_cmd($vmid, "device_add" , %options);
4392 }
4393
4394 sub qemu_devicedel {
4395     my ($vmid, $deviceid) = @_;
4396
4397     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4398 }
4399
4400 sub qemu_iothread_add {
4401     my ($vmid, $deviceid, $device) = @_;
4402
4403     if ($device->{iothread}) {
4404         my $iothreads = vm_iothreads_list($vmid);
4405         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4406     }
4407 }
4408
4409 sub qemu_iothread_del {
4410     my ($vmid, $deviceid, $device) = @_;
4411
4412     if ($device->{iothread}) {
4413         my $iothreads = vm_iothreads_list($vmid);
4414         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4415     }
4416 }
4417
4418 sub qemu_objectadd {
4419     my ($vmid, $objectid, $qomtype) = @_;
4420
4421     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4422
4423     return 1;
4424 }
4425
4426 sub qemu_objectdel {
4427     my ($vmid, $objectid) = @_;
4428
4429     mon_cmd($vmid, "object-del", id => $objectid);
4430
4431     return 1;
4432 }
4433
4434 sub qemu_driveadd {
4435     my ($storecfg, $vmid, $device) = @_;
4436
4437     my $kvmver = get_running_qemu_version($vmid);
4438     my $io_uring = min_version($kvmver, 6, 0);
4439     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4440     $drive =~ s/\\/\\\\/g;
4441     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4442
4443     # If the command succeeds qemu prints: "OK"
4444     return 1 if $ret =~ m/OK/s;
4445
4446     die "adding drive failed: $ret\n";
4447 }
4448
4449 sub qemu_drivedel {
4450     my ($vmid, $deviceid) = @_;
4451
4452     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4453     $ret =~ s/^\s+//;
4454
4455     return 1 if $ret eq "";
4456
4457     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4458     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4459
4460     die "deleting drive $deviceid failed : $ret\n";
4461 }
4462
4463 sub qemu_deviceaddverify {
4464     my ($vmid, $deviceid) = @_;
4465
4466     for (my $i = 0; $i <= 5; $i++) {
4467          my $devices_list = vm_devices_list($vmid);
4468          return 1 if defined($devices_list->{$deviceid});
4469          sleep 1;
4470     }
4471
4472     die "error on hotplug device '$deviceid'\n";
4473 }
4474
4475
4476 sub qemu_devicedelverify {
4477     my ($vmid, $deviceid) = @_;
4478
4479     # need to verify that the device is correctly removed as device_del
4480     # is async and empty return is not reliable
4481
4482     for (my $i = 0; $i <= 5; $i++) {
4483          my $devices_list = vm_devices_list($vmid);
4484          return 1 if !defined($devices_list->{$deviceid});
4485          sleep 1;
4486     }
4487
4488     die "error on hot-unplugging device '$deviceid'\n";
4489 }
4490
4491 sub qemu_findorcreatescsihw {
4492     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4493
4494     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4495
4496     my $scsihwid="$controller_prefix$controller";
4497     my $devices_list = vm_devices_list($vmid);
4498
4499     if (!defined($devices_list->{$scsihwid})) {
4500         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4501     }
4502
4503     return 1;
4504 }
4505
4506 sub qemu_deletescsihw {
4507     my ($conf, $vmid, $opt) = @_;
4508
4509     my $device = parse_drive($opt, $conf->{$opt});
4510
4511     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4512         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4513         return 1;
4514     }
4515
4516     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4517
4518     my $devices_list = vm_devices_list($vmid);
4519     foreach my $opt (keys %{$devices_list}) {
4520         if (is_valid_drivename($opt)) {
4521             my $drive = parse_drive($opt, $conf->{$opt});
4522             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4523                 return 1;
4524             }
4525         }
4526     }
4527
4528     my $scsihwid="scsihw$controller";
4529
4530     vm_deviceunplug($vmid, $conf, $scsihwid);
4531
4532     return 1;
4533 }
4534
4535 sub qemu_add_pci_bridge {
4536     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4537
4538     my $bridges = {};
4539
4540     my $bridgeid;
4541
4542     print_pci_addr($device, $bridges, $arch, $machine_type);
4543
4544     while (my ($k, $v) = each %$bridges) {
4545         $bridgeid = $k;
4546     }
4547     return 1 if !defined($bridgeid) || $bridgeid < 1;
4548
4549     my $bridge = "pci.$bridgeid";
4550     my $devices_list = vm_devices_list($vmid);
4551
4552     if (!defined($devices_list->{$bridge})) {
4553         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4554     }
4555
4556     return 1;
4557 }
4558
4559 sub qemu_set_link_status {
4560     my ($vmid, $device, $up) = @_;
4561
4562     mon_cmd($vmid, "set_link", name => $device,
4563                up => $up ? JSON::true : JSON::false);
4564 }
4565
4566 sub qemu_netdevadd {
4567     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4568
4569     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4570     my %options =  split(/[=,]/, $netdev);
4571
4572     if (defined(my $vhost = $options{vhost})) {
4573         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4574     }
4575
4576     if (defined(my $queues = $options{queues})) {
4577         $options{queues} = $queues + 0;
4578     }
4579
4580     mon_cmd($vmid, "netdev_add",  %options);
4581     return 1;
4582 }
4583
4584 sub qemu_netdevdel {
4585     my ($vmid, $deviceid) = @_;
4586
4587     mon_cmd($vmid, "netdev_del", id => $deviceid);
4588 }
4589
4590 sub qemu_usb_hotplug {
4591     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4592
4593     return if !$device;
4594
4595     # remove the old one first
4596     vm_deviceunplug($vmid, $conf, $deviceid);
4597
4598     # check if xhci controller is necessary and available
4599     my $devicelist = vm_devices_list($vmid);
4600
4601     if (!$devicelist->{xhci}) {
4602         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4603         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4604     }
4605
4606     # add the new one
4607     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4608 }
4609
4610 sub qemu_cpu_hotplug {
4611     my ($vmid, $conf, $vcpus) = @_;
4612
4613     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4614
4615     my $sockets = 1;
4616     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4617     $sockets = $conf->{sockets} if  $conf->{sockets};
4618     my $cores = $conf->{cores} || 1;
4619     my $maxcpus = $sockets * $cores;
4620
4621     $vcpus = $maxcpus if !$vcpus;
4622
4623     die "you can't add more vcpus than maxcpus\n"
4624         if $vcpus > $maxcpus;
4625
4626     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4627
4628     if ($vcpus < $currentvcpus) {
4629
4630         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4631
4632             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4633                 qemu_devicedel($vmid, "cpu$i");
4634                 my $retry = 0;
4635                 my $currentrunningvcpus = undef;
4636                 while (1) {
4637                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4638                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4639                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4640                     $retry++;
4641                     sleep 1;
4642                 }
4643                 #update conf after each succesfull cpu unplug
4644                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4645                 PVE::QemuConfig->write_config($vmid, $conf);
4646             }
4647         } else {
4648             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4649         }
4650
4651         return;
4652     }
4653
4654     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4655     die "vcpus in running vm does not match its configuration\n"
4656         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4657
4658     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4659
4660         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4661             my $cpustr = print_cpu_device($conf, $i);
4662             qemu_deviceadd($vmid, $cpustr);
4663
4664             my $retry = 0;
4665             my $currentrunningvcpus = undef;
4666             while (1) {
4667                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4668                 last if scalar(@{$currentrunningvcpus}) == $i;
4669                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4670                 sleep 1;
4671                 $retry++;
4672             }
4673             #update conf after each succesfull cpu hotplug
4674             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4675             PVE::QemuConfig->write_config($vmid, $conf);
4676         }
4677     } else {
4678
4679         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4680             mon_cmd($vmid, "cpu-add", id => int($i));
4681         }
4682     }
4683 }
4684
4685 sub qemu_block_set_io_throttle {
4686     my ($vmid, $deviceid,
4687         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4688         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4689         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4690         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4691
4692     return if !check_running($vmid) ;
4693
4694     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4695         bps => int($bps),
4696         bps_rd => int($bps_rd),
4697         bps_wr => int($bps_wr),
4698         iops => int($iops),
4699         iops_rd => int($iops_rd),
4700         iops_wr => int($iops_wr),
4701         bps_max => int($bps_max),
4702         bps_rd_max => int($bps_rd_max),
4703         bps_wr_max => int($bps_wr_max),
4704         iops_max => int($iops_max),
4705         iops_rd_max => int($iops_rd_max),
4706         iops_wr_max => int($iops_wr_max),
4707         bps_max_length => int($bps_max_length),
4708         bps_rd_max_length => int($bps_rd_max_length),
4709         bps_wr_max_length => int($bps_wr_max_length),
4710         iops_max_length => int($iops_max_length),
4711         iops_rd_max_length => int($iops_rd_max_length),
4712         iops_wr_max_length => int($iops_wr_max_length),
4713     );
4714
4715 }
4716
4717 sub qemu_block_resize {
4718     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4719
4720     my $running = check_running($vmid);
4721
4722     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4723
4724     return if !$running;
4725
4726     my $padding = (1024 - $size % 1024) % 1024;
4727     $size = $size + $padding;
4728
4729     mon_cmd(
4730         $vmid,
4731         "block_resize",
4732         device => $deviceid,
4733         size => int($size),
4734         timeout => 60,
4735     );
4736 }
4737
4738 sub qemu_volume_snapshot {
4739     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4740
4741     my $running = check_running($vmid);
4742
4743     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4744         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4745     } else {
4746         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4747     }
4748 }
4749
4750 sub qemu_volume_snapshot_delete {
4751     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4752
4753     my $running = check_running($vmid);
4754
4755     if($running) {
4756
4757         $running = undef;
4758         my $conf = PVE::QemuConfig->load_config($vmid);
4759         PVE::QemuConfig->foreach_volume($conf, sub {
4760             my ($ds, $drive) = @_;
4761             $running = 1 if $drive->{file} eq $volid;
4762         });
4763     }
4764
4765     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4766         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4767     } else {
4768         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4769     }
4770 }
4771
4772 sub set_migration_caps {
4773     my ($vmid, $savevm) = @_;
4774
4775     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4776
4777     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4778     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4779
4780     my $cap_ref = [];
4781
4782     my $enabled_cap = {
4783         "auto-converge" => 1,
4784         "xbzrle" => 1,
4785         "x-rdma-pin-all" => 0,
4786         "zero-blocks" => 0,
4787         "compress" => 0,
4788         "dirty-bitmaps" => $dirty_bitmaps,
4789     };
4790
4791     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4792
4793     for my $supported_capability (@$supported_capabilities) {
4794         push @$cap_ref, {
4795             capability => $supported_capability->{capability},
4796             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4797         };
4798     }
4799
4800     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4801 }
4802
4803 sub foreach_volid {
4804     my ($conf, $func, @param) = @_;
4805
4806     my $volhash = {};
4807
4808     my $test_volid = sub {
4809         my ($key, $drive, $snapname) = @_;
4810
4811         my $volid = $drive->{file};
4812         return if !$volid;
4813
4814         $volhash->{$volid}->{cdrom} //= 1;
4815         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4816
4817         my $replicate = $drive->{replicate} // 1;
4818         $volhash->{$volid}->{replicate} //= 0;
4819         $volhash->{$volid}->{replicate} = 1 if $replicate;
4820
4821         $volhash->{$volid}->{shared} //= 0;
4822         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4823
4824         $volhash->{$volid}->{referenced_in_config} //= 0;
4825         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4826
4827         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4828             if defined($snapname);
4829
4830         my $size = $drive->{size};
4831         $volhash->{$volid}->{size} //= $size if $size;
4832
4833         $volhash->{$volid}->{is_vmstate} //= 0;
4834         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4835
4836         $volhash->{$volid}->{is_tpmstate} //= 0;
4837         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4838
4839         $volhash->{$volid}->{is_unused} //= 0;
4840         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4841
4842         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4843     };
4844
4845     my $include_opts = {
4846         extra_keys => ['vmstate'],
4847         include_unused => 1,
4848     };
4849
4850     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4851     foreach my $snapname (keys %{$conf->{snapshots}}) {
4852         my $snap = $conf->{snapshots}->{$snapname};
4853         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4854     }
4855
4856     foreach my $volid (keys %$volhash) {
4857         &$func($volid, $volhash->{$volid}, @param);
4858     }
4859 }
4860
4861 my $fast_plug_option = {
4862     'description' => 1,
4863     'hookscript' => 1,
4864     'lock' => 1,
4865     'migrate_downtime' => 1,
4866     'migrate_speed' => 1,
4867     'name' => 1,
4868     'onboot' => 1,
4869     'protection' => 1,
4870     'shares' => 1,
4871     'startup' => 1,
4872     'tags' => 1,
4873     'vmstatestorage' => 1,
4874 };
4875
4876 for my $opt (keys %$confdesc_cloudinit) {
4877     $fast_plug_option->{$opt} = 1;
4878 };
4879
4880 # hotplug changes in [PENDING]
4881 # $selection hash can be used to only apply specified options, for
4882 # example: { cores => 1 } (only apply changed 'cores')
4883 # $errors ref is used to return error messages
4884 sub vmconfig_hotplug_pending {
4885     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4886
4887     my $defaults = load_defaults();
4888     my $arch = get_vm_arch($conf);
4889     my $machine_type = get_vm_machine($conf, undef, $arch);
4890
4891     # commit values which do not have any impact on running VM first
4892     # Note: those option cannot raise errors, we we do not care about
4893     # $selection and always apply them.
4894
4895     my $add_error = sub {
4896         my ($opt, $msg) = @_;
4897         $errors->{$opt} = "hotplug problem - $msg";
4898     };
4899
4900     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4901
4902     my $cloudinit_record_changed = sub {
4903         my ($conf, $opt, $old, $new) = @_;
4904         return if !$cloudinit_pending_properties->{$opt};
4905
4906         my $ci = ($conf->{cloudinit} //= {});
4907
4908         my $recorded = $ci->{$opt};
4909         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4910
4911         if (defined($new)) {
4912             if (defined($old)) {
4913                 # an existing value is being modified
4914                 if (defined($recorded)) {
4915                     # the value was already not in sync
4916                     if ($new eq $recorded) {
4917                         # a value is being reverted to the cloud-init state:
4918                         delete $ci->{$opt};
4919                         delete $added{$opt};
4920                     } else {
4921                         # the value was changed multiple times, do nothing
4922                     }
4923                 } elsif ($added{$opt}) {
4924                     # the value had been marked as added and is being changed, do nothing
4925                 } else {
4926                     # the value is new, record it:
4927                     $ci->{$opt} = $old;
4928                 }
4929             } else {
4930                 # a new value is being added
4931                 if (defined($recorded)) {
4932                     # it was already not in sync
4933                     if ($new eq $recorded) {
4934                         # a value is being reverted to the cloud-init state:
4935                         delete $ci->{$opt};
4936                         delete $added{$opt};
4937                     } else {
4938                         # the value had temporarily been removed, do nothing
4939                     }
4940                 } elsif ($added{$opt}) {
4941                     # the value had been marked as added already, do nothing
4942                 } else {
4943                     # the value is new, add it
4944                     $added{$opt} = 1;
4945                 }
4946             }
4947         } elsif (!defined($old)) {
4948             # a non-existent value is being removed? ignore...
4949         } else {
4950             # a value is being deleted
4951             if (defined($recorded)) {
4952                 # a value was already recorded, just keep it
4953             } elsif ($added{$opt}) {
4954                 # the value was marked as added, remove it
4955                 delete $added{$opt};
4956             } else {
4957                 # a previously unrecorded value is being removed, record the old value:
4958                 $ci->{$opt} = $old;
4959             }
4960         }
4961
4962         my $added = join(',', sort keys %added);
4963         $ci->{added} = $added if length($added);
4964     };
4965
4966     my $changes = 0;
4967     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4968         if ($fast_plug_option->{$opt}) {
4969             my $new = delete $conf->{pending}->{$opt};
4970             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4971             $conf->{$opt} = $new;
4972             $changes = 1;
4973         }
4974     }
4975
4976     if ($changes) {
4977         PVE::QemuConfig->write_config($vmid, $conf);
4978     }
4979
4980     my $ostype = $conf->{ostype};
4981     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4982     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4983     my $usb_hotplug = $hotplug_features->{usb}
4984         && min_version($version, 7, 1)
4985         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4986
4987     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4988     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4989
4990     foreach my $opt (sort keys %$pending_delete_hash) {
4991         next if $selection && !$selection->{$opt};
4992         my $force = $pending_delete_hash->{$opt}->{force};
4993         eval {
4994             if ($opt eq 'hotplug') {
4995                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4996             } elsif ($opt eq 'tablet') {
4997                 die "skip\n" if !$hotplug_features->{usb};
4998                 if ($defaults->{tablet}) {
4999                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5000                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5001                         if $arch eq 'aarch64';
5002                 } else {
5003                     vm_deviceunplug($vmid, $conf, 'tablet');
5004                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5005                 }
5006             } elsif ($opt =~ m/^usb(\d+)$/) {
5007                 my $index = $1;
5008                 die "skip\n" if !$usb_hotplug;
5009                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5010                 vm_deviceunplug($vmid, $conf, $opt);
5011             } elsif ($opt eq 'vcpus') {
5012                 die "skip\n" if !$hotplug_features->{cpu};
5013                 qemu_cpu_hotplug($vmid, $conf, undef);
5014             } elsif ($opt eq 'balloon') {
5015                 # enable balloon device is not hotpluggable
5016                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5017                 # here we reset the ballooning value to memory
5018                 my $balloon = $conf->{memory} || $defaults->{memory};
5019                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5020             } elsif ($fast_plug_option->{$opt}) {
5021                 # do nothing
5022             } elsif ($opt =~ m/^net(\d+)$/) {
5023                 die "skip\n" if !$hotplug_features->{network};
5024                 vm_deviceunplug($vmid, $conf, $opt);
5025             } elsif (is_valid_drivename($opt)) {
5026                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5027                 vm_deviceunplug($vmid, $conf, $opt);
5028                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5029             } elsif ($opt =~ m/^memory$/) {
5030                 die "skip\n" if !$hotplug_features->{memory};
5031                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5032             } elsif ($opt eq 'cpuunits') {
5033                 $cgroup->change_cpu_shares(undef);
5034             } elsif ($opt eq 'cpulimit') {
5035                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5036             } else {
5037                 die "skip\n";
5038             }
5039         };
5040         if (my $err = $@) {
5041             &$add_error($opt, $err) if $err ne "skip\n";
5042         } else {
5043             my $old = delete $conf->{$opt};
5044             $cloudinit_record_changed->($conf, $opt, $old, undef);
5045             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5046         }
5047     }
5048
5049     my $cloudinit_opt;
5050     foreach my $opt (keys %{$conf->{pending}}) {
5051         next if $selection && !$selection->{$opt};
5052         my $value = $conf->{pending}->{$opt};
5053         eval {
5054             if ($opt eq 'hotplug') {
5055                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5056             } elsif ($opt eq 'tablet') {
5057                 die "skip\n" if !$hotplug_features->{usb};
5058                 if ($value == 1) {
5059                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5060                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5061                         if $arch eq 'aarch64';
5062                 } elsif ($value == 0) {
5063                     vm_deviceunplug($vmid, $conf, 'tablet');
5064                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5065                 }
5066             } elsif ($opt =~ m/^usb(\d+)$/) {
5067                 my $index = $1;
5068                 die "skip\n" if !$usb_hotplug;
5069                 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5070                 my $id = $opt;
5071                 if ($d->{host} =~ m/^spice$/i)  {
5072                     $id = "usbredirdev$index";
5073                 }
5074                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5075             } elsif ($opt eq 'vcpus') {
5076                 die "skip\n" if !$hotplug_features->{cpu};
5077                 qemu_cpu_hotplug($vmid, $conf, $value);
5078             } elsif ($opt eq 'balloon') {
5079                 # enable/disable balloning device is not hotpluggable
5080                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5081                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5082                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5083
5084                 # allow manual ballooning if shares is set to zero
5085                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5086                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5087                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5088                 }
5089             } elsif ($opt =~ m/^net(\d+)$/) {
5090                 # some changes can be done without hotplug
5091                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5092                                     $vmid, $opt, $value, $arch, $machine_type);
5093             } elsif (is_valid_drivename($opt)) {
5094                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5095                 # some changes can be done without hotplug
5096                 my $drive = parse_drive($opt, $value);
5097                 if (drive_is_cloudinit($drive)) {
5098                     $cloudinit_opt = [$opt, $drive];
5099                     # apply all the other changes first, then generate the cloudinit disk
5100                     die "skip\n";
5101                 }
5102                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5103                                      $vmid, $opt, $value, $arch, $machine_type);
5104             } elsif ($opt =~ m/^memory$/) { #dimms
5105                 die "skip\n" if !$hotplug_features->{memory};
5106                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5107             } elsif ($opt eq 'cpuunits') {
5108                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5109                 $cgroup->change_cpu_shares($new_cpuunits);
5110             } elsif ($opt eq 'cpulimit') {
5111                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5112                 $cgroup->change_cpu_quota($cpulimit, 100000);
5113             } elsif ($opt eq 'agent') {
5114                 vmconfig_update_agent($conf, $opt, $value);
5115             } else {
5116                 die "skip\n";  # skip non-hot-pluggable options
5117             }
5118         };
5119         if (my $err = $@) {
5120             &$add_error($opt, $err) if $err ne "skip\n";
5121         } else {
5122             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5123             $conf->{$opt} = $value;
5124             delete $conf->{pending}->{$opt};
5125         }
5126     }
5127
5128     if (defined($cloudinit_opt)) {
5129         my ($opt, $drive) = @$cloudinit_opt;
5130         my $value = $conf->{pending}->{$opt};
5131         eval {
5132             my $temp = {%$conf, $opt => $value};
5133             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5134             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5135                                  $vmid, $opt, $value, $arch, $machine_type);
5136         };
5137         if (my $err = $@) {
5138             &$add_error($opt, $err) if $err ne "skip\n";
5139         } else {
5140             $conf->{$opt} = $value;
5141             delete $conf->{pending}->{$opt};
5142         }
5143     }
5144
5145     # unplug xhci controller if no usb device is left
5146     if ($usb_hotplug) {
5147         my $has_usb = 0;
5148         for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5149             next if !defined($conf->{"usb$i"});
5150             $has_usb = 1;
5151             last;
5152         }
5153         if (!$has_usb) {
5154             vm_deviceunplug($vmid, $conf, 'xhci');
5155         }
5156     }
5157
5158     PVE::QemuConfig->write_config($vmid, $conf);
5159
5160     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5161         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5162     }
5163 }
5164
5165 sub try_deallocate_drive {
5166     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5167
5168     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5169         my $volid = $drive->{file};
5170         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5171             my $sid = PVE::Storage::parse_volume_id($volid);
5172             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5173
5174             # check if the disk is really unused
5175             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5176                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5177             PVE::Storage::vdisk_free($storecfg, $volid);
5178             return 1;
5179         } else {
5180             # If vm is not owner of this disk remove from config
5181             return 1;
5182         }
5183     }
5184
5185     return;
5186 }
5187
5188 sub vmconfig_delete_or_detach_drive {
5189     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5190
5191     my $drive = parse_drive($opt, $conf->{$opt});
5192
5193     my $rpcenv = PVE::RPCEnvironment::get();
5194     my $authuser = $rpcenv->get_user();
5195
5196     if ($force) {
5197         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5198         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5199     } else {
5200         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5201     }
5202 }
5203
5204
5205
5206 sub vmconfig_apply_pending {
5207     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5208
5209     return if !scalar(keys %{$conf->{pending}});
5210
5211     my $add_apply_error = sub {
5212         my ($opt, $msg) = @_;
5213         my $err_msg = "unable to apply pending change $opt : $msg";
5214         $errors->{$opt} = $err_msg;
5215         warn $err_msg;
5216     };
5217
5218     # cold plug
5219
5220     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5221     foreach my $opt (sort keys %$pending_delete_hash) {
5222         my $force = $pending_delete_hash->{$opt}->{force};
5223         eval {
5224             if ($opt =~ m/^unused/) {
5225                 die "internal error";
5226             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5227                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5228             }
5229         };
5230         if (my $err = $@) {
5231             $add_apply_error->($opt, $err);
5232         } else {
5233             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5234             delete $conf->{$opt};
5235         }
5236     }
5237
5238     PVE::QemuConfig->cleanup_pending($conf);
5239
5240     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5241
5242     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5243         next if $opt eq 'delete'; # just to be sure
5244         eval {
5245             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5246                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5247             }
5248         };
5249         if (my $err = $@) {
5250             $add_apply_error->($opt, $err);
5251         } else {
5252
5253             if (is_valid_drivename($opt)) {
5254                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5255                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5256             }
5257
5258             $conf->{$opt} = delete $conf->{pending}->{$opt};
5259         }
5260     }
5261
5262     # write all changes at once to avoid unnecessary i/o
5263     PVE::QemuConfig->write_config($vmid, $conf);
5264     if ($generate_cloudinit) {
5265         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5266             # After successful generation and if there were changes to be applied, update the
5267             # config to drop the {cloudinit} entry.
5268             PVE::QemuConfig->write_config($vmid, $conf);
5269         }
5270     }
5271 }
5272
5273 sub vmconfig_update_net {
5274     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5275
5276     my $newnet = parse_net($value);
5277
5278     if ($conf->{$opt}) {
5279         my $oldnet = parse_net($conf->{$opt});
5280
5281         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5282             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5283             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5284             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5285             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5286
5287             # for non online change, we try to hot-unplug
5288             die "skip\n" if !$hotplug;
5289             vm_deviceunplug($vmid, $conf, $opt);
5290         } else {
5291
5292             die "internal error" if $opt !~ m/net(\d+)/;
5293             my $iface = "tap${vmid}i$1";
5294
5295             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5296                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5297                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5298                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5299                 PVE::Network::tap_unplug($iface);
5300
5301                 if ($have_sdn) {
5302                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5303                 } else {
5304                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5305                 }
5306             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5307                 # Rate can be applied on its own but any change above needs to
5308                 # include the rate in tap_plug since OVS resets everything.
5309                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5310             }
5311
5312             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5313                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5314             }
5315
5316             return 1;
5317         }
5318     }
5319
5320     if ($hotplug) {
5321         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5322     } else {
5323         die "skip\n";
5324     }
5325 }
5326
5327 sub vmconfig_update_agent {
5328     my ($conf, $opt, $value) = @_;
5329
5330     die "skip\n" if !$conf->{$opt};
5331
5332     my $hotplug_options = { fstrim_cloned_disks => 1 };
5333
5334     my $old_agent = parse_guest_agent($conf);
5335     my $agent = parse_guest_agent({$opt => $value});
5336
5337     for my $option (keys %$agent) { # added/changed options
5338         next if defined($hotplug_options->{$option});
5339         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5340     }
5341
5342     for my $option (keys %$old_agent) { # removed options
5343         next if defined($hotplug_options->{$option});
5344         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5345     }
5346
5347     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5348 }
5349
5350 sub vmconfig_update_disk {
5351     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5352
5353     my $drive = parse_drive($opt, $value);
5354
5355     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5356         my $media = $drive->{media} || 'disk';
5357         my $oldmedia = $old_drive->{media} || 'disk';
5358         die "unable to change media type\n" if $media ne $oldmedia;
5359
5360         if (!drive_is_cdrom($old_drive)) {
5361
5362             if ($drive->{file} ne $old_drive->{file}) {
5363
5364                 die "skip\n" if !$hotplug;
5365
5366                 # unplug and register as unused
5367                 vm_deviceunplug($vmid, $conf, $opt);
5368                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5369
5370             } else {
5371                 # update existing disk
5372
5373                 # skip non hotpluggable value
5374                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5375                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5376                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5377                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5378                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5379                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5380                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5381                     die "skip\n";
5382                 }
5383
5384                 # apply throttle
5385                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5386                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5387                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5388                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5389                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5390                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5391                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5392                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5393                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5394                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5395                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5396                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5397                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5398                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5399                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5400                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5401                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5402                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5403
5404                     qemu_block_set_io_throttle(
5405                         $vmid,"drive-$opt",
5406                         ($drive->{mbps} || 0)*1024*1024,
5407                         ($drive->{mbps_rd} || 0)*1024*1024,
5408                         ($drive->{mbps_wr} || 0)*1024*1024,
5409                          $drive->{iops} || 0,
5410                          $drive->{iops_rd} || 0,
5411                          $drive->{iops_wr} || 0,
5412                         ($drive->{mbps_max} || 0)*1024*1024,
5413                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5414                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5415                          $drive->{iops_max} || 0,
5416                          $drive->{iops_rd_max} || 0,
5417                          $drive->{iops_wr_max} || 0,
5418                          $drive->{bps_max_length} || 1,
5419                          $drive->{bps_rd_max_length} || 1,
5420                          $drive->{bps_wr_max_length} || 1,
5421                          $drive->{iops_max_length} || 1,
5422                          $drive->{iops_rd_max_length} || 1,
5423                          $drive->{iops_wr_max_length} || 1,
5424                     );
5425
5426                 }
5427
5428                 return 1;
5429             }
5430
5431         } else { # cdrom
5432
5433             if ($drive->{file} eq 'none') {
5434                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5435                 if (drive_is_cloudinit($old_drive)) {
5436                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5437                 }
5438             } else {
5439                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5440
5441                 # force eject if locked
5442                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5443
5444                 if ($path) {
5445                     mon_cmd($vmid, "blockdev-change-medium",
5446                         id => "$opt", filename => "$path");
5447                 }
5448             }
5449
5450             return 1;
5451         }
5452     }
5453
5454     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5455     # hotplug new disks
5456     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5457     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5458 }
5459
5460 sub vmconfig_update_cloudinit_drive {
5461     my ($storecfg, $conf, $vmid) = @_;
5462
5463     my $cloudinit_ds = undef;
5464     my $cloudinit_drive = undef;
5465
5466     PVE::QemuConfig->foreach_volume($conf, sub {
5467         my ($ds, $drive) = @_;
5468         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5469             $cloudinit_ds = $ds;
5470             $cloudinit_drive = $drive;
5471         }
5472     });
5473
5474     return if !$cloudinit_drive;
5475
5476     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5477         PVE::QemuConfig->write_config($vmid, $conf);
5478     }
5479
5480     my $running = PVE::QemuServer::check_running($vmid);
5481
5482     if ($running) {
5483         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5484         if ($path) {
5485             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5486             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5487         }
5488     }
5489 }
5490
5491 # called in locked context by incoming migration
5492 sub vm_migrate_get_nbd_disks {
5493     my ($storecfg, $conf, $replicated_volumes) = @_;
5494
5495     my $local_volumes = {};
5496     PVE::QemuConfig->foreach_volume($conf, sub {
5497         my ($ds, $drive) = @_;
5498
5499         return if drive_is_cdrom($drive);
5500         return if $ds eq 'tpmstate0';
5501
5502         my $volid = $drive->{file};
5503
5504         return if !$volid;
5505
5506         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5507
5508         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5509         return if $scfg->{shared};
5510
5511         # replicated disks re-use existing state via bitmap
5512         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5513         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5514     });
5515     return $local_volumes;
5516 }
5517
5518 # called in locked context by incoming migration
5519 sub vm_migrate_alloc_nbd_disks {
5520     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5521
5522     my $nbd = {};
5523     foreach my $opt (sort keys %$source_volumes) {
5524         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5525
5526         if ($use_existing) {
5527             $nbd->{$opt}->{drivestr} = print_drive($drive);
5528             $nbd->{$opt}->{volid} = $volid;
5529             $nbd->{$opt}->{replicated} = 1;
5530             next;
5531         }
5532
5533         # storage mapping + volname = regular migration
5534         # storage mapping + format = remote migration
5535         # order of precedence, filtered by whether storage supports it:
5536         # 1. explicit requested format
5537         # 2. format of current volume
5538         # 3. default format of storage
5539         if (!$storagemap->{identity}) {
5540             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5541             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5542             if (!$format || !grep { $format eq $_ } @$validFormats) {
5543                 if ($volname) {
5544                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5545                     my $fileFormat = qemu_img_format($scfg, $volname);
5546                     $format = $fileFormat
5547                         if grep { $fileFormat eq $_ } @$validFormats;
5548                 }
5549                 $format //= $defFormat;
5550             }
5551         } else {
5552             # can't happen for remote migration, so $volname is always defined
5553             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5554             $format = qemu_img_format($scfg, $volname);
5555         }
5556
5557         my $size = $drive->{size} / 1024;
5558         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5559         my $newdrive = $drive;
5560         $newdrive->{format} = $format;
5561         $newdrive->{file} = $newvolid;
5562         my $drivestr = print_drive($newdrive);
5563         $nbd->{$opt}->{drivestr} = $drivestr;
5564         $nbd->{$opt}->{volid} = $newvolid;
5565     }
5566
5567     return $nbd;
5568 }
5569
5570 # see vm_start_nolock for parameters, additionally:
5571 # migrate_opts:
5572 #   storagemap = parsed storage map for allocating NBD disks
5573 sub vm_start {
5574     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5575
5576     return PVE::QemuConfig->lock_config($vmid, sub {
5577         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5578
5579         die "you can't start a vm if it's a template\n"
5580             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5581
5582         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5583         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5584
5585         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5586
5587         if ($has_backup_lock && $running) {
5588             # a backup is currently running, attempt to start the guest in the
5589             # existing QEMU instance
5590             return vm_resume($vmid);
5591         }
5592
5593         PVE::QemuConfig->check_lock($conf)
5594             if !($params->{skiplock} || $has_suspended_lock);
5595
5596         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5597
5598         die "VM $vmid already running\n" if $running;
5599
5600         if (my $storagemap = $migrate_opts->{storagemap}) {
5601             my $replicated = $migrate_opts->{replicated_volumes};
5602             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5603             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5604
5605             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5606                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5607             }
5608         }
5609
5610         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5611     });
5612 }
5613
5614
5615 # params:
5616 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5617 #   skiplock => 0/1, skip checking for config lock
5618 #   skiptemplate => 0/1, skip checking whether VM is template
5619 #   forcemachine => to force QEMU machine (rollback/migration)
5620 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5621 #   timeout => in seconds
5622 #   paused => start VM in paused state (backup)
5623 #   resume => resume from hibernation
5624 #   pbs-backing => {
5625 #      sata0 => {
5626 #         repository
5627 #         snapshot
5628 #         keyfile
5629 #         archive
5630 #      },
5631 #      virtio2 => ...
5632 #   }
5633 # migrate_opts:
5634 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5635 #   migratedfrom => source node
5636 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5637 #   network => CIDR of migration network
5638 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5639 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5640 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5641 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5642 #       contained in config
5643 sub vm_start_nolock {
5644     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5645
5646     my $statefile = $params->{statefile};
5647     my $resume = $params->{resume};
5648
5649     my $migratedfrom = $migrate_opts->{migratedfrom};
5650     my $migration_type = $migrate_opts->{type};
5651
5652     my $res = {};
5653
5654     # clean up leftover reboot request files
5655     eval { clear_reboot_request($vmid); };
5656     warn $@ if $@;
5657
5658     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5659         vmconfig_apply_pending($vmid, $conf, $storecfg);
5660         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5661     }
5662
5663     # don't regenerate the ISO if the VM is started as part of a live migration
5664     # this way we can reuse the old ISO with the correct config
5665     if (!$migratedfrom) {
5666         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5667             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5668             # $conf->{cloudinit}, so we could just not do this?
5669             # But we do it above, so for now let's be consistent.
5670             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5671         }
5672     }
5673
5674     # override offline migrated volumes, conf is out of date still
5675     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5676         for my $key (sort keys $offline_volumes->%*) {
5677             my $parsed = parse_drive($key, $conf->{$key});
5678             $parsed->{file} = $offline_volumes->{$key};
5679             $conf->{$key} = print_drive($parsed);
5680         }
5681     }
5682
5683     my $defaults = load_defaults();
5684
5685     # set environment variable useful inside network script
5686     # for remote migration the config is available on the target node!
5687     if (!$migrate_opts->{remote_node}) {
5688         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5689     }
5690
5691     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5692
5693     my $forcemachine = $params->{forcemachine};
5694     my $forcecpu = $params->{forcecpu};
5695     if ($resume) {
5696         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5697         $forcemachine = $conf->{runningmachine};
5698         $forcecpu = $conf->{runningcpu};
5699         print "Resuming suspended VM\n";
5700     }
5701
5702     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5703         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5704
5705     my $migration_ip;
5706     my $get_migration_ip = sub {
5707         my ($nodename) = @_;
5708
5709         return $migration_ip if defined($migration_ip);
5710
5711         my $cidr = $migrate_opts->{network};
5712
5713         if (!defined($cidr)) {
5714             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5715             $cidr = $dc_conf->{migration}->{network};
5716         }
5717
5718         if (defined($cidr)) {
5719             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5720
5721             die "could not get IP: no address configured on local " .
5722                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5723
5724             die "could not get IP: multiple addresses configured on local " .
5725                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5726
5727             $migration_ip = @$ips[0];
5728         }
5729
5730         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5731             if !defined($migration_ip);
5732
5733         return $migration_ip;
5734     };
5735
5736     if ($statefile) {
5737         if ($statefile eq 'tcp') {
5738             my $migrate = $res->{migrate} = { proto => 'tcp' };
5739             $migrate->{addr} = "localhost";
5740             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5741             my $nodename = nodename();
5742
5743             if (!defined($migration_type)) {
5744                 if (defined($datacenterconf->{migration}->{type})) {
5745                     $migration_type = $datacenterconf->{migration}->{type};
5746                 } else {
5747                     $migration_type = 'secure';
5748                 }
5749             }
5750
5751             if ($migration_type eq 'insecure') {
5752                 $migrate->{addr} = $get_migration_ip->($nodename);
5753                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5754             }
5755
5756             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5757             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5758             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5759             push @$cmd, '-incoming', $migrate->{uri};
5760             push @$cmd, '-S';
5761
5762         } elsif ($statefile eq 'unix') {
5763             # should be default for secure migrations as a ssh TCP forward
5764             # tunnel is not deterministic reliable ready and fails regurarly
5765             # to set up in time, so use UNIX socket forwards
5766             my $migrate = $res->{migrate} = { proto => 'unix' };
5767             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5768             unlink $migrate->{addr};
5769
5770             $migrate->{uri} = "unix:$migrate->{addr}";
5771             push @$cmd, '-incoming', $migrate->{uri};
5772             push @$cmd, '-S';
5773
5774         } elsif (-e $statefile) {
5775             push @$cmd, '-loadstate', $statefile;
5776         } else {
5777             my $statepath = PVE::Storage::path($storecfg, $statefile);
5778             push @$vollist, $statefile;
5779             push @$cmd, '-loadstate', $statepath;
5780         }
5781     } elsif ($params->{paused}) {
5782         push @$cmd, '-S';
5783     }
5784
5785     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5786
5787     my $pci_devices = {}; # host pci devices
5788     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5789         my $dev = $conf->{"hostpci$i"} or next;
5790         $pci_devices->{$i} = parse_hostpci($dev);
5791     }
5792
5793     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5794     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5795
5796     # map to a flat list of pci ids
5797     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5798
5799     # reserve all PCI IDs before actually doing anything with them
5800     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5801
5802     eval {
5803         my $uuid;
5804         for my $id (sort keys %$pci_devices) {
5805             my $d = $pci_devices->{$id};
5806             for my $dev ($d->{pciid}->@*) {
5807                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5808
5809                 # nvidia grid needs the qemu parameter '-uuid' set
5810                 # use smbios uuid or mdev uuid as fallback for that
5811                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5812                     if (defined($conf->{smbios1})) {
5813                         my $smbios_conf = parse_smbios1($conf->{smbios1});
5814                         $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5815                     }
5816                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id) if !defined($uuid);
5817                 }
5818             }
5819         }
5820         push @$cmd, '-uuid', $uuid if defined($uuid);
5821     };
5822     if (my $err = $@) {
5823         eval { cleanup_pci_devices($vmid, $conf) };
5824         warn $@ if $@;
5825         die $err;
5826     }
5827
5828     PVE::Storage::activate_volumes($storecfg, $vollist);
5829
5830     eval {
5831         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5832     };
5833     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5834     # timeout should be more than enough here...
5835     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5836
5837     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5838
5839     my %run_params = (
5840         timeout => $statefile ? undef : $start_timeout,
5841         umask => 0077,
5842         noerr => 1,
5843     );
5844
5845     # when migrating, prefix QEMU output so other side can pick up any
5846     # errors that might occur and show the user
5847     if ($migratedfrom) {
5848         $run_params{quiet} = 1;
5849         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5850     }
5851
5852     my %systemd_properties = (
5853         Slice => 'qemu.slice',
5854         KillMode => 'process',
5855         SendSIGKILL => 0,
5856         TimeoutStopUSec => ULONG_MAX, # infinity
5857     );
5858
5859     if (PVE::CGroup::cgroup_mode() == 2) {
5860         $systemd_properties{CPUWeight} = $cpuunits;
5861     } else {
5862         $systemd_properties{CPUShares} = $cpuunits;
5863     }
5864
5865     if (my $cpulimit = $conf->{cpulimit}) {
5866         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5867     }
5868     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5869
5870     my $run_qemu = sub {
5871         PVE::Tools::run_fork sub {
5872             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5873
5874             my $tpmpid;
5875             if (my $tpm = $conf->{tpmstate0}) {
5876                 # start the TPM emulator so QEMU can connect on start
5877                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5878             }
5879
5880             my $exitcode = run_command($cmd, %run_params);
5881             if ($exitcode) {
5882                 if ($tpmpid) {
5883                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5884                     kill 'TERM', $tpmpid;
5885                 }
5886                 die "QEMU exited with code $exitcode\n";
5887             }
5888         };
5889     };
5890
5891     if ($conf->{hugepages}) {
5892
5893         my $code = sub {
5894             my $hotplug_features =
5895                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5896             my $hugepages_topology =
5897                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5898
5899             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5900
5901             PVE::QemuServer::Memory::hugepages_mount();
5902             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5903
5904             eval { $run_qemu->() };
5905             if (my $err = $@) {
5906                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5907                     if !$conf->{keephugepages};
5908                 die $err;
5909             }
5910
5911             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5912                 if !$conf->{keephugepages};
5913         };
5914         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5915
5916     } else {
5917         eval { $run_qemu->() };
5918     }
5919
5920     if (my $err = $@) {
5921         # deactivate volumes if start fails
5922         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5923         warn $@ if $@;
5924         eval { cleanup_pci_devices($vmid, $conf) };
5925         warn $@ if $@;
5926
5927         die "start failed: $err";
5928     }
5929
5930     # re-reserve all PCI IDs now that we can know the actual VM PID
5931     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5932     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5933     warn $@ if $@;
5934
5935     if (defined($res->{migrate})) {
5936         print "migration listens on $res->{migrate}->{uri}\n";
5937     } elsif ($statefile) {
5938         eval { mon_cmd($vmid, "cont"); };
5939         warn $@ if $@;
5940     }
5941
5942     #start nbd server for storage migration
5943     if (my $nbd = $migrate_opts->{nbd}) {
5944         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5945
5946         my $migrate_storage_uri;
5947         # nbd_protocol_version > 0 for unix socket support
5948         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5949             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5950             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5951             $migrate_storage_uri = "nbd:unix:$socket_path";
5952             $res->{migrate}->{unix_sockets} = [$socket_path];
5953         } else {
5954             my $nodename = nodename();
5955             my $localip = $get_migration_ip->($nodename);
5956             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5957             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5958
5959             mon_cmd($vmid, "nbd-server-start", addr => {
5960                 type => 'inet',
5961                 data => {
5962                     host => "${localip}",
5963                     port => "${storage_migrate_port}",
5964                 },
5965             });
5966             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5967             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5968         }
5969
5970         my $block_info = mon_cmd($vmid, "query-block");
5971         $block_info = { map { $_->{device} => $_ } $block_info->@* };
5972
5973         foreach my $opt (sort keys %$nbd) {
5974             my $drivestr = $nbd->{$opt}->{drivestr};
5975             my $volid = $nbd->{$opt}->{volid};
5976
5977             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5978
5979             mon_cmd(
5980                 $vmid,
5981                 "block-export-add",
5982                 id => "drive-$opt",
5983                 'node-name' => $block_node,
5984                 writable => JSON::true,
5985                 type => "nbd",
5986                 name => "drive-$opt", # NBD export name
5987             );
5988
5989             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5990             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5991             print "re-using replicated volume: $opt - $volid\n"
5992                 if $nbd->{$opt}->{replicated};
5993
5994             $res->{drives}->{$opt} = $nbd->{$opt};
5995             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5996         }
5997     }
5998
5999     if ($migratedfrom) {
6000         eval {
6001             set_migration_caps($vmid);
6002         };
6003         warn $@ if $@;
6004
6005         if ($spice_port) {
6006             print "spice listens on port $spice_port\n";
6007             $res->{spice_port} = $spice_port;
6008             if ($migrate_opts->{spice_ticket}) {
6009                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6010                     $migrate_opts->{spice_ticket});
6011                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6012             }
6013         }
6014
6015     } else {
6016         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6017             if !$statefile && $conf->{balloon};
6018
6019         foreach my $opt (keys %$conf) {
6020             next if $opt !~  m/^net\d+$/;
6021             my $nicconf = parse_net($conf->{$opt});
6022             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6023         }
6024         add_nets_bridge_fdb($conf, $vmid);
6025     }
6026
6027    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6028         eval {
6029             mon_cmd(
6030                 $vmid,
6031                 'qom-set',
6032                 path => "machine/peripheral/balloon0",
6033                 property => "guest-stats-polling-interval",
6034                 value => 2
6035             );
6036         };
6037         log_warn("could not set polling interval for ballooning - $@") if $@;
6038     }
6039
6040     if ($resume) {
6041         print "Resumed VM, removing state\n";
6042         if (my $vmstate = $conf->{vmstate}) {
6043             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6044             PVE::Storage::vdisk_free($storecfg, $vmstate);
6045         }
6046         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6047         PVE::QemuConfig->write_config($vmid, $conf);
6048     }
6049
6050     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6051
6052     return $res;
6053 }
6054
6055 sub vm_commandline {
6056     my ($storecfg, $vmid, $snapname) = @_;
6057
6058     my $conf = PVE::QemuConfig->load_config($vmid);
6059
6060     my ($forcemachine, $forcecpu);
6061     if ($snapname) {
6062         my $snapshot = $conf->{snapshots}->{$snapname};
6063         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6064
6065         # check for machine or CPU overrides in snapshot
6066         $forcemachine = $snapshot->{runningmachine};
6067         $forcecpu = $snapshot->{runningcpu};
6068
6069         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6070
6071         $conf = $snapshot;
6072     }
6073
6074     my $defaults = load_defaults();
6075
6076     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6077
6078     return PVE::Tools::cmd2string($cmd);
6079 }
6080
6081 sub vm_reset {
6082     my ($vmid, $skiplock) = @_;
6083
6084     PVE::QemuConfig->lock_config($vmid, sub {
6085
6086         my $conf = PVE::QemuConfig->load_config($vmid);
6087
6088         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6089
6090         mon_cmd($vmid, "system_reset");
6091     });
6092 }
6093
6094 sub get_vm_volumes {
6095     my ($conf) = @_;
6096
6097     my $vollist = [];
6098     foreach_volid($conf, sub {
6099         my ($volid, $attr) = @_;
6100
6101         return if $volid =~ m|^/|;
6102
6103         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6104         return if !$sid;
6105
6106         push @$vollist, $volid;
6107     });
6108
6109     return $vollist;
6110 }
6111
6112 sub cleanup_pci_devices {
6113     my ($vmid, $conf) = @_;
6114
6115     foreach my $key (keys %$conf) {
6116         next if $key !~ m/^hostpci(\d+)$/;
6117         my $hostpciindex = $1;
6118         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6119         my $d = parse_hostpci($conf->{$key});
6120         if ($d->{mdev}) {
6121             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6122             # don't want to break ABI just for this two liner
6123             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6124
6125             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6126             # out when we do it first. so wait for 10 seconds and then try it
6127             my $pciid = $d->{pciid}->[0]->{id};
6128             my $info = PVE::SysFSTools::pci_device_info("$pciid");
6129             if ($info->{vendor} eq '10de') {
6130                 sleep 10;
6131             }
6132
6133             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6134         }
6135     }
6136     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6137 }
6138
6139 sub vm_stop_cleanup {
6140     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6141
6142     eval {
6143
6144         if (!$keepActive) {
6145             my $vollist = get_vm_volumes($conf);
6146             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6147
6148             if (my $tpmdrive = $conf->{tpmstate0}) {
6149                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6150                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6151                 if ($storeid) {
6152                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6153                 }
6154             }
6155         }
6156
6157         foreach my $ext (qw(mon qmp pid vnc qga)) {
6158             unlink "/var/run/qemu-server/${vmid}.$ext";
6159         }
6160
6161         if ($conf->{ivshmem}) {
6162             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6163             # just delete it for now, VMs which have this already open do not
6164             # are affected, but new VMs will get a separated one. If this
6165             # becomes an issue we either add some sort of ref-counting or just
6166             # add a "don't delete on stop" flag to the ivshmem format.
6167             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6168         }
6169
6170         cleanup_pci_devices($vmid, $conf);
6171
6172         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6173     };
6174     warn $@ if $@; # avoid errors - just warn
6175 }
6176
6177 # call only in locked context
6178 sub _do_vm_stop {
6179     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6180
6181     my $pid = check_running($vmid, $nocheck);
6182     return if !$pid;
6183
6184     my $conf;
6185     if (!$nocheck) {
6186         $conf = PVE::QemuConfig->load_config($vmid);
6187         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6188         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6189             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6190             $timeout = $opts->{down} if $opts->{down};
6191         }
6192         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6193     }
6194
6195     eval {
6196         if ($shutdown) {
6197             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6198                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6199             } else {
6200                 mon_cmd($vmid, "system_powerdown");
6201             }
6202         } else {
6203             mon_cmd($vmid, "quit");
6204         }
6205     };
6206     my $err = $@;
6207
6208     if (!$err) {
6209         $timeout = 60 if !defined($timeout);
6210
6211         my $count = 0;
6212         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6213             $count++;
6214             sleep 1;
6215         }
6216
6217         if ($count >= $timeout) {
6218             if ($force) {
6219                 warn "VM still running - terminating now with SIGTERM\n";
6220                 kill 15, $pid;
6221             } else {
6222                 die "VM quit/powerdown failed - got timeout\n";
6223             }
6224         } else {
6225             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6226             return;
6227         }
6228     } else {
6229        if (!check_running($vmid, $nocheck)) {
6230             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6231             return;
6232        }
6233        if ($force) {
6234             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6235             kill 15, $pid;
6236         } else {
6237             die "VM quit/powerdown failed\n";
6238         }
6239     }
6240
6241     # wait again
6242     $timeout = 10;
6243
6244     my $count = 0;
6245     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6246         $count++;
6247         sleep 1;
6248     }
6249
6250     if ($count >= $timeout) {
6251         warn "VM still running - terminating now with SIGKILL\n";
6252         kill 9, $pid;
6253         sleep 1;
6254     }
6255
6256     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6257 }
6258
6259 # Note: use $nocheck to skip tests if VM configuration file exists.
6260 # We need that when migration VMs to other nodes (files already moved)
6261 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6262 sub vm_stop {
6263     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6264
6265     $force = 1 if !defined($force) && !$shutdown;
6266
6267     if ($migratedfrom){
6268         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6269         kill 15, $pid if $pid;
6270         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6271         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6272         return;
6273     }
6274
6275     PVE::QemuConfig->lock_config($vmid, sub {
6276         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6277    });
6278 }
6279
6280 sub vm_reboot {
6281     my ($vmid, $timeout) = @_;
6282
6283     PVE::QemuConfig->lock_config($vmid, sub {
6284         eval {
6285
6286             # only reboot if running, as qmeventd starts it again on a stop event
6287             return if !check_running($vmid);
6288
6289             create_reboot_request($vmid);
6290
6291             my $storecfg = PVE::Storage::config();
6292             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6293
6294         };
6295         if (my $err = $@) {
6296             # avoid that the next normal shutdown will be confused for a reboot
6297             clear_reboot_request($vmid);
6298             die $err;
6299         }
6300    });
6301 }
6302
6303 # note: if using the statestorage parameter, the caller has to check privileges
6304 sub vm_suspend {
6305     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6306
6307     my $conf;
6308     my $path;
6309     my $storecfg;
6310     my $vmstate;
6311
6312     PVE::QemuConfig->lock_config($vmid, sub {
6313
6314         $conf = PVE::QemuConfig->load_config($vmid);
6315
6316         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6317         PVE::QemuConfig->check_lock($conf)
6318             if !($skiplock || $is_backing_up);
6319
6320         die "cannot suspend to disk during backup\n"
6321             if $is_backing_up && $includestate;
6322
6323         if ($includestate) {
6324             $conf->{lock} = 'suspending';
6325             my $date = strftime("%Y-%m-%d", localtime(time()));
6326             $storecfg = PVE::Storage::config();
6327             if (!$statestorage) {
6328                 $statestorage = find_vmstate_storage($conf, $storecfg);
6329                 # check permissions for the storage
6330                 my $rpcenv = PVE::RPCEnvironment::get();
6331                 if ($rpcenv->{type} ne 'cli') {
6332                     my $authuser = $rpcenv->get_user();
6333                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6334                 }
6335             }
6336
6337
6338             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6339                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6340             $path = PVE::Storage::path($storecfg, $vmstate);
6341             PVE::QemuConfig->write_config($vmid, $conf);
6342         } else {
6343             mon_cmd($vmid, "stop");
6344         }
6345     });
6346
6347     if ($includestate) {
6348         # save vm state
6349         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6350
6351         eval {
6352             set_migration_caps($vmid, 1);
6353             mon_cmd($vmid, "savevm-start", statefile => $path);
6354             for(;;) {
6355                 my $state = mon_cmd($vmid, "query-savevm");
6356                 if (!$state->{status}) {
6357                     die "savevm not active\n";
6358                 } elsif ($state->{status} eq 'active') {
6359                     sleep(1);
6360                     next;
6361                 } elsif ($state->{status} eq 'completed') {
6362                     print "State saved, quitting\n";
6363                     last;
6364                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6365                     die "query-savevm failed with error '$state->{error}'\n"
6366                 } else {
6367                     die "query-savevm returned status '$state->{status}'\n";
6368                 }
6369             }
6370         };
6371         my $err = $@;
6372
6373         PVE::QemuConfig->lock_config($vmid, sub {
6374             $conf = PVE::QemuConfig->load_config($vmid);
6375             if ($err) {
6376                 # cleanup, but leave suspending lock, to indicate something went wrong
6377                 eval {
6378                     mon_cmd($vmid, "savevm-end");
6379                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6380                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6381                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6382                     PVE::QemuConfig->write_config($vmid, $conf);
6383                 };
6384                 warn $@ if $@;
6385                 die $err;
6386             }
6387
6388             die "lock changed unexpectedly\n"
6389                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6390
6391             mon_cmd($vmid, "quit");
6392             $conf->{lock} = 'suspended';
6393             PVE::QemuConfig->write_config($vmid, $conf);
6394         });
6395     }
6396 }
6397
6398 # $nocheck is set when called as part of a migration - in this context the
6399 # location of the config file (source or target node) is not deterministic,
6400 # since migration cannot wait for pmxcfs to process the rename
6401 sub vm_resume {
6402     my ($vmid, $skiplock, $nocheck) = @_;
6403
6404     PVE::QemuConfig->lock_config($vmid, sub {
6405         my $res = mon_cmd($vmid, 'query-status');
6406         my $resume_cmd = 'cont';
6407         my $reset = 0;
6408         my $conf;
6409         if ($nocheck) {
6410             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6411             if ($@) {
6412                 my $vmlist = PVE::Cluster::get_vmlist();
6413                 if (exists($vmlist->{ids}->{$vmid})) {
6414                     my $node = $vmlist->{ids}->{$vmid}->{node};
6415                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6416                 }
6417                 if (!$conf) {
6418                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6419                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6420                 }
6421             }
6422         } else {
6423             $conf = PVE::QemuConfig->load_config($vmid);
6424         }
6425
6426         if ($res->{status}) {
6427             return if $res->{status} eq 'running'; # job done, go home
6428             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6429             $reset = 1 if $res->{status} eq 'shutdown';
6430         }
6431
6432         if (!$nocheck) {
6433             PVE::QemuConfig->check_lock($conf)
6434                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6435         }
6436
6437         if ($reset) {
6438             # required if a VM shuts down during a backup and we get a resume
6439             # request before the backup finishes for example
6440             mon_cmd($vmid, "system_reset");
6441         }
6442
6443         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6444
6445         mon_cmd($vmid, $resume_cmd);
6446     });
6447 }
6448
6449 sub vm_sendkey {
6450     my ($vmid, $skiplock, $key) = @_;
6451
6452     PVE::QemuConfig->lock_config($vmid, sub {
6453
6454         my $conf = PVE::QemuConfig->load_config($vmid);
6455
6456         # there is no qmp command, so we use the human monitor command
6457         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6458         die $res if $res ne '';
6459     });
6460 }
6461
6462 sub check_bridge_access {
6463     my ($rpcenv, $authuser, $conf) = @_;
6464
6465     return 1 if $authuser eq 'root@pam';
6466
6467     for my $opt (sort keys $conf->%*) {
6468         next if $opt !~ m/^net\d+$/;
6469         my $net = parse_net($conf->{$opt});
6470         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6471         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6472     }
6473     return 1;
6474 };
6475
6476 # vzdump restore implementaion
6477
6478 sub tar_archive_read_firstfile {
6479     my $archive = shift;
6480
6481     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6482
6483     # try to detect archive type first
6484     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6485         die "unable to open file '$archive'\n";
6486     my $firstfile = <$fh>;
6487     kill 15, $pid;
6488     close $fh;
6489
6490     die "ERROR: archive contaions no data\n" if !$firstfile;
6491     chomp $firstfile;
6492
6493     return $firstfile;
6494 }
6495
6496 sub tar_restore_cleanup {
6497     my ($storecfg, $statfile) = @_;
6498
6499     print STDERR "starting cleanup\n";
6500
6501     if (my $fd = IO::File->new($statfile, "r")) {
6502         while (defined(my $line = <$fd>)) {
6503             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6504                 my $volid = $2;
6505                 eval {
6506                     if ($volid =~ m|^/|) {
6507                         unlink $volid || die 'unlink failed\n';
6508                     } else {
6509                         PVE::Storage::vdisk_free($storecfg, $volid);
6510                     }
6511                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6512                 };
6513                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6514             } else {
6515                 print STDERR "unable to parse line in statfile - $line";
6516             }
6517         }
6518         $fd->close();
6519     }
6520 }
6521
6522 sub restore_file_archive {
6523     my ($archive, $vmid, $user, $opts) = @_;
6524
6525     return restore_vma_archive($archive, $vmid, $user, $opts)
6526         if $archive eq '-';
6527
6528     my $info = PVE::Storage::archive_info($archive);
6529     my $format = $opts->{format} // $info->{format};
6530     my $comp = $info->{compression};
6531
6532     # try to detect archive format
6533     if ($format eq 'tar') {
6534         return restore_tar_archive($archive, $vmid, $user, $opts);
6535     } else {
6536         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6537     }
6538 }
6539
6540 # hepler to remove disks that will not be used after restore
6541 my $restore_cleanup_oldconf = sub {
6542     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6543
6544     my $kept_disks = {};
6545
6546     PVE::QemuConfig->foreach_volume($oldconf, sub {
6547         my ($ds, $drive) = @_;
6548
6549         return if drive_is_cdrom($drive, 1);
6550
6551         my $volid = $drive->{file};
6552         return if !$volid || $volid =~ m|^/|;
6553
6554         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6555         return if !$path || !$owner || ($owner != $vmid);
6556
6557         # Note: only delete disk we want to restore
6558         # other volumes will become unused
6559         if ($virtdev_hash->{$ds}) {
6560             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6561             if (my $err = $@) {
6562                 warn $err;
6563             }
6564         } else {
6565             $kept_disks->{$volid} = 1;
6566         }
6567     });
6568
6569     # after the restore we have no snapshots anymore
6570     for my $snapname (keys $oldconf->{snapshots}->%*) {
6571         my $snap = $oldconf->{snapshots}->{$snapname};
6572         if ($snap->{vmstate}) {
6573             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6574             if (my $err = $@) {
6575                 warn $err;
6576             }
6577         }
6578
6579         for my $volid (keys $kept_disks->%*) {
6580             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6581             warn $@ if $@;
6582         }
6583     }
6584 };
6585
6586 # Helper to parse vzdump backup device hints
6587 #
6588 # $rpcenv: Environment, used to ckeck storage permissions
6589 # $user: User ID, to check storage permissions
6590 # $storecfg: Storage configuration
6591 # $fh: the file handle for reading the configuration
6592 # $devinfo: should contain device sizes for all backu-up'ed devices
6593 # $options: backup options (pool, default storage)
6594 #
6595 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6596 my $parse_backup_hints = sub {
6597     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6598
6599     my $check_storage = sub { # assert if an image can be allocate
6600         my ($storeid, $scfg) = @_;
6601         die "Content type 'images' is not available on storage '$storeid'\n"
6602             if !$scfg->{content}->{images};
6603         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6604             if $user ne 'root@pam';
6605     };
6606
6607     my $virtdev_hash = {};
6608     while (defined(my $line = <$fh>)) {
6609         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6610             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6611             die "archive does not contain data for drive '$virtdev'\n"
6612                 if !$devinfo->{$devname};
6613
6614             if (defined($options->{storage})) {
6615                 $storeid = $options->{storage} || 'local';
6616             } elsif (!$storeid) {
6617                 $storeid = 'local';
6618             }
6619             $format = 'raw' if !$format;
6620             $devinfo->{$devname}->{devname} = $devname;
6621             $devinfo->{$devname}->{virtdev} = $virtdev;
6622             $devinfo->{$devname}->{format} = $format;
6623             $devinfo->{$devname}->{storeid} = $storeid;
6624
6625             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6626             $check_storage->($storeid, $scfg); # permission and content type check
6627
6628             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6629         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6630             my $virtdev = $1;
6631             my $drive = parse_drive($virtdev, $2);
6632
6633             if (drive_is_cloudinit($drive)) {
6634                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6635                 $storeid = $options->{storage} if defined ($options->{storage});
6636                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6637                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6638
6639                 $check_storage->($storeid, $scfg); # permission and content type check
6640
6641                 $virtdev_hash->{$virtdev} = {
6642                     format => $format,
6643                     storeid => $storeid,
6644                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6645                     is_cloudinit => 1,
6646                 };
6647             }
6648         }
6649     }
6650
6651     return $virtdev_hash;
6652 };
6653
6654 # Helper to allocate and activate all volumes required for a restore
6655 #
6656 # $storecfg: Storage configuration
6657 # $virtdev_hash: as returned by parse_backup_hints()
6658 #
6659 # Returns: { $virtdev => $volid }
6660 my $restore_allocate_devices = sub {
6661     my ($storecfg, $virtdev_hash, $vmid) = @_;
6662
6663     my $map = {};
6664     foreach my $virtdev (sort keys %$virtdev_hash) {
6665         my $d = $virtdev_hash->{$virtdev};
6666         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6667         my $storeid = $d->{storeid};
6668         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6669
6670         # test if requested format is supported
6671         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6672         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6673         $d->{format} = $defFormat if !$supported;
6674
6675         my $name;
6676         if ($d->{is_cloudinit}) {
6677             $name = "vm-$vmid-cloudinit";
6678             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6679             if ($scfg->{path}) {
6680                 $name .= ".$d->{format}";
6681             }
6682         }
6683
6684         my $volid = PVE::Storage::vdisk_alloc(
6685             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6686
6687         print STDERR "new volume ID is '$volid'\n";
6688         $d->{volid} = $volid;
6689
6690         PVE::Storage::activate_volumes($storecfg, [$volid]);
6691
6692         $map->{$virtdev} = $volid;
6693     }
6694
6695     return $map;
6696 };
6697
6698 sub restore_update_config_line {
6699     my ($cookie, $map, $line, $unique) = @_;
6700
6701     return '' if $line =~ m/^\#qmdump\#/;
6702     return '' if $line =~ m/^\#vzdump\#/;
6703     return '' if $line =~ m/^lock:/;
6704     return '' if $line =~ m/^unused\d+:/;
6705     return '' if $line =~ m/^parent:/;
6706
6707     my $res = '';
6708
6709     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6710     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6711         # try to convert old 1.X settings
6712         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6713         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6714             my ($model, $macaddr) = split(/\=/, $devconfig);
6715             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6716             my $net = {
6717                 model => $model,
6718                 bridge => "vmbr$ind",
6719                 macaddr => $macaddr,
6720             };
6721             my $netstr = print_net($net);
6722
6723             $res .= "net$cookie->{netcount}: $netstr\n";
6724             $cookie->{netcount}++;
6725         }
6726     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6727         my ($id, $netstr) = ($1, $2);
6728         my $net = parse_net($netstr);
6729         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6730         $netstr = print_net($net);
6731         $res .= "$id: $netstr\n";
6732     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6733         my $virtdev = $1;
6734         my $value = $3;
6735         my $di = parse_drive($virtdev, $value);
6736         if (defined($di->{backup}) && !$di->{backup}) {
6737             $res .= "#$line";
6738         } elsif ($map->{$virtdev}) {
6739             delete $di->{format}; # format can change on restore
6740             $di->{file} = $map->{$virtdev};
6741             $value = print_drive($di);
6742             $res .= "$virtdev: $value\n";
6743         } else {
6744             $res .= $line;
6745         }
6746     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6747         my $vmgenid = $1;
6748         if ($vmgenid ne '0') {
6749             # always generate a new vmgenid if there was a valid one setup
6750             $vmgenid = generate_uuid();
6751         }
6752         $res .= "vmgenid: $vmgenid\n";
6753     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6754         my ($uuid, $uuid_str);
6755         UUID::generate($uuid);
6756         UUID::unparse($uuid, $uuid_str);
6757         my $smbios1 = parse_smbios1($2);
6758         $smbios1->{uuid} = $uuid_str;
6759         $res .= $1.print_smbios1($smbios1)."\n";
6760     } else {
6761         $res .= $line;
6762     }
6763
6764     return $res;
6765 }
6766
6767 my $restore_deactivate_volumes = sub {
6768     my ($storecfg, $virtdev_hash) = @_;
6769
6770     my $vollist = [];
6771     for my $dev (values $virtdev_hash->%*) {
6772         push $vollist->@*, $dev->{volid} if $dev->{volid};
6773     }
6774
6775     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6776     print STDERR $@ if $@;
6777 };
6778
6779 my $restore_destroy_volumes = sub {
6780     my ($storecfg, $virtdev_hash) = @_;
6781
6782     for my $dev (values $virtdev_hash->%*) {
6783         my $volid = $dev->{volid} or next;
6784         eval {
6785             PVE::Storage::vdisk_free($storecfg, $volid);
6786             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6787         };
6788         print STDERR "unable to cleanup '$volid' - $@" if $@;
6789     }
6790 };
6791
6792 my $restore_merge_config = sub {
6793     my ($filename, $backup_conf_raw, $override_conf) = @_;
6794
6795     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6796     for my $key (keys $override_conf->%*) {
6797         $backup_conf->{$key} = $override_conf->{$key};
6798     }
6799
6800     return $backup_conf;
6801 };
6802
6803 sub scan_volids {
6804     my ($cfg, $vmid) = @_;
6805
6806     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6807
6808     my $volid_hash = {};
6809     foreach my $storeid (keys %$info) {
6810         foreach my $item (@{$info->{$storeid}}) {
6811             next if !($item->{volid} && $item->{size});
6812             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6813             $volid_hash->{$item->{volid}} = $item;
6814         }
6815     }
6816
6817     return $volid_hash;
6818 }
6819
6820 sub update_disk_config {
6821     my ($vmid, $conf, $volid_hash) = @_;
6822
6823     my $changes;
6824     my $prefix = "VM $vmid";
6825
6826     # used and unused disks
6827     my $referenced = {};
6828
6829     # Note: it is allowed to define multiple storages with same path (alias), so
6830     # we need to check both 'volid' and real 'path' (two different volid can point
6831     # to the same path).
6832
6833     my $referencedpath = {};
6834
6835     # update size info
6836     PVE::QemuConfig->foreach_volume($conf, sub {
6837         my ($opt, $drive) = @_;
6838
6839         my $volid = $drive->{file};
6840         return if !$volid;
6841         my $volume = $volid_hash->{$volid};
6842
6843         # mark volid as "in-use" for next step
6844         $referenced->{$volid} = 1;
6845         if ($volume && (my $path = $volume->{path})) {
6846             $referencedpath->{$path} = 1;
6847         }
6848
6849         return if drive_is_cdrom($drive);
6850         return if !$volume;
6851
6852         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6853         if (defined($updated)) {
6854             $changes = 1;
6855             $conf->{$opt} = print_drive($updated);
6856             print "$prefix ($opt): $msg\n";
6857         }
6858     });
6859
6860     # remove 'unusedX' entry if volume is used
6861     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6862         my ($opt, $drive) = @_;
6863
6864         my $volid = $drive->{file};
6865         return if !$volid;
6866
6867         my $path;
6868         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6869         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6870             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6871             $changes = 1;
6872             delete $conf->{$opt};
6873         }
6874
6875         $referenced->{$volid} = 1;
6876         $referencedpath->{$path} = 1 if $path;
6877     });
6878
6879     foreach my $volid (sort keys %$volid_hash) {
6880         next if $volid =~ m/vm-$vmid-state-/;
6881         next if $referenced->{$volid};
6882         my $path = $volid_hash->{$volid}->{path};
6883         next if !$path; # just to be sure
6884         next if $referencedpath->{$path};
6885         $changes = 1;
6886         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6887         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6888         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6889     }
6890
6891     return $changes;
6892 }
6893
6894 sub rescan {
6895     my ($vmid, $nolock, $dryrun) = @_;
6896
6897     my $cfg = PVE::Storage::config();
6898
6899     print "rescan volumes...\n";
6900     my $volid_hash = scan_volids($cfg, $vmid);
6901
6902     my $updatefn =  sub {
6903         my ($vmid) = @_;
6904
6905         my $conf = PVE::QemuConfig->load_config($vmid);
6906
6907         PVE::QemuConfig->check_lock($conf);
6908
6909         my $vm_volids = {};
6910         foreach my $volid (keys %$volid_hash) {
6911             my $info = $volid_hash->{$volid};
6912             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6913         }
6914
6915         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6916
6917         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6918     };
6919
6920     if (defined($vmid)) {
6921         if ($nolock) {
6922             &$updatefn($vmid);
6923         } else {
6924             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6925         }
6926     } else {
6927         my $vmlist = config_list();
6928         foreach my $vmid (keys %$vmlist) {
6929             if ($nolock) {
6930                 &$updatefn($vmid);
6931             } else {
6932                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6933             }
6934         }
6935     }
6936 }
6937
6938 sub restore_proxmox_backup_archive {
6939     my ($archive, $vmid, $user, $options) = @_;
6940
6941     my $storecfg = PVE::Storage::config();
6942
6943     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6944     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6945
6946     my $fingerprint = $scfg->{fingerprint};
6947     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6948
6949     my $repo = PVE::PBSClient::get_repository($scfg);
6950     my $namespace = $scfg->{namespace};
6951
6952     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6953     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6954     local $ENV{PBS_PASSWORD} = $password;
6955     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6956
6957     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6958         PVE::Storage::parse_volname($storecfg, $archive);
6959
6960     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6961
6962     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6963
6964     my $tmpdir = "/var/tmp/vzdumptmp$$";
6965     rmtree $tmpdir;
6966     mkpath $tmpdir;
6967
6968     my $conffile = PVE::QemuConfig->config_file($vmid);
6969      # disable interrupts (always do cleanups)
6970     local $SIG{INT} =
6971         local $SIG{TERM} =
6972         local $SIG{QUIT} =
6973         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6974
6975     # Note: $oldconf is undef if VM does not exists
6976     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6977     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6978     my $new_conf_raw = '';
6979
6980     my $rpcenv = PVE::RPCEnvironment::get();
6981     my $devinfo = {}; # info about drives included in backup
6982     my $virtdev_hash = {}; # info about allocated drives
6983
6984     eval {
6985         # enable interrupts
6986         local $SIG{INT} =
6987             local $SIG{TERM} =
6988             local $SIG{QUIT} =
6989             local $SIG{HUP} =
6990             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6991
6992         my $cfgfn = "$tmpdir/qemu-server.conf";
6993         my $firewall_config_fn = "$tmpdir/fw.conf";
6994         my $index_fn = "$tmpdir/index.json";
6995
6996         my $cmd = "restore";
6997
6998         my $param = [$pbs_backup_name, "index.json", $index_fn];
6999         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7000         my $index = PVE::Tools::file_get_contents($index_fn);
7001         $index = decode_json($index);
7002
7003         foreach my $info (@{$index->{files}}) {
7004             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7005                 my $devname = $1;
7006                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7007                     $devinfo->{$devname}->{size} = $1;
7008                 } else {
7009                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7010                 }
7011             }
7012         }
7013
7014         my $is_qemu_server_backup = scalar(
7015             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7016         );
7017         if (!$is_qemu_server_backup) {
7018             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7019         }
7020         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7021
7022         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7023         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7024
7025         if ($has_firewall_config) {
7026             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7027             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7028
7029             my $pve_firewall_dir = '/etc/pve/firewall';
7030             mkdir $pve_firewall_dir; # make sure the dir exists
7031             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7032         }
7033
7034         my $fh = IO::File->new($cfgfn, "r") ||
7035             die "unable to read qemu-server.conf - $!\n";
7036
7037         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7038
7039         # fixme: rate limit?
7040
7041         # create empty/temp config
7042         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7043
7044         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7045
7046         # allocate volumes
7047         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7048
7049         foreach my $virtdev (sort keys %$virtdev_hash) {
7050             my $d = $virtdev_hash->{$virtdev};
7051             next if $d->{is_cloudinit}; # no need to restore cloudinit
7052
7053             # this fails if storage is unavailable
7054             my $volid = $d->{volid};
7055             my $path = PVE::Storage::path($storecfg, $volid);
7056
7057             # for live-restore we only want to preload the efidisk and TPM state
7058             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7059
7060             my @ns_arg;
7061             if (defined(my $ns = $scfg->{namespace})) {
7062                 @ns_arg = ('--ns', $ns);
7063             }
7064
7065             my $pbs_restore_cmd = [
7066                 '/usr/bin/pbs-restore',
7067                 '--repository', $repo,
7068                 @ns_arg,
7069                 $pbs_backup_name,
7070                 "$d->{devname}.img.fidx",
7071                 $path,
7072                 '--verbose',
7073                 ];
7074
7075             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7076             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7077
7078             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7079                 push @$pbs_restore_cmd, '--skip-zero';
7080             }
7081
7082             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7083             print "restore proxmox backup image: $dbg_cmdstring\n";
7084             run_command($pbs_restore_cmd);
7085         }
7086
7087         $fh->seek(0, 0) || die "seek failed - $!\n";
7088
7089         my $cookie = { netcount => 0 };
7090         while (defined(my $line = <$fh>)) {
7091             $new_conf_raw .= restore_update_config_line(
7092                 $cookie,
7093                 $map,
7094                 $line,
7095                 $options->{unique},
7096             );
7097         }
7098
7099         $fh->close();
7100     };
7101     my $err = $@;
7102
7103     if ($err || !$options->{live}) {
7104         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7105     }
7106
7107     rmtree $tmpdir;
7108
7109     if ($err) {
7110         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7111         die $err;
7112     }
7113
7114     if ($options->{live}) {
7115         # keep lock during live-restore
7116         $new_conf_raw .= "\nlock: create";
7117     }
7118
7119     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7120     check_bridge_access($rpcenv, $user, $new_conf);
7121     PVE::QemuConfig->write_config($vmid, $new_conf);
7122
7123     eval { rescan($vmid, 1); };
7124     warn $@ if $@;
7125
7126     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7127
7128     if ($options->{live}) {
7129         # enable interrupts
7130         local $SIG{INT} =
7131             local $SIG{TERM} =
7132             local $SIG{QUIT} =
7133             local $SIG{HUP} =
7134             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7135
7136         my $conf = PVE::QemuConfig->load_config($vmid);
7137         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7138
7139         # these special drives are already restored before start
7140         delete $devinfo->{'drive-efidisk0'};
7141         delete $devinfo->{'drive-tpmstate0-backup'};
7142
7143         my $pbs_opts = {
7144             repo => $repo,
7145             keyfile => $keyfile,
7146             snapshot => $pbs_backup_name,
7147             namespace => $namespace,
7148         };
7149         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7150
7151         PVE::QemuConfig->remove_lock($vmid, "create");
7152     }
7153 }
7154
7155 sub pbs_live_restore {
7156     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7157
7158     print "starting VM for live-restore\n";
7159     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7160
7161     my $pbs_backing = {};
7162     for my $ds (keys %$restored_disks) {
7163         $ds =~ m/^drive-(.*)$/;
7164         my $confname = $1;
7165         $pbs_backing->{$confname} = {
7166             repository => $opts->{repo},
7167             snapshot => $opts->{snapshot},
7168             archive => "$ds.img.fidx",
7169         };
7170         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7171         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7172
7173         my $drive = parse_drive($confname, $conf->{$confname});
7174         print "restoring '$ds' to '$drive->{file}'\n";
7175     }
7176
7177     my $drives_streamed = 0;
7178     eval {
7179         # make sure HA doesn't interrupt our restore by stopping the VM
7180         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7181             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7182         }
7183
7184         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7185         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7186         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7187
7188         my $qmeventd_fd = register_qmeventd_handle($vmid);
7189
7190         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7191         # this will effectively collapse the backing image chain consisting of
7192         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7193         # removes itself once all backing images vanish with 'auto-remove=on')
7194         my $jobs = {};
7195         for my $ds (sort keys %$restored_disks) {
7196             my $job_id = "restore-$ds";
7197             mon_cmd($vmid, 'block-stream',
7198                 'job-id' => $job_id,
7199                 device => "$ds",
7200             );
7201             $jobs->{$job_id} = {};
7202         }
7203
7204         mon_cmd($vmid, 'cont');
7205         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7206
7207         print "restore-drive jobs finished successfully, removing all tracking block devices"
7208             ." to disconnect from Proxmox Backup Server\n";
7209
7210         for my $ds (sort keys %$restored_disks) {
7211             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7212         }
7213
7214         close($qmeventd_fd);
7215     };
7216
7217     my $err = $@;
7218
7219     if ($err) {
7220         warn "An error occurred during live-restore: $err\n";
7221         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7222         die "live-restore failed\n";
7223     }
7224 }
7225
7226 sub restore_vma_archive {
7227     my ($archive, $vmid, $user, $opts, $comp) = @_;
7228
7229     my $readfrom = $archive;
7230
7231     my $cfg = PVE::Storage::config();
7232     my $commands = [];
7233     my $bwlimit = $opts->{bwlimit};
7234
7235     my $dbg_cmdstring = '';
7236     my $add_pipe = sub {
7237         my ($cmd) = @_;
7238         push @$commands, $cmd;
7239         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7240         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7241         $readfrom = '-';
7242     };
7243
7244     my $input = undef;
7245     if ($archive eq '-') {
7246         $input = '<&STDIN';
7247     } else {
7248         # If we use a backup from a PVE defined storage we also consider that
7249         # storage's rate limit:
7250         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7251         if (defined($volid)) {
7252             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7253             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7254             if ($readlimit) {
7255                 print STDERR "applying read rate limit: $readlimit\n";
7256                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7257                 $add_pipe->($cstream);
7258             }
7259         }
7260     }
7261
7262     if ($comp) {
7263         my $info = PVE::Storage::decompressor_info('vma', $comp);
7264         my $cmd = $info->{decompressor};
7265         push @$cmd, $readfrom;
7266         $add_pipe->($cmd);
7267     }
7268
7269     my $tmpdir = "/var/tmp/vzdumptmp$$";
7270     rmtree $tmpdir;
7271
7272     # disable interrupts (always do cleanups)
7273     local $SIG{INT} =
7274         local $SIG{TERM} =
7275         local $SIG{QUIT} =
7276         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7277
7278     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7279     POSIX::mkfifo($mapfifo, 0600);
7280     my $fifofh;
7281     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7282
7283     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7284
7285     my $oldtimeout;
7286     my $timeout = 5;
7287
7288     my $devinfo = {}; # info about drives included in backup
7289     my $virtdev_hash = {}; # info about allocated drives
7290
7291     my $rpcenv = PVE::RPCEnvironment::get();
7292
7293     my $conffile = PVE::QemuConfig->config_file($vmid);
7294
7295     # Note: $oldconf is undef if VM does not exist
7296     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7297     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7298     my $new_conf_raw = '';
7299
7300     my %storage_limits;
7301
7302     my $print_devmap = sub {
7303         my $cfgfn = "$tmpdir/qemu-server.conf";
7304
7305         # we can read the config - that is already extracted
7306         my $fh = IO::File->new($cfgfn, "r") ||
7307             die "unable to read qemu-server.conf - $!\n";
7308
7309         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7310         if (-f $fwcfgfn) {
7311             my $pve_firewall_dir = '/etc/pve/firewall';
7312             mkdir $pve_firewall_dir; # make sure the dir exists
7313             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7314         }
7315
7316         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7317
7318         foreach my $info (values %{$virtdev_hash}) {
7319             my $storeid = $info->{storeid};
7320             next if defined($storage_limits{$storeid});
7321
7322             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7323             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7324             $storage_limits{$storeid} = $limit * 1024;
7325         }
7326
7327         foreach my $devname (keys %$devinfo) {
7328             die "found no device mapping information for device '$devname'\n"
7329                 if !$devinfo->{$devname}->{virtdev};
7330         }
7331
7332         # create empty/temp config
7333         if ($oldconf) {
7334             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7335             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7336         }
7337
7338         # allocate volumes
7339         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7340
7341         # print restore information to $fifofh
7342         foreach my $virtdev (sort keys %$virtdev_hash) {
7343             my $d = $virtdev_hash->{$virtdev};
7344             next if $d->{is_cloudinit}; # no need to restore cloudinit
7345
7346             my $storeid = $d->{storeid};
7347             my $volid = $d->{volid};
7348
7349             my $map_opts = '';
7350             if (my $limit = $storage_limits{$storeid}) {
7351                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7352             }
7353
7354             my $write_zeros = 1;
7355             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7356                 $write_zeros = 0;
7357             }
7358
7359             my $path = PVE::Storage::path($cfg, $volid);
7360
7361             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7362
7363             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7364         }
7365
7366         $fh->seek(0, 0) || die "seek failed - $!\n";
7367
7368         my $cookie = { netcount => 0 };
7369         while (defined(my $line = <$fh>)) {
7370             $new_conf_raw .= restore_update_config_line(
7371                 $cookie,
7372                 $map,
7373                 $line,
7374                 $opts->{unique},
7375             );
7376         }
7377
7378         $fh->close();
7379     };
7380
7381     eval {
7382         # enable interrupts
7383         local $SIG{INT} =
7384             local $SIG{TERM} =
7385             local $SIG{QUIT} =
7386             local $SIG{HUP} =
7387             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7388         local $SIG{ALRM} = sub { die "got timeout\n"; };
7389
7390         $oldtimeout = alarm($timeout);
7391
7392         my $parser = sub {
7393             my $line = shift;
7394
7395             print "$line\n";
7396
7397             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7398                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7399                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7400             } elsif ($line =~ m/^CTIME: /) {
7401                 # we correctly received the vma config, so we can disable
7402                 # the timeout now for disk allocation (set to 10 minutes, so
7403                 # that we always timeout if something goes wrong)
7404                 alarm(600);
7405                 &$print_devmap();
7406                 print $fifofh "done\n";
7407                 my $tmp = $oldtimeout || 0;
7408                 $oldtimeout = undef;
7409                 alarm($tmp);
7410                 close($fifofh);
7411                 $fifofh = undef;
7412             }
7413         };
7414
7415         print "restore vma archive: $dbg_cmdstring\n";
7416         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7417     };
7418     my $err = $@;
7419
7420     alarm($oldtimeout) if $oldtimeout;
7421
7422     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7423
7424     close($fifofh) if $fifofh;
7425     unlink $mapfifo;
7426     rmtree $tmpdir;
7427
7428     if ($err) {
7429         $restore_destroy_volumes->($cfg, $virtdev_hash);
7430         die $err;
7431     }
7432
7433     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7434     check_bridge_access($rpcenv, $user, $new_conf);
7435     PVE::QemuConfig->write_config($vmid, $new_conf);
7436
7437     eval { rescan($vmid, 1); };
7438     warn $@ if $@;
7439
7440     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7441 }
7442
7443 sub restore_tar_archive {
7444     my ($archive, $vmid, $user, $opts) = @_;
7445
7446     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7447         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7448         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7449     }
7450
7451     if ($archive ne '-') {
7452         my $firstfile = tar_archive_read_firstfile($archive);
7453         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7454             if $firstfile ne 'qemu-server.conf';
7455     }
7456
7457     my $storecfg = PVE::Storage::config();
7458
7459     # avoid zombie disks when restoring over an existing VM -> cleanup first
7460     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7461     # skiplock=1 because qmrestore has set the 'create' lock itself already
7462     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7463     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7464
7465     my $tocmd = "/usr/lib/qemu-server/qmextract";
7466
7467     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7468     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7469     $tocmd .= ' --prealloc' if $opts->{prealloc};
7470     $tocmd .= ' --info' if $opts->{info};
7471
7472     # tar option "xf" does not autodetect compression when read from STDIN,
7473     # so we pipe to zcat
7474     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7475         PVE::Tools::shellquote("--to-command=$tocmd");
7476
7477     my $tmpdir = "/var/tmp/vzdumptmp$$";
7478     mkpath $tmpdir;
7479
7480     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7481     local $ENV{VZDUMP_VMID} = $vmid;
7482     local $ENV{VZDUMP_USER} = $user;
7483
7484     my $conffile = PVE::QemuConfig->config_file($vmid);
7485     my $new_conf_raw = '';
7486
7487     # disable interrupts (always do cleanups)
7488     local $SIG{INT} =
7489         local $SIG{TERM} =
7490         local $SIG{QUIT} =
7491         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7492
7493     eval {
7494         # enable interrupts
7495         local $SIG{INT} =
7496             local $SIG{TERM} =
7497             local $SIG{QUIT} =
7498             local $SIG{HUP} =
7499             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7500
7501         if ($archive eq '-') {
7502             print "extracting archive from STDIN\n";
7503             run_command($cmd, input => "<&STDIN");
7504         } else {
7505             print "extracting archive '$archive'\n";
7506             run_command($cmd);
7507         }
7508
7509         return if $opts->{info};
7510
7511         # read new mapping
7512         my $map = {};
7513         my $statfile = "$tmpdir/qmrestore.stat";
7514         if (my $fd = IO::File->new($statfile, "r")) {
7515             while (defined (my $line = <$fd>)) {
7516                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7517                     $map->{$1} = $2 if $1;
7518                 } else {
7519                     print STDERR "unable to parse line in statfile - $line\n";
7520                 }
7521             }
7522             $fd->close();
7523         }
7524
7525         my $confsrc = "$tmpdir/qemu-server.conf";
7526
7527         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7528
7529         my $cookie = { netcount => 0 };
7530         while (defined (my $line = <$srcfd>)) {
7531             $new_conf_raw .= restore_update_config_line(
7532                 $cookie,
7533                 $map,
7534                 $line,
7535                 $opts->{unique},
7536             );
7537         }
7538
7539         $srcfd->close();
7540     };
7541     if (my $err = $@) {
7542         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7543         die $err;
7544     }
7545
7546     rmtree $tmpdir;
7547
7548     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7549
7550     PVE::Cluster::cfs_update(); # make sure we read new file
7551
7552     eval { rescan($vmid, 1); };
7553     warn $@ if $@;
7554 };
7555
7556 sub foreach_storage_used_by_vm {
7557     my ($conf, $func) = @_;
7558
7559     my $sidhash = {};
7560
7561     PVE::QemuConfig->foreach_volume($conf, sub {
7562         my ($ds, $drive) = @_;
7563         return if drive_is_cdrom($drive);
7564
7565         my $volid = $drive->{file};
7566
7567         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7568         $sidhash->{$sid} = $sid if $sid;
7569     });
7570
7571     foreach my $sid (sort keys %$sidhash) {
7572         &$func($sid);
7573     }
7574 }
7575
7576 my $qemu_snap_storage = {
7577     rbd => 1,
7578 };
7579 sub do_snapshots_with_qemu {
7580     my ($storecfg, $volid, $deviceid) = @_;
7581
7582     return if $deviceid =~ m/tpmstate0/;
7583
7584     my $storage_name = PVE::Storage::parse_volume_id($volid);
7585     my $scfg = $storecfg->{ids}->{$storage_name};
7586     die "could not find storage '$storage_name'\n" if !defined($scfg);
7587
7588     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7589         return 1;
7590     }
7591
7592     if ($volid =~ m/\.(qcow2|qed)$/){
7593         return 1;
7594     }
7595
7596     return;
7597 }
7598
7599 sub qga_check_running {
7600     my ($vmid, $nowarn) = @_;
7601
7602     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7603     if ($@) {
7604         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7605         return 0;
7606     }
7607     return 1;
7608 }
7609
7610 sub template_create {
7611     my ($vmid, $conf, $disk) = @_;
7612
7613     my $storecfg = PVE::Storage::config();
7614
7615     PVE::QemuConfig->foreach_volume($conf, sub {
7616         my ($ds, $drive) = @_;
7617
7618         return if drive_is_cdrom($drive);
7619         return if $disk && $ds ne $disk;
7620
7621         my $volid = $drive->{file};
7622         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7623
7624         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7625         $drive->{file} = $voliddst;
7626         $conf->{$ds} = print_drive($drive);
7627         PVE::QemuConfig->write_config($vmid, $conf);
7628     });
7629 }
7630
7631 sub convert_iscsi_path {
7632     my ($path) = @_;
7633
7634     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7635         my $portal = $1;
7636         my $target = $2;
7637         my $lun = $3;
7638
7639         my $initiator_name = get_initiator_name();
7640
7641         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7642                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7643     }
7644
7645     die "cannot convert iscsi path '$path', unkown format\n";
7646 }
7647
7648 sub qemu_img_convert {
7649     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7650
7651     my $storecfg = PVE::Storage::config();
7652     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7653     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7654
7655     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7656
7657     my $cachemode;
7658     my $src_path;
7659     my $src_is_iscsi = 0;
7660     my $src_format;
7661
7662     if ($src_storeid) {
7663         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7664         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7665         $src_format = qemu_img_format($src_scfg, $src_volname);
7666         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7667         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7668         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7669     } elsif (-f $src_volid || -b $src_volid) {
7670         $src_path = $src_volid;
7671         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7672             $src_format = $1;
7673         }
7674     }
7675
7676     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7677
7678     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7679     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7680     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7681     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7682
7683     my $cmd = [];
7684     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7685     push @$cmd, '-l', "snapshot.name=$snapname"
7686         if $snapname && $src_format && $src_format eq "qcow2";
7687     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7688     push @$cmd, '-T', $cachemode if defined($cachemode);
7689     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7690
7691     if ($src_is_iscsi) {
7692         push @$cmd, '--image-opts';
7693         $src_path = convert_iscsi_path($src_path);
7694     } elsif ($src_format) {
7695         push @$cmd, '-f', $src_format;
7696     }
7697
7698     if ($dst_is_iscsi) {
7699         push @$cmd, '--target-image-opts';
7700         $dst_path = convert_iscsi_path($dst_path);
7701     } else {
7702         push @$cmd, '-O', $dst_format;
7703     }
7704
7705     push @$cmd, $src_path;
7706
7707     if (!$dst_is_iscsi && $is_zero_initialized) {
7708         push @$cmd, "zeroinit:$dst_path";
7709     } else {
7710         push @$cmd, $dst_path;
7711     }
7712
7713     my $parser = sub {
7714         my $line = shift;
7715         if($line =~ m/\((\S+)\/100\%\)/){
7716             my $percent = $1;
7717             my $transferred = int($size * $percent / 100);
7718             my $total_h = render_bytes($size, 1);
7719             my $transferred_h = render_bytes($transferred, 1);
7720
7721             print "transferred $transferred_h of $total_h ($percent%)\n";
7722         }
7723
7724     };
7725
7726     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7727     my $err = $@;
7728     die "copy failed: $err" if $err;
7729 }
7730
7731 sub qemu_img_format {
7732     my ($scfg, $volname) = @_;
7733
7734     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7735         return $1;
7736     } else {
7737         return "raw";
7738     }
7739 }
7740
7741 sub qemu_drive_mirror {
7742     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7743
7744     $jobs = {} if !$jobs;
7745
7746     my $qemu_target;
7747     my $format;
7748     $jobs->{"drive-$drive"} = {};
7749
7750     if ($dst_volid =~ /^nbd:/) {
7751         $qemu_target = $dst_volid;
7752         $format = "nbd";
7753     } else {
7754         my $storecfg = PVE::Storage::config();
7755         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7756
7757         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7758
7759         $format = qemu_img_format($dst_scfg, $dst_volname);
7760
7761         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7762
7763         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7764     }
7765
7766     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7767     $opts->{format} = $format if $format;
7768
7769     if (defined($src_bitmap)) {
7770         $opts->{sync} = 'incremental';
7771         $opts->{bitmap} = $src_bitmap;
7772         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7773     }
7774
7775     if (defined($bwlimit)) {
7776         $opts->{speed} = $bwlimit * 1024;
7777         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7778     } else {
7779         print "drive mirror is starting for drive-$drive\n";
7780     }
7781
7782     # if a job already runs for this device we get an error, catch it for cleanup
7783     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7784     if (my $err = $@) {
7785         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7786         warn "$@\n" if $@;
7787         die "mirroring error: $err\n";
7788     }
7789
7790     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7791 }
7792
7793 # $completion can be either
7794 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7795 # 'cancel': wait until all jobs are ready, block-job-cancel them
7796 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7797 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7798 sub qemu_drive_mirror_monitor {
7799     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7800
7801     $completion //= 'complete';
7802     $op //= "mirror";
7803
7804     eval {
7805         my $err_complete = 0;
7806
7807         my $starttime = time ();
7808         while (1) {
7809             die "block job ('$op') timed out\n" if $err_complete > 300;
7810
7811             my $stats = mon_cmd($vmid, "query-block-jobs");
7812             my $ctime = time();
7813
7814             my $running_jobs = {};
7815             for my $stat (@$stats) {
7816                 next if $stat->{type} ne $op;
7817                 $running_jobs->{$stat->{device}} = $stat;
7818             }
7819
7820             my $readycounter = 0;
7821
7822             for my $job_id (sort keys %$jobs) {
7823                 my $job = $running_jobs->{$job_id};
7824
7825                 my $vanished = !defined($job);
7826                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7827                 if($complete || ($vanished && $completion eq 'auto')) {
7828                     print "$job_id: $op-job finished\n";
7829                     delete $jobs->{$job_id};
7830                     next;
7831                 }
7832
7833                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7834
7835                 my $busy = $job->{busy};
7836                 my $ready = $job->{ready};
7837                 if (my $total = $job->{len}) {
7838                     my $transferred = $job->{offset} || 0;
7839                     my $remaining = $total - $transferred;
7840                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7841
7842                     my $duration = $ctime - $starttime;
7843                     my $total_h = render_bytes($total, 1);
7844                     my $transferred_h = render_bytes($transferred, 1);
7845
7846                     my $status = sprintf(
7847                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7848                         render_duration($duration),
7849                     );
7850
7851                     if ($ready) {
7852                         if ($busy) {
7853                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7854                         } else {
7855                             $status .= ", ready";
7856                         }
7857                     }
7858                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7859                     $jobs->{$job_id}->{ready} = $ready;
7860                 }
7861
7862                 $readycounter++ if $job->{ready};
7863             }
7864
7865             last if scalar(keys %$jobs) == 0;
7866
7867             if ($readycounter == scalar(keys %$jobs)) {
7868                 print "all '$op' jobs are ready\n";
7869
7870                 # do the complete later (or has already been done)
7871                 last if $completion eq 'skip' || $completion eq 'auto';
7872
7873                 if ($vmiddst && $vmiddst != $vmid) {
7874                     my $agent_running = $qga && qga_check_running($vmid);
7875                     if ($agent_running) {
7876                         print "freeze filesystem\n";
7877                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7878                         warn $@ if $@;
7879                     } else {
7880                         print "suspend vm\n";
7881                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7882                         warn $@ if $@;
7883                     }
7884
7885                     # if we clone a disk for a new target vm, we don't switch the disk
7886                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7887
7888                     if ($agent_running) {
7889                         print "unfreeze filesystem\n";
7890                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7891                         warn $@ if $@;
7892                     } else {
7893                         print "resume vm\n";
7894                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7895                         warn $@ if $@;
7896                     }
7897
7898                     last;
7899                 } else {
7900
7901                     for my $job_id (sort keys %$jobs) {
7902                         # try to switch the disk if source and destination are on the same guest
7903                         print "$job_id: Completing block job_id...\n";
7904
7905                         my $op;
7906                         if ($completion eq 'complete') {
7907                             $op = 'block-job-complete';
7908                         } elsif ($completion eq 'cancel') {
7909                             $op = 'block-job-cancel';
7910                         } else {
7911                             die "invalid completion value: $completion\n";
7912                         }
7913                         eval { mon_cmd($vmid, $op, device => $job_id) };
7914                         if ($@ =~ m/cannot be completed/) {
7915                             print "$job_id: block job cannot be completed, trying again.\n";
7916                             $err_complete++;
7917                         }else {
7918                             print "$job_id: Completed successfully.\n";
7919                             $jobs->{$job_id}->{complete} = 1;
7920                         }
7921                     }
7922                 }
7923             }
7924             sleep 1;
7925         }
7926     };
7927     my $err = $@;
7928
7929     if ($err) {
7930         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7931         die "block job ($op) error: $err";
7932     }
7933 }
7934
7935 sub qemu_blockjobs_cancel {
7936     my ($vmid, $jobs) = @_;
7937
7938     foreach my $job (keys %$jobs) {
7939         print "$job: Cancelling block job\n";
7940         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7941         $jobs->{$job}->{cancel} = 1;
7942     }
7943
7944     while (1) {
7945         my $stats = mon_cmd($vmid, "query-block-jobs");
7946
7947         my $running_jobs = {};
7948         foreach my $stat (@$stats) {
7949             $running_jobs->{$stat->{device}} = $stat;
7950         }
7951
7952         foreach my $job (keys %$jobs) {
7953
7954             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7955                 print "$job: Done.\n";
7956                 delete $jobs->{$job};
7957             }
7958         }
7959
7960         last if scalar(keys %$jobs) == 0;
7961
7962         sleep 1;
7963     }
7964 }
7965
7966 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7967 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7968 my sub clone_disk_check_io_uring {
7969     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7970
7971     return if !$use_drive_mirror;
7972
7973     # Don't complain when not changing storage.
7974     # Assume if it works for the source, it'll work for the target too.
7975     return if $src_storeid eq $dst_storeid;
7976
7977     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7978     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7979
7980     my $cache_direct = drive_uses_cache_direct($src_drive);
7981
7982     my $src_uses_io_uring;
7983     if ($src_drive->{aio}) {
7984         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
7985     } else {
7986         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
7987     }
7988
7989     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
7990         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
7991 }
7992
7993 sub clone_disk {
7994     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7995
7996     my ($vmid, $running) = $source->@{qw(vmid running)};
7997     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7998
7999     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8000     my ($storage, $format) = $dest->@{qw(storage format)};
8001
8002     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8003
8004     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8005         die "cloning from/to EFI disk requires EFI disk\n"
8006             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8007         die "cloning from/to TPM state requires TPM state\n"
8008             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8009
8010         # This would lead to two device nodes in QEMU pointing to the same backing image!
8011         die "cannot change drive name when cloning disk from/to the same VM\n"
8012             if $use_drive_mirror && $vmid == $newvmid;
8013     }
8014
8015     die "cannot move TPM state while VM is running\n"
8016         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8017
8018     my $newvolid;
8019
8020     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8021     print "$src_drivename " if $src_drivename;
8022     print "($drive->{file})\n";
8023
8024     if (!$full) {
8025         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8026         push @$newvollist, $newvolid;
8027     } else {
8028         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8029         my $storeid = $storage || $src_storeid;
8030
8031         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8032
8033         my $name = undef;
8034         my $size = undef;
8035         if (drive_is_cloudinit($drive)) {
8036             $name = "vm-$newvmid-cloudinit";
8037             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8038             if ($scfg->{path}) {
8039                 $name .= ".$dst_format";
8040             }
8041             $snapname = undef;
8042             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8043         } elsif ($dst_drivename eq 'efidisk0') {
8044             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8045         } elsif ($dst_drivename eq 'tpmstate0') {
8046             $dst_format = 'raw';
8047             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8048         } else {
8049             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8050
8051             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8052         }
8053         $newvolid = PVE::Storage::vdisk_alloc(
8054             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8055         );
8056         push @$newvollist, $newvolid;
8057
8058         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8059
8060         if (drive_is_cloudinit($drive)) {
8061             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8062             # if this is the case, we have to complete any block-jobs still there from
8063             # previous drive-mirrors
8064             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8065                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8066             }
8067             goto no_data_clone;
8068         }
8069
8070         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8071         if ($use_drive_mirror) {
8072             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8073                 $completion, $qga, $bwlimit);
8074         } else {
8075             if ($dst_drivename eq 'efidisk0') {
8076                 # the relevant data on the efidisk may be smaller than the source
8077                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8078                 # that is given by the OVMF_VARS.fd
8079                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8080                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8081
8082                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8083
8084                 # better for Ceph if block size is not too small, see bug #3324
8085                 my $bs = 1024*1024;
8086
8087                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8088
8089                 if ($src_format eq 'qcow2' && $snapname) {
8090                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8091                         if !min_version(kvm_user_version(), 6, 2);
8092                     push $cmd->@*, '-l', $snapname;
8093                 }
8094                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8095                 run_command($cmd);
8096             } else {
8097                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8098             }
8099         }
8100     }
8101
8102 no_data_clone:
8103     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8104
8105     my $disk = dclone($drive);
8106     delete $disk->{format};
8107     $disk->{file} = $newvolid;
8108     $disk->{size} = $size if defined($size);
8109
8110     return $disk;
8111 }
8112
8113 sub get_running_qemu_version {
8114     my ($vmid) = @_;
8115     my $res = mon_cmd($vmid, "query-version");
8116     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8117 }
8118
8119 sub qemu_use_old_bios_files {
8120     my ($machine_type) = @_;
8121
8122     return if !$machine_type;
8123
8124     my $use_old_bios_files = undef;
8125
8126     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8127         $machine_type = $1;
8128         $use_old_bios_files = 1;
8129     } else {
8130         my $version = extract_version($machine_type, kvm_user_version());
8131         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8132         # load new efi bios files on migration. So this hack is required to allow
8133         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8134         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8135         $use_old_bios_files = !min_version($version, 2, 4);
8136     }
8137
8138     return ($use_old_bios_files, $machine_type);
8139 }
8140
8141 sub get_efivars_size {
8142     my ($conf, $efidisk) = @_;
8143
8144     my $arch = get_vm_arch($conf);
8145     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8146     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8147     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8148     return -s $ovmf_vars;
8149 }
8150
8151 sub update_efidisk_size {
8152     my ($conf) = @_;
8153
8154     return if !defined($conf->{efidisk0});
8155
8156     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8157     $disk->{size} = get_efivars_size($conf);
8158     $conf->{efidisk0} = print_drive($disk);
8159
8160     return;
8161 }
8162
8163 sub update_tpmstate_size {
8164     my ($conf) = @_;
8165
8166     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8167     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8168     $conf->{tpmstate0} = print_drive($disk);
8169 }
8170
8171 sub create_efidisk($$$$$$$) {
8172     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8173
8174     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8175
8176     my $vars_size_b = -s $ovmf_vars;
8177     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8178     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8179     PVE::Storage::activate_volumes($storecfg, [$volid]);
8180
8181     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8182     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8183
8184     return ($volid, $size/1024);
8185 }
8186
8187 sub vm_iothreads_list {
8188     my ($vmid) = @_;
8189
8190     my $res = mon_cmd($vmid, 'query-iothreads');
8191
8192     my $iothreads = {};
8193     foreach my $iothread (@$res) {
8194         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8195     }
8196
8197     return $iothreads;
8198 }
8199
8200 sub scsihw_infos {
8201     my ($conf, $drive) = @_;
8202
8203     my $maxdev = 0;
8204
8205     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8206         $maxdev = 7;
8207     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8208         $maxdev = 1;
8209     } else {
8210         $maxdev = 256;
8211     }
8212
8213     my $controller = int($drive->{index} / $maxdev);
8214     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8215         ? "virtioscsi"
8216         : "scsihw";
8217
8218     return ($maxdev, $controller, $controller_prefix);
8219 }
8220
8221 sub resolve_dst_disk_format {
8222         my ($storecfg, $storeid, $src_volname, $format) = @_;
8223         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8224
8225         if (!$format) {
8226             # if no target format is specified, use the source disk format as hint
8227             if ($src_volname) {
8228                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8229                 $format = qemu_img_format($scfg, $src_volname);
8230             } else {
8231                 return $defFormat;
8232             }
8233         }
8234
8235         # test if requested format is supported - else use default
8236         my $supported = grep { $_ eq $format } @$validFormats;
8237         $format = $defFormat if !$supported;
8238         return $format;
8239 }
8240
8241 # NOTE: if this logic changes, please update docs & possibly gui logic
8242 sub find_vmstate_storage {
8243     my ($conf, $storecfg) = @_;
8244
8245     # first, return storage from conf if set
8246     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8247
8248     my ($target, $shared, $local);
8249
8250     foreach_storage_used_by_vm($conf, sub {
8251         my ($sid) = @_;
8252         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8253         my $dst = $scfg->{shared} ? \$shared : \$local;
8254         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8255     });
8256
8257     # second, use shared storage where VM has at least one disk
8258     # third, use local storage where VM has at least one disk
8259     # fall back to local storage
8260     $target = $shared // $local // 'local';
8261
8262     return $target;
8263 }
8264
8265 sub generate_uuid {
8266     my ($uuid, $uuid_str);
8267     UUID::generate($uuid);
8268     UUID::unparse($uuid, $uuid_str);
8269     return $uuid_str;
8270 }
8271
8272 sub generate_smbios1_uuid {
8273     return "uuid=".generate_uuid();
8274 }
8275
8276 sub nbd_stop {
8277     my ($vmid) = @_;
8278
8279     mon_cmd($vmid, 'nbd-server-stop');
8280 }
8281
8282 sub create_reboot_request {
8283     my ($vmid) = @_;
8284     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8285         or die "failed to create reboot trigger file: $!\n";
8286     close($fh);
8287 }
8288
8289 sub clear_reboot_request {
8290     my ($vmid) = @_;
8291     my $path = "/run/qemu-server/$vmid.reboot";
8292     my $res = 0;
8293
8294     $res = unlink($path);
8295     die "could not remove reboot request for $vmid: $!"
8296         if !$res && $! != POSIX::ENOENT;
8297
8298     return $res;
8299 }
8300
8301 sub bootorder_from_legacy {
8302     my ($conf, $bootcfg) = @_;
8303
8304     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8305     my $bootindex_hash = {};
8306     my $i = 1;
8307     foreach my $o (split(//, $boot)) {
8308         $bootindex_hash->{$o} = $i*100;
8309         $i++;
8310     }
8311
8312     my $bootorder = {};
8313
8314     PVE::QemuConfig->foreach_volume($conf, sub {
8315         my ($ds, $drive) = @_;
8316
8317         if (drive_is_cdrom ($drive, 1)) {
8318             if ($bootindex_hash->{d}) {
8319                 $bootorder->{$ds} = $bootindex_hash->{d};
8320                 $bootindex_hash->{d} += 1;
8321             }
8322         } elsif ($bootindex_hash->{c}) {
8323             $bootorder->{$ds} = $bootindex_hash->{c}
8324                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8325             $bootindex_hash->{c} += 1;
8326         }
8327     });
8328
8329     if ($bootindex_hash->{n}) {
8330         for (my $i = 0; $i < $MAX_NETS; $i++) {
8331             my $netname = "net$i";
8332             next if !$conf->{$netname};
8333             $bootorder->{$netname} = $bootindex_hash->{n};
8334             $bootindex_hash->{n} += 1;
8335         }
8336     }
8337
8338     return $bootorder;
8339 }
8340
8341 # Generate default device list for 'boot: order=' property. Matches legacy
8342 # default boot order, but with explicit device names. This is important, since
8343 # the fallback for when neither 'order' nor the old format is specified relies
8344 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8345 sub get_default_bootdevices {
8346     my ($conf) = @_;
8347
8348     my @ret = ();
8349
8350     # harddisk
8351     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8352     push @ret, $first if $first;
8353
8354     # cdrom
8355     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8356     push @ret, $first if $first;
8357
8358     # network
8359     for (my $i = 0; $i < $MAX_NETS; $i++) {
8360         my $netname = "net$i";
8361         next if !$conf->{$netname};
8362         push @ret, $netname;
8363         last;
8364     }
8365
8366     return \@ret;
8367 }
8368
8369 sub device_bootorder {
8370     my ($conf) = @_;
8371
8372     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8373
8374     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8375
8376     my $bootorder = {};
8377     if (!defined($boot) || $boot->{legacy}) {
8378         $bootorder = bootorder_from_legacy($conf, $boot);
8379     } elsif ($boot->{order}) {
8380         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8381         for my $dev (PVE::Tools::split_list($boot->{order})) {
8382             $bootorder->{$dev} = $i++;
8383         }
8384     }
8385
8386     return $bootorder;
8387 }
8388
8389 sub register_qmeventd_handle {
8390     my ($vmid) = @_;
8391
8392     my $fh;
8393     my $peer = "/var/run/qmeventd.sock";
8394     my $count = 0;
8395
8396     for (;;) {
8397         $count++;
8398         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8399         last if $fh;
8400         if ($! != EINTR && $! != EAGAIN) {
8401             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8402         }
8403         if ($count > 4) {
8404             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8405               . "after $count retries\n";
8406         }
8407         usleep(25000);
8408     }
8409
8410     # send handshake to mark VM as backing up
8411     print $fh to_json({vzdump => {vmid => "$vmid"}});
8412
8413     # return handle to be closed later when inhibit is no longer required
8414     return $fh;
8415 }
8416
8417 # bash completion helper
8418
8419 sub complete_backup_archives {
8420     my ($cmdname, $pname, $cvalue) = @_;
8421
8422     my $cfg = PVE::Storage::config();
8423
8424     my $storeid;
8425
8426     if ($cvalue =~ m/^([^:]+):/) {
8427         $storeid = $1;
8428     }
8429
8430     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8431
8432     my $res = [];
8433     foreach my $id (keys %$data) {
8434         foreach my $item (@{$data->{$id}}) {
8435             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8436             push @$res, $item->{volid} if defined($item->{volid});
8437         }
8438     }
8439
8440     return $res;
8441 }
8442
8443 my $complete_vmid_full = sub {
8444     my ($running) = @_;
8445
8446     my $idlist = vmstatus();
8447
8448     my $res = [];
8449
8450     foreach my $id (keys %$idlist) {
8451         my $d = $idlist->{$id};
8452         if (defined($running)) {
8453             next if $d->{template};
8454             next if $running && $d->{status} ne 'running';
8455             next if !$running && $d->{status} eq 'running';
8456         }
8457         push @$res, $id;
8458
8459     }
8460     return $res;
8461 };
8462
8463 sub complete_vmid {
8464     return &$complete_vmid_full();
8465 }
8466
8467 sub complete_vmid_stopped {
8468     return &$complete_vmid_full(0);
8469 }
8470
8471 sub complete_vmid_running {
8472     return &$complete_vmid_full(1);
8473 }
8474
8475 sub complete_storage {
8476
8477     my $cfg = PVE::Storage::config();
8478     my $ids = $cfg->{ids};
8479
8480     my $res = [];
8481     foreach my $sid (keys %$ids) {
8482         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8483         next if !$ids->{$sid}->{content}->{images};
8484         push @$res, $sid;
8485     }
8486
8487     return $res;
8488 }
8489
8490 sub complete_migration_storage {
8491     my ($cmd, $param, $current_value, $all_args) = @_;
8492
8493     my $targetnode = @$all_args[1];
8494
8495     my $cfg = PVE::Storage::config();
8496     my $ids = $cfg->{ids};
8497
8498     my $res = [];
8499     foreach my $sid (keys %$ids) {
8500         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8501         next if !$ids->{$sid}->{content}->{images};
8502         push @$res, $sid;
8503     }
8504
8505     return $res;
8506 }
8507
8508 sub vm_is_paused {
8509     my ($vmid) = @_;
8510     my $qmpstatus = eval {
8511         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8512         mon_cmd($vmid, "query-status");
8513     };
8514     warn "$@\n" if $@;
8515     return $qmpstatus && $qmpstatus->{status} eq "paused";
8516 }
8517
8518 sub check_volume_storage_type {
8519     my ($storecfg, $vol) = @_;
8520
8521     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8522     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8523     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8524
8525     die "storage '$storeid' does not support content-type '$vtype'\n"
8526         if !$scfg->{content}->{$vtype};
8527
8528     return 1;
8529 }
8530
8531 sub add_nets_bridge_fdb {
8532     my ($conf, $vmid) = @_;
8533
8534     for my $opt (keys %$conf) {
8535         next if $opt !~ m/^net(\d+)$/;
8536         my $iface = "tap${vmid}i$1";
8537         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8538         my $net = parse_net($conf->{$opt}, 1) or next;
8539
8540         my $mac = $net->{macaddr};
8541         if (!$mac) {
8542             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8543                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8544             next;
8545         }
8546
8547         my $bridge = $net->{bridge};
8548         if (!$bridge) {
8549             log_warn("Interface '$iface' not attached to any bridge.");
8550             next;
8551         }
8552         if ($have_sdn) {
8553             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8554         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8555             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8556         }
8557     }
8558 }
8559
8560 sub del_nets_bridge_fdb {
8561     my ($conf, $vmid) = @_;
8562
8563     for my $opt (keys %$conf) {
8564         next if $opt !~ m/^net(\d+)$/;
8565         my $iface = "tap${vmid}i$1";
8566
8567         my $net = parse_net($conf->{$opt}) or next;
8568         my $mac = $net->{macaddr} or next;
8569
8570         my $bridge = $net->{bridge};
8571         if ($have_sdn) {
8572             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8573         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8574             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8575         }
8576     }
8577 }
8578
8579 1;