PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196 };
 197
 198 my $ivshmem_fmt = {
 199     size => {
 200         type => 'integer',
 201         minimum => 1,
 202         description => "The size of the file in MB.",
 203     },
 204     name => {
 205         type => 'string',
 206         pattern => '[a-zA-Z0-9\-]+',
 207         optional => 1,
 208         format_description => 'string',
 209         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 210     },
 211 };
 212
 213 my $audio_fmt = {
 214     device => {
 215         type => 'string',
 216         enum => [qw(ich9-intel-hda intel-hda AC97)],
 217         description =>  "Configure an audio device."
 218     },
 219     driver =>  {
 220         type => 'string',
 221         enum => ['spice', 'none'],
 222         default => 'spice',
 223         optional => 1,
 224         description => "Driver backend for the audio device."
 225     },
 226 };
 227
 228 my $spice_enhancements_fmt = {
 229     foldersharing => {
 230         type => 'boolean',
 231         optional => 1,
 232         default => '0',
 233         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 234     },
 235     videostreaming =>  {
 236         type => 'string',
 237         enum => ['off', 'all', 'filter'],
 238         default => 'off',
 239         optional => 1,
 240         description => "Enable video streaming. Uses compression for detected video streams."
 241     },
 242 };
 243
 244 my $rng_fmt = {
 245     source => {
 246         type => 'string',
 247         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 248         default_key => 1,
 249         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 250             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 251             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 252             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 253             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 254             ." a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 259             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 260             ." `0` to disable limiting (potentially dangerous!).",
 261         optional => 1,
 262
 263         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 264         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 265         # reading from /dev/urandom
 266         default => 1024,
 267     },
 268     period => {
 269         type => 'integer',
 270         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 271             ." the guest to retrieve another 'max_bytes' of entropy.",
 272         optional => 1,
 273         default => 1000,
 274     },
 275 };
 276
 277 my $meta_info_fmt = {
 278     'ctime' => {
 279         type => 'integer',
 280         description => "The guest creation timestamp as UNIX epoch time",
 281         minimum => 0,
 282         optional => 1,
 283     },
 284     'creation-qemu' => {
 285         type => 'string',
 286         description => "The QEMU (machine) version from the time this VM was created.",
 287         pattern => '\d+(\.\d+)+',
 288         optional => 1,
 289     },
 290 };
 291
 292 my $confdesc = {
 293     onboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Specifies whether a VM will be started during system bootup.",
 297         default => 0,
 298     },
 299     autostart => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Automatic restart after crash (currently ignored).",
 303         default => 0,
 304     },
 305     hotplug => {
 306         optional => 1,
 307         type => 'string', format => 'pve-hotplug-features',
 308         description => "Selectively enable hotplug features. This is a comma separated list of"
 309             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 310             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 311             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 312             ." windows > 7.",
 313         default => 'network,disk,usb',
 314     },
 315     reboot => {
 316         optional => 1,
 317         type => 'boolean',
 318         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 319         default => 1,
 320     },
 321     lock => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Lock/unlock the VM.",
 325         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 326     },
 327     cpulimit => {
 328         optional => 1,
 329         type => 'number',
 330         description => "Limit of CPU usage.",
 331         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 332             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 333         minimum => 0,
 334         maximum => 128,
 335         default => 0,
 336     },
 337     cpuunits => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 341         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 342             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 343             ." weights of all the other running VMs.",
 344         minimum => 1,
 345         maximum => 262144,
 346         default => 'cgroup v1: 1024, cgroup v2: 100',
 347     },
 348     memory => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 352             ." you use the balloon device.",
 353         minimum => 16,
 354         default => 512,
 355     },
 356     balloon => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 360         minimum => 0,
 361     },
 362     shares => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 366             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 367             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 368         minimum => 0,
 369         maximum => 50000,
 370         default => 1000,
 371     },
 372     keyboard => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Keyboard layout for VNC server. This option is generally not required and"
 376         ." is often better handled from within the guest OS.",
 377         enum => PVE::Tools::kvmkeymaplist(),
 378         default => undef,
 379     },
 380     name => {
 381         optional => 1,
 382         type => 'string', format => 'dns-name',
 383         description => "Set a name for the VM. Only used on the configuration web interface.",
 384     },
 385     scsihw => {
 386         optional => 1,
 387         type => 'string',
 388         description => "SCSI controller model",
 389         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 390         default => 'lsi',
 391     },
 392     description => {
 393         optional => 1,
 394         type => 'string',
 395         description => "Description for the VM. Shown in the web-interface VM's summary."
 396             ." This is saved as comment inside the configuration file.",
 397         maxLength => 1024 * 8,
 398     },
 399     ostype => {
 400         optional => 1,
 401         type => 'string',
 402         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 403         description => "Specify guest operating system.",
 404         verbose_description => <<EODESC,
 405 Specify guest operating system. This is used to enable special
 406 optimization/features for specific operating systems:
 407
 408 [horizontal]
 409 other;; unspecified OS
 410 wxp;; Microsoft Windows XP
 411 w2k;; Microsoft Windows 2000
 412 w2k3;; Microsoft Windows 2003
 413 w2k8;; Microsoft Windows 2008
 414 wvista;; Microsoft Windows Vista
 415 win7;; Microsoft Windows 7
 416 win8;; Microsoft Windows 8/2012/2012r2
 417 win10;; Microsoft Windows 10/2016/2019
 418 win11;; Microsoft Windows 11/2022
 419 l24;; Linux 2.4 Kernel
 420 l26;; Linux 2.6 - 6.X Kernel
 421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 422 EODESC
 423     },
 424     boot => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-boot',
 427         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 428             ." key or 'legacy=' is deprecated.",
 429     },
 430     bootdisk => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-bootdisk',
 433         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 434         pattern => '(ide|sata|scsi|virtio)\d+',
 435     },
 436     smp => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPUs. Please use option -sockets instead.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     sockets => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of CPU sockets.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     cores => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of cores per socket.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     numa => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable NUMA.",
 461         default => 0,
 462     },
 463     hugepages => {
 464         optional => 1,
 465         type => 'string',
 466         description => "Enable/disable hugepages memory.",
 467         enum => [qw(any 2 1024)],
 468     },
 469     keephugepages => {
 470         optional => 1,
 471         type => 'boolean',
 472         default => 0,
 473         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 474             ." after VM shutdown and can be used for subsequent starts.",
 475     },
 476     vcpus => {
 477         optional => 1,
 478         type => 'integer',
 479         description => "Number of hotplugged vcpus.",
 480         minimum => 1,
 481         default => 0,
 482     },
 483     acpi => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Enable/disable ACPI.",
 487         default => 1,
 488     },
 489     agent => {
 490         optional => 1,
 491         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 492         type => 'string',
 493         format => $agent_fmt,
 494     },
 495     kvm => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable KVM hardware virtualization.",
 499         default => 1,
 500     },
 501     tdf => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable time drift fix.",
 505         default => 0,
 506     },
 507     localtime => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 511             ." the `ostype` indicates a Microsoft Windows OS.",
 512     },
 513     freeze => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 517     },
 518     vga => {
 519         optional => 1,
 520         type => 'string', format => $vga_fmt,
 521         description => "Configure the VGA hardware.",
 522         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 523             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 524             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 525             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 526             ." display server. For win* OS you can select how many independent displays you want,"
 527             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 528             ." using a serial device as terminal.",
 529     },
 530     watchdog => {
 531         optional => 1,
 532         type => 'string', format => 'pve-qm-watchdog',
 533         description => "Create a virtual hardware watchdog device.",
 534         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 535             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 536             ." else the watchdog will reset the guest (or execute the respective action specified)",
 537     },
 538     startdate => {
 539         optional => 1,
 540         type => 'string',
 541         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 542         description => "Set the initial date of the real time clock. Valid format for date are:"
 543             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 544         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 545         default => 'now',
 546     },
 547     startup =>  get_standard_option('pve-startup-order'),
 548     template => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Enable/disable Template.",
 552         default => 0,
 553     },
 554     args => {
 555         optional => 1,
 556         type => 'string',
 557         description => "Arbitrary arguments passed to kvm.",
 558         verbose_description => <<EODESCR,
 559 Arbitrary arguments passed to kvm, for example:
 560
 561 args: -no-reboot -no-hpet
 562
 563 NOTE: this option is for experts only.
 564 EODESCR
 565     },
 566     tablet => {
 567         optional => 1,
 568         type => 'boolean',
 569         default => 1,
 570         description => "Enable/disable the USB tablet device.",
 571         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 572             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 573             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 574             ." may consider disabling this to save some context switches. This is turned off by"
 575             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 576     },
 577     migrate_speed => {
 578         optional => 1,
 579         type => 'integer',
 580         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 581         minimum => 0,
 582         default => 0,
 583     },
 584     migrate_downtime => {
 585         optional => 1,
 586         type => 'number',
 587         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 588         minimum => 0,
 589         default => 0.1,
 590     },
 591     cdrom => {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-ide',
 594         typetext => '<volume>',
 595         description => "This is an alias for option -ide2",
 596     },
 597     cpu => {
 598         optional => 1,
 599         description => "Emulated CPU type.",
 600         type => 'string',
 601         format => 'pve-vm-cpu-conf',
 602     },
 603     parent => get_standard_option('pve-snapshot-name', {
 604         optional => 1,
 605         description => "Parent snapshot name. This is used internally, and should not be modified.",
 606     }),
 607     snaptime => {
 608         optional => 1,
 609         description => "Timestamp for snapshots.",
 610         type => 'integer',
 611         minimum => 0,
 612     },
 613     vmstate => {
 614         optional => 1,
 615         type => 'string', format => 'pve-volume-id',
 616         description => "Reference to a volume which stores the VM state. This is used internally"
 617             ." for snapshots.",
 618     },
 619     vmstatestorage => get_standard_option('pve-storage-id', {
 620         description => "Default storage for VM state volumes/files.",
 621         optional => 1,
 622     }),
 623     runningmachine => get_standard_option('pve-qemu-machine', {
 624         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 625             ." for snapshots.",
 626     }),
 627     runningcpu => {
 628         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 629             ." internally for snapshots.",
 630         optional => 1,
 631         type => 'string',
 632         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 633         format_description => 'QEMU -cpu parameter'
 634     },
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 652             ." remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 667             ." to disable explicitly.",
 668         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 669             ." value identifier to the guest OS. This allows to notify the guest operating system"
 670             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 671             ." execution or creation from a template). The guest operating system notices the"
 672             ." change, and is then able to react as appropriate by marking its copies of"
 673             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 674             ."Note that auto-creation only works when done through API/CLI create or update methods"
 675             .", but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 689             ." the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704     tags => {
 705         type => 'string', format => 'pve-tag-list',
 706         description => 'Tags of the VM. This is only meta information.',
 707         optional => 1,
 708     },
 709     rng0 => {
 710         type => 'string',
 711         format => $rng_fmt,
 712         description => "Configure a VirtIO-based Random Number Generator.",
 713         optional => 1,
 714     },
 715     meta => {
 716         type => 'string',
 717         format => $meta_info_fmt,
 718         description => "Some (read-only) meta-information about this guest.",
 719         optional => 1,
 720     },
 721     affinity => {
 722         type => 'string', format => 'pve-cpuset',
 723         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 724         optional => 1,
 725     },
 726 };
 727
 728 my $cicustom_fmt = {
 729     meta => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all meta data passed to the VM via"
 733             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     network => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     user => {
 745         type => 'string',
 746         optional => 1,
 747         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 748         format => 'pve-volume-id',
 749         format_description => 'volume',
 750     },
 751     vendor => {
 752         type => 'string',
 753         optional => 1,
 754         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 760
 761 my $confdesc_cloudinit = {
 762     citype => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'Specifies the cloud-init configuration format. The default depends on the'
 766             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 767             .' and `configdrive2` for windows.',
 768         enum => ['configdrive2', 'nocloud', 'opennebula'],
 769     },
 770     ciuser => {
 771         optional => 1,
 772         type => 'string',
 773         description => "cloud-init: User name to change ssh keys and password for instead of the"
 774             ." image's configured default user.",
 775     },
 776     cipassword => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Password to assign the user. Using this is generally not'
 780             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 781             .' support hashed passwords.',
 782     },
 783     cicustom => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Specify custom files to replace the automatically generated'
 787             .' ones at start.',
 788         format => 'pve-qm-cicustom',
 789     },
 790     searchdomain => {
 791         optional => 1,
 792         type => 'string',
 793         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     nameserver => {
 798         optional => 1,
 799         type => 'string', format => 'address-list',
 800         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 801             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 802             .' are set.',
 803     },
 804     sshkeys => {
 805         optional => 1,
 806         type => 'string',
 807         format => 'urlencoded',
 808         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 809     },
 810 };
 811
 812 # what about other qemu settings ?
 813 #cpu => 'string',
 814 #machine => 'string',
 815 #fda => 'file',
 816 #fdb => 'file',
 817 #mtdblock => 'file',
 818 #sd => 'file',
 819 #pflash => 'file',
 820 #snapshot => 'bool',
 821 #bootp => 'file',
 822 ##tftp => 'dir',
 823 ##smb => 'dir',
 824 #kernel => 'file',
 825 #append => 'string',
 826 #initrd => 'file',
 827 ##soundhw => 'string',
 828
 829 while (my ($k, $v) = each %$confdesc) {
 830     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 831 }
 832
 833 my $MAX_USB_DEVICES = 14;
 834 my $MAX_NETS = 32;
 835 my $MAX_SERIAL_PORTS = 4;
 836 my $MAX_PARALLEL_PORTS = 3;
 837 my $MAX_NUMA = 8;
 838
 839 my $numa_fmt = {
 840     cpus => {
 841         type => "string",
 842         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 843         description => "CPUs accessing this NUMA node.",
 844         format_description => "id[-id];...",
 845     },
 846     memory => {
 847         type => "number",
 848         description => "Amount of memory this NUMA node provides.",
 849         optional => 1,
 850     },
 851     hostnodes => {
 852         type => "string",
 853         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 854         description => "Host NUMA nodes to use.",
 855         format_description => "id[-id];...",
 856         optional => 1,
 857     },
 858     policy => {
 859         type => 'string',
 860         enum => [qw(preferred bind interleave)],
 861         description => "NUMA allocation policy.",
 862         optional => 1,
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 866 my $numadesc = {
 867     optional => 1,
 868     type => 'string', format => $numa_fmt,
 869     description => "NUMA topology.",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 872
 873 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 874     $confdesc->{"numa$i"} = $numadesc;
 875 }
 876
 877 my $nic_model_list = [
 878     'e1000',
 879     'e1000-82540em',
 880     'e1000-82544gc',
 881     'e1000-82545em',
 882     'e1000e',
 883     'i82551',
 884     'i82557b',
 885     'i82559er',
 886     'ne2k_isa',
 887     'ne2k_pci',
 888     'pcnet',
 889     'rtl8139',
 890     'virtio',
 891     'vmxnet3',
 892 ];
 893 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 894
 895 my $net_fmt_bridge_descr = <<__EOD__;
 896 Bridge to attach the network device to. The Proxmox VE standard bridge
 897 is called 'vmbr0'.
 898
 899 If you do not specify a bridge, we create a kvm user (NATed) network
 900 device, which provides DHCP and DNS services. The following addresses
 901 are used:
 902
 903  10.0.2.2   Gateway
 904  10.0.2.3   DNS Server
 905  10.0.2.4   SMB Server
 906
 907 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 908 __EOD__
 909
 910 my $net_fmt = {
 911     macaddr  => get_standard_option('mac-addr', {
 912         description => "MAC address. That address must be unique withing your network. This is"
 913             ." automatically generated if not specified.",
 914     }),
 915     model => {
 916         type => 'string',
 917         description => "Network Card Model. The 'virtio' model provides the best performance with"
 918             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 919             ." best to use 'e1000'.",
 920         enum => $nic_model_list,
 921         default_key => 1,
 922     },
 923     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 924     bridge => get_standard_option('pve-bridge-id', {
 925         description => $net_fmt_bridge_descr,
 926         optional => 1,
 927     }),
 928     queues => {
 929         type => 'integer',
 930         minimum => 0, maximum => 64,
 931         description => 'Number of packet queues to be used on the device.',
 932         optional => 1,
 933     },
 934     rate => {
 935         type => 'number',
 936         minimum => 0,
 937         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 938         optional => 1,
 939     },
 940     tag => {
 941         type => 'integer',
 942         minimum => 1, maximum => 4094,
 943         description => 'VLAN tag to apply to packets on this interface.',
 944         optional => 1,
 945     },
 946     trunks => {
 947         type => 'string',
 948         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 949         description => 'VLAN trunks to pass through this interface.',
 950         format_description => 'vlanid[;vlanid...]',
 951         optional => 1,
 952     },
 953     firewall => {
 954         type => 'boolean',
 955         description => 'Whether this interface should be protected by the firewall.',
 956         optional => 1,
 957     },
 958     link_down => {
 959         type => 'boolean',
 960         description => 'Whether this interface should be disconnected (like pulling the plug).',
 961         optional => 1,
 962     },
 963     mtu => {
 964         type => 'integer',
 965         minimum => 1, maximum => 65520,
 966         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 967         optional => 1,
 968     },
 969 };
 970
 971 my $netdesc = {
 972     optional => 1,
 973     type => 'string', format => $net_fmt,
 974     description => "Specify network devices.",
 975 };
 976
 977 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 978
 979 my $ipconfig_fmt = {
 980     ip => {
 981         type => 'string',
 982         format => 'pve-ipv4-config',
 983         format_description => 'IPv4Format/CIDR',
 984         description => 'IPv4 address in CIDR format.',
 985         optional => 1,
 986         default => 'dhcp',
 987     },
 988     gw => {
 989         type => 'string',
 990         format => 'ipv4',
 991         format_description => 'GatewayIPv4',
 992         description => 'Default gateway for IPv4 traffic.',
 993         optional => 1,
 994         requires => 'ip',
 995     },
 996     ip6 => {
 997         type => 'string',
 998         format => 'pve-ipv6-config',
 999         format_description => 'IPv6Format/CIDR',
1000         description => 'IPv6 address in CIDR format.',
1001         optional => 1,
1002         default => 'dhcp',
1003     },
1004     gw6 => {
1005         type => 'string',
1006         format => 'ipv6',
1007         format_description => 'GatewayIPv6',
1008         description => 'Default gateway for IPv6 traffic.',
1009         optional => 1,
1010         requires => 'ip6',
1011     },
1012 };
1013 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1014 my $ipconfigdesc = {
1015     optional => 1,
1016     type => 'string', format => 'pve-qm-ipconfig',
1017     description => <<'EODESCR',
1018 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1019
1020 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1021
1022 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1023 gateway should be provided.
1024 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1025 cloud-init 19.4 or newer.
1026
1027 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1028 dhcp on IPv4.
1029 EODESCR
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1032
1033 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1034     $confdesc->{"net$i"} = $netdesc;
1035     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1036 }
1037
1038 foreach my $key (keys %$confdesc_cloudinit) {
1039     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1040 }
1041
1042 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1043 sub pve_verify_cpuset {
1044     my ($set_text, $noerr) = @_;
1045
1046     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1047
1048     if ($@) {
1049         return if $noerr;
1050         die "unable to parse cpuset option\n";
1051     }
1052
1053     return PVE::CpuSet->new($members)->short_string();
1054 }
1055
1056 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1057 sub verify_volume_id_or_qm_path {
1058     my ($volid, $noerr) = @_;
1059
1060     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1061
1062     return verify_volume_id_or_absolute_path($volid, $noerr);
1063 }
1064
1065 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1066 sub verify_volume_id_or_absolute_path {
1067     my ($volid, $noerr) = @_;
1068
1069     return $volid if $volid =~ m|^/|;
1070
1071     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1072     if ($@) {
1073         return if $noerr;
1074         die $@;
1075     }
1076     return $volid;
1077 }
1078
1079 my $usb_fmt = {
1080     host => {
1081         default_key => 1,
1082         type => 'string', format => 'pve-qm-usb-device',
1083         format_description => 'HOSTUSBDEVICE|spice',
1084         description => <<EODESCR,
1085 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1086
1087  'bus-port(.port)*' (decimal numbers) or
1088  'vendor_id:product_id' (hexadeciaml numbers) or
1089  'spice'
1090
1091 You can use the 'lsusb -t' command to list existing usb devices.
1092
1093 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1094 machines - use with special care.
1095
1096 The value 'spice' can be used to add a usb redirection devices for spice.
1097 EODESCR
1098     },
1099     usb3 => {
1100         optional => 1,
1101         type => 'boolean',
1102         description => "Specifies whether if given host option is a USB3 device or port."
1103             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1104             ." is irrelevant (all devices are plugged into a xhci controller).",
1105         default => 0,
1106     },
1107 };
1108
1109 my $usbdesc = {
1110     optional => 1,
1111     type => 'string', format => $usb_fmt,
1112     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1113         ." l26 or windows > 7, n can be up to 14).",
1114 };
1115 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1116
1117 my $serialdesc = {
1118         optional => 1,
1119         type => 'string',
1120         pattern => '(/dev/.+|socket)',
1121         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1122         verbose_description =>  <<EODESCR,
1123 Create a serial device inside the VM (n is 0 to 3), and pass through a
1124 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1125 host side (use 'qm terminal' to open a terminal connection).
1126
1127 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1128 use with special care.
1129
1130 CAUTION: Experimental! User reported problems with this option.
1131 EODESCR
1132 };
1133
1134 my $paralleldesc= {
1135         optional => 1,
1136         type => 'string',
1137         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1138         description =>  "Map host parallel devices (n is 0 to 2).",
1139         verbose_description =>  <<EODESCR,
1140 Map host parallel devices (n is 0 to 2).
1141
1142 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1143 machines - use with special care.
1144
1145 CAUTION: Experimental! User reported problems with this option.
1146 EODESCR
1147 };
1148
1149 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1150     $confdesc->{"parallel$i"} = $paralleldesc;
1151 }
1152
1153 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1154     $confdesc->{"serial$i"} = $serialdesc;
1155 }
1156
1157 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1158     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1159 }
1160
1161 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1162     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1163 }
1164
1165 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1166     $confdesc->{"usb$i"} = $usbdesc;
1167 }
1168
1169 my $boot_fmt = {
1170     legacy => {
1171         optional => 1,
1172         default_key => 1,
1173         type => 'string',
1174         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1175                      . " Deprecated, use 'order=' instead.",
1176         pattern => '[acdn]{1,4}',
1177         format_description => "[acdn]{1,4}",
1178
1179         # note: this is also the fallback if boot: is not given at all
1180         default => 'cdn',
1181     },
1182     order => {
1183         optional => 1,
1184         type => 'string',
1185         format => 'pve-qm-bootdev-list',
1186         format_description => "device[;device...]",
1187         description => <<EODESC,
1188 The guest will attempt to boot from devices in the order they appear here.
1189
1190 Disks, optical drives and passed-through storage USB devices will be directly
1191 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1192 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1193
1194 Note that only devices in this list will be marked as bootable and thus loaded
1195 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1196 (e.g. software-raid), you need to specify all of them here.
1197
1198 Overrides the deprecated 'legacy=[acdn]*' value when given.
1199 EODESC
1200     },
1201 };
1202 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1203
1204 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1205 sub verify_bootdev {
1206     my ($dev, $noerr) = @_;
1207
1208     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1209     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1210
1211     my $check = sub {
1212         my ($base) = @_;
1213         return 0 if $dev !~ m/^$base\d+$/;
1214         return 0 if !$confdesc->{$dev};
1215         return 1;
1216     };
1217
1218     return $dev if $check->("net");
1219     return $dev if $check->("usb");
1220     return $dev if $check->("hostpci");
1221
1222     return if $noerr;
1223     die "invalid boot device '$dev'\n";
1224 }
1225
1226 sub print_bootorder {
1227     my ($devs) = @_;
1228     return "" if !@$devs;
1229     my $data = { order => join(';', @$devs) };
1230     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1231 }
1232
1233 my $kvm_api_version = 0;
1234
1235 sub kvm_version {
1236     return $kvm_api_version if $kvm_api_version;
1237
1238     open my $fh, '<', '/dev/kvm' or return;
1239
1240     # 0xae00 => KVM_GET_API_VERSION
1241     $kvm_api_version = ioctl($fh, 0xae00, 0);
1242     close($fh);
1243
1244     return $kvm_api_version;
1245 }
1246
1247 my $kvm_user_version = {};
1248 my $kvm_mtime = {};
1249
1250 sub kvm_user_version {
1251     my ($binary) = @_;
1252
1253     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1254     my $st = stat($binary);
1255
1256     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1257     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1258         $cachedmtime == $st->mtime;
1259
1260     $kvm_user_version->{$binary} = 'unknown';
1261     $kvm_mtime->{$binary} = $st->mtime;
1262
1263     my $code = sub {
1264         my $line = shift;
1265         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1266             $kvm_user_version->{$binary} = $2;
1267         }
1268     };
1269
1270     eval { run_command([$binary, '--version'], outfunc => $code); };
1271     warn $@ if $@;
1272
1273     return $kvm_user_version->{$binary};
1274
1275 }
1276 my sub extract_version {
1277     my ($machine_type, $version) = @_;
1278     $version = kvm_user_version() if !defined($version);
1279     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1280 }
1281
1282 sub kernel_has_vhost_net {
1283     return -c '/dev/vhost-net';
1284 }
1285
1286 sub option_exists {
1287     my $key = shift;
1288     return defined($confdesc->{$key});
1289 }
1290
1291 my $cdrom_path;
1292 sub get_cdrom_path {
1293
1294     return $cdrom_path if defined($cdrom_path);
1295
1296     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1297
1298     if (!defined($cdrom_path)) {
1299         log_warn("no physical CD-ROM available, ignoring");
1300         $cdrom_path = '';
1301     }
1302
1303     return $cdrom_path;
1304 }
1305
1306 sub get_iso_path {
1307     my ($storecfg, $vmid, $cdrom) = @_;
1308
1309     if ($cdrom eq 'cdrom') {
1310         return get_cdrom_path();
1311     } elsif ($cdrom eq 'none') {
1312         return '';
1313     } elsif ($cdrom =~ m|^/|) {
1314         return $cdrom;
1315     } else {
1316         return PVE::Storage::path($storecfg, $cdrom);
1317     }
1318 }
1319
1320 # try to convert old style file names to volume IDs
1321 sub filename_to_volume_id {
1322     my ($vmid, $file, $media) = @_;
1323
1324      if (!($file eq 'none' || $file eq 'cdrom' ||
1325           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1326
1327         return if $file =~ m|/|;
1328
1329         if ($media && $media eq 'cdrom') {
1330             $file = "local:iso/$file";
1331         } else {
1332             $file = "local:$vmid/$file";
1333         }
1334     }
1335
1336     return $file;
1337 }
1338
1339 sub verify_media_type {
1340     my ($opt, $vtype, $media) = @_;
1341
1342     return if !$media;
1343
1344     my $etype;
1345     if ($media eq 'disk') {
1346         $etype = 'images';
1347     } elsif ($media eq 'cdrom') {
1348         $etype = 'iso';
1349     } else {
1350         die "internal error";
1351     }
1352
1353     return if ($vtype eq $etype);
1354
1355     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1356 }
1357
1358 sub cleanup_drive_path {
1359     my ($opt, $storecfg, $drive) = @_;
1360
1361     # try to convert filesystem paths to volume IDs
1362
1363     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1364         ($drive->{file} !~ m|^/dev/.+|) &&
1365         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1366         ($drive->{file} !~ m/^\d+$/)) {
1367         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1368         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1369             if !$vtype;
1370         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1371         verify_media_type($opt, $vtype, $drive->{media});
1372         $drive->{file} = $volid;
1373     }
1374
1375     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1376 }
1377
1378 sub parse_hotplug_features {
1379     my ($data) = @_;
1380
1381     my $res = {};
1382
1383     return $res if $data eq '0';
1384
1385     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1386
1387     foreach my $feature (PVE::Tools::split_list($data)) {
1388         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1389             $res->{$1} = 1;
1390         } else {
1391             die "invalid hotplug feature '$feature'\n";
1392         }
1393     }
1394     return $res;
1395 }
1396
1397 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1398 sub pve_verify_hotplug_features {
1399     my ($value, $noerr) = @_;
1400
1401     return $value if parse_hotplug_features($value);
1402
1403     return if $noerr;
1404
1405     die "unable to parse hotplug option\n";
1406 }
1407
1408 sub scsi_inquiry {
1409     my($fh, $noerr) = @_;
1410
1411     my $SG_IO = 0x2285;
1412     my $SG_GET_VERSION_NUM = 0x2282;
1413
1414     my $versionbuf = "\x00" x 8;
1415     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1416     if (!$ret) {
1417         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1418         return;
1419     }
1420     my $version = unpack("I", $versionbuf);
1421     if ($version < 30000) {
1422         die "scsi generic interface too old\n"  if !$noerr;
1423         return;
1424     }
1425
1426     my $buf = "\x00" x 36;
1427     my $sensebuf = "\x00" x 8;
1428     my $cmd = pack("C x3 C x1", 0x12, 36);
1429
1430     # see /usr/include/scsi/sg.h
1431     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1432
1433     my $packet = pack(
1434         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1435     );
1436
1437     $ret = ioctl($fh, $SG_IO, $packet);
1438     if (!$ret) {
1439         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1440         return;
1441     }
1442
1443     my @res = unpack($sg_io_hdr_t, $packet);
1444     if ($res[17] || $res[18]) {
1445         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1446         return;
1447     }
1448
1449     my $res = {};
1450     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1451
1452     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1453     $res->{type} &= 0x1F;
1454
1455     return $res;
1456 }
1457
1458 sub path_is_scsi {
1459     my ($path) = @_;
1460
1461     my $fh = IO::File->new("+<$path") || return;
1462     my $res = scsi_inquiry($fh, 1);
1463     close($fh);
1464
1465     return $res;
1466 }
1467
1468 sub print_tabletdevice_full {
1469     my ($conf, $arch) = @_;
1470
1471     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1472
1473     # we use uhci for old VMs because tablet driver was buggy in older qemu
1474     my $usbbus;
1475     if ($q35 || $arch eq 'aarch64') {
1476         $usbbus = 'ehci';
1477     } else {
1478         $usbbus = 'uhci';
1479     }
1480
1481     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1482 }
1483
1484 sub print_keyboarddevice_full {
1485     my ($conf, $arch) = @_;
1486
1487     return if $arch ne 'aarch64';
1488
1489     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1490 }
1491
1492 my sub get_drive_id {
1493     my ($drive) = @_;
1494     return "$drive->{interface}$drive->{index}";
1495 }
1496
1497 sub print_drivedevice_full {
1498     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1499
1500     my $device = '';
1501     my $maxdev = 0;
1502
1503     my $drive_id = get_drive_id($drive);
1504     if ($drive->{interface} eq 'virtio') {
1505         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1506         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1507         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1508     } elsif ($drive->{interface} eq 'scsi') {
1509
1510         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1511         my $unit = $drive->{index} % $maxdev;
1512         my $devicetype = 'hd';
1513         my $path = '';
1514         if (drive_is_cdrom($drive)) {
1515             $devicetype = 'cd';
1516         } else {
1517             if ($drive->{file} =~ m|^/|) {
1518                 $path = $drive->{file};
1519                 if (my $info = path_is_scsi($path)) {
1520                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1521                         $devicetype = 'block';
1522                     } elsif ($info->{type} == 1) { # tape
1523                         $devicetype = 'generic';
1524                     }
1525                 }
1526             } else {
1527                  $path = PVE::Storage::path($storecfg, $drive->{file});
1528             }
1529
1530             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1531             my $version = extract_version($machine_type, kvm_user_version());
1532             if ($path =~ m/^iscsi\:\/\// &&
1533                !min_version($version, 4, 1)) {
1534                 $devicetype = 'generic';
1535             }
1536         }
1537
1538         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1539             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1540         } else {
1541             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1542                 .",lun=$drive->{index}";
1543         }
1544         $device .= ",drive=drive-$drive_id,id=$drive_id";
1545
1546         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1547             $device .= ",rotation_rate=1";
1548         }
1549         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1550
1551     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1552         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1553         my $controller = int($drive->{index} / $maxdev);
1554         my $unit = $drive->{index} % $maxdev;
1555         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1556
1557         $device = "ide-$devicetype";
1558         if ($drive->{interface} eq 'ide') {
1559             $device .= ",bus=ide.$controller,unit=$unit";
1560         } else {
1561             $device .= ",bus=ahci$controller.$unit";
1562         }
1563         $device .= ",drive=drive-$drive_id,id=$drive_id";
1564
1565         if ($devicetype eq 'hd') {
1566             if (my $model = $drive->{model}) {
1567                 $model = URI::Escape::uri_unescape($model);
1568                 $device .= ",model=$model";
1569             }
1570             if ($drive->{ssd}) {
1571                 $device .= ",rotation_rate=1";
1572             }
1573         }
1574         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1575     } elsif ($drive->{interface} eq 'usb') {
1576         die "implement me";
1577         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1578     } else {
1579         die "unsupported interface type";
1580     }
1581
1582     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1583
1584     if (my $serial = $drive->{serial}) {
1585         $serial = URI::Escape::uri_unescape($serial);
1586         $device .= ",serial=$serial";
1587     }
1588
1589
1590     return $device;
1591 }
1592
1593 sub get_initiator_name {
1594     my $initiator;
1595
1596     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1597     while (defined(my $line = <$fh>)) {
1598         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1599         $initiator = $1;
1600         last;
1601     }
1602     $fh->close();
1603
1604     return $initiator;
1605 }
1606
1607 my sub storage_allows_io_uring_default {
1608     my ($scfg, $cache_direct) = @_;
1609
1610     # io_uring with cache mode writeback or writethrough on krbd will hang...
1611     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1612
1613     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1614     # sometimes, just plain disable...
1615     return if $scfg && $scfg->{type} eq 'lvm';
1616
1617     # io_uring causes problems when used with CIFS since kernel 5.15
1618     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1619     return if $scfg && $scfg->{type} eq 'cifs';
1620
1621     return 1;
1622 }
1623
1624 my sub drive_uses_cache_direct {
1625     my ($drive, $scfg) = @_;
1626
1627     my $cache_direct = 0;
1628
1629     if (my $cache = $drive->{cache}) {
1630         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1631     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1632         $cache_direct = 1;
1633     }
1634
1635     return $cache_direct;
1636 }
1637
1638 sub print_drive_commandline_full {
1639     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1640
1641     my $path;
1642     my $volid = $drive->{file};
1643     my $format = $drive->{format};
1644     my $drive_id = get_drive_id($drive);
1645
1646     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1647     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1648
1649     if (drive_is_cdrom($drive)) {
1650         $path = get_iso_path($storecfg, $vmid, $volid);
1651         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1652     } else {
1653         if ($storeid) {
1654             $path = PVE::Storage::path($storecfg, $volid);
1655             $format //= qemu_img_format($scfg, $volname);
1656         } else {
1657             $path = $volid;
1658             $format //= "raw";
1659         }
1660    }
1661
1662    my $is_rbd = $path =~ m/^rbd:/;
1663
1664     my $opts = '';
1665     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1666     foreach my $o (@qemu_drive_options) {
1667         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1668     }
1669
1670     # snapshot only accepts on|off
1671     if (defined($drive->{snapshot})) {
1672         my $v = $drive->{snapshot} ? 'on' : 'off';
1673         $opts .= ",snapshot=$v";
1674     }
1675
1676     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1677         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1678     }
1679
1680     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1681         my ($dir, $qmpname) = @$type;
1682         if (my $v = $drive->{"mbps$dir"}) {
1683             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1684         }
1685         if (my $v = $drive->{"mbps${dir}_max"}) {
1686             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1687         }
1688         if (my $v = $drive->{"bps${dir}_max_length"}) {
1689             $opts .= ",throttling.bps$qmpname-max-length=$v";
1690         }
1691         if (my $v = $drive->{"iops${dir}"}) {
1692             $opts .= ",throttling.iops$qmpname=$v";
1693         }
1694         if (my $v = $drive->{"iops${dir}_max"}) {
1695             $opts .= ",throttling.iops$qmpname-max=$v";
1696         }
1697         if (my $v = $drive->{"iops${dir}_max_length"}) {
1698             $opts .= ",throttling.iops$qmpname-max-length=$v";
1699         }
1700     }
1701
1702     if ($pbs_name) {
1703         $format = "rbd" if $is_rbd;
1704         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1705             if !$format;
1706         $opts .= ",format=alloc-track,file.driver=$format";
1707     } elsif ($format) {
1708         $opts .= ",format=$format";
1709     }
1710
1711     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1712
1713     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1714
1715     if (!$drive->{aio}) {
1716         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1717             # io_uring supports all cache modes
1718             $opts .= ",aio=io_uring";
1719         } else {
1720             # aio native works only with O_DIRECT
1721             if($cache_direct) {
1722                 $opts .= ",aio=native";
1723             } else {
1724                 $opts .= ",aio=threads";
1725             }
1726         }
1727     }
1728
1729     if (!drive_is_cdrom($drive)) {
1730         my $detectzeroes;
1731         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1732             $detectzeroes = 'off';
1733         } elsif ($drive->{discard}) {
1734             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1735         } else {
1736             # This used to be our default with discard not being specified:
1737             $detectzeroes = 'on';
1738         }
1739
1740         # note: 'detect-zeroes' works per blockdev and we want it to persist
1741         # after the alloc-track is removed, so put it on 'file' directly
1742         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1743         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1744     }
1745
1746     if ($pbs_name) {
1747         $opts .= ",backing=$pbs_name";
1748         $opts .= ",auto-remove=on";
1749     }
1750
1751     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1752     my $file_param = "file";
1753     if ($pbs_name) {
1754         # non-rbd drivers require the underlying file to be a seperate block
1755         # node, so add a second .file indirection
1756         $file_param .= ".file" if !$is_rbd;
1757         $file_param .= ".filename";
1758     }
1759     my $pathinfo = $path ? "$file_param=$path," : '';
1760
1761     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1762 }
1763
1764 sub print_pbs_blockdev {
1765     my ($pbs_conf, $pbs_name) = @_;
1766     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1767     $blockdev .= ",repository=$pbs_conf->{repository}";
1768     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1769     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1770     $blockdev .= ",archive=$pbs_conf->{archive}";
1771     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1772     return $blockdev;
1773 }
1774
1775 sub print_netdevice_full {
1776     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1777
1778     my $device = $net->{model};
1779     if ($net->{model} eq 'virtio') {
1780          $device = 'virtio-net-pci';
1781      };
1782
1783     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1784     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1785     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1786         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1787         # and out of each queue plus one config interrupt and control vector queue
1788         my $vectors = $net->{queues} * 2 + 2;
1789         $tmpstr .= ",vectors=$vectors,mq=on";
1790         if (min_version($machine_version, 7, 1)) {
1791             $tmpstr .= ",packed=on";
1792         }
1793     }
1794
1795     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1796         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1797     }
1798
1799     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1800
1801     if (my $mtu = $net->{mtu}) {
1802         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1803             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1804             if ($mtu == 1) {
1805                  $mtu = $bridge_mtu;
1806             } elsif ($mtu < 576) {
1807                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1808             } elsif ($mtu > $bridge_mtu) {
1809                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1810             }
1811             $tmpstr .= ",host_mtu=$mtu";
1812         } else {
1813             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1814         }
1815     }
1816
1817     if ($use_old_bios_files) {
1818         my $romfile;
1819         if ($device eq 'virtio-net-pci') {
1820             $romfile = 'pxe-virtio.rom';
1821         } elsif ($device eq 'e1000') {
1822             $romfile = 'pxe-e1000.rom';
1823         } elsif ($device eq 'e1000e') {
1824             $romfile = 'pxe-e1000e.rom';
1825         } elsif ($device eq 'ne2k') {
1826             $romfile = 'pxe-ne2k_pci.rom';
1827         } elsif ($device eq 'pcnet') {
1828             $romfile = 'pxe-pcnet.rom';
1829         } elsif ($device eq 'rtl8139') {
1830             $romfile = 'pxe-rtl8139.rom';
1831         }
1832         $tmpstr .= ",romfile=$romfile" if $romfile;
1833     }
1834
1835     return $tmpstr;
1836 }
1837
1838 sub print_netdev_full {
1839     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1840
1841     my $i = '';
1842     if ($netid =~ m/^net(\d+)$/) {
1843         $i = int($1);
1844     }
1845
1846     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1847
1848     my $ifname = "tap${vmid}i$i";
1849
1850     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1851     die "interface name '$ifname' is too long (max 15 character)\n"
1852         if length($ifname) >= 16;
1853
1854     my $vhostparam = '';
1855     if (is_native($arch)) {
1856         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1857     }
1858
1859     my $vmname = $conf->{name} || "vm$vmid";
1860
1861     my $netdev = "";
1862     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1863
1864     if ($net->{bridge}) {
1865         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1866             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1867     } else {
1868         $netdev = "type=user,id=$netid,hostname=$vmname";
1869     }
1870
1871     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1872
1873     return $netdev;
1874 }
1875
1876 my $vga_map = {
1877     'cirrus' => 'cirrus-vga',
1878     'std' => 'VGA',
1879     'vmware' => 'vmware-svga',
1880     'virtio' => 'virtio-vga',
1881     'virtio-gl' => 'virtio-vga-gl',
1882 };
1883
1884 sub print_vga_device {
1885     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1886
1887     my $type = $vga_map->{$vga->{type}};
1888     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1889         $type = 'virtio-gpu';
1890     }
1891     my $vgamem_mb = $vga->{memory};
1892
1893     my $max_outputs = '';
1894     if ($qxlnum) {
1895         $type = $id ? 'qxl' : 'qxl-vga';
1896
1897         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1898             # set max outputs so linux can have up to 4 qxl displays with one device
1899             if (min_version($machine_version, 4, 1)) {
1900                 $max_outputs = ",max_outputs=4";
1901             }
1902         }
1903     }
1904
1905     die "no devicetype for $vga->{type}\n" if !$type;
1906
1907     my $memory = "";
1908     if ($vgamem_mb) {
1909         if ($vga->{type} =~ /^virtio/) {
1910             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1911             $memory = ",max_hostmem=$bytes";
1912         } elsif ($qxlnum) {
1913             # from https://www.spice-space.org/multiple-monitors.html
1914             $memory = ",vgamem_mb=$vga->{memory}";
1915             my $ram = $vgamem_mb * 4;
1916             my $vram = $vgamem_mb * 2;
1917             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1918         } else {
1919             $memory = ",vgamem_mb=$vga->{memory}";
1920         }
1921     } elsif ($qxlnum && $id) {
1922         $memory = ",ram_size=67108864,vram_size=33554432";
1923     }
1924
1925     my $edidoff = "";
1926     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1927         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1928     }
1929
1930     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1931     my $vgaid = "vga" . ($id // '');
1932     my $pciaddr;
1933     if ($q35 && $vgaid eq 'vga') {
1934         # the first display uses pcie.0 bus on q35 machines
1935         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1936     } else {
1937         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1938     }
1939
1940     if ($vga->{type} eq 'virtio-gl') {
1941         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1942         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1943             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1944
1945         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1946             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1947     }
1948
1949     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1950 }
1951
1952 sub parse_number_sets {
1953     my ($set) = @_;
1954     my $res = [];
1955     foreach my $part (split(/;/, $set)) {
1956         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1957             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1958             push @$res, [ $1, $2 ];
1959         } else {
1960             die "invalid range: $part\n";
1961         }
1962     }
1963     return $res;
1964 }
1965
1966 sub parse_numa {
1967     my ($data) = @_;
1968
1969     my $res = parse_property_string($numa_fmt, $data);
1970     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1971     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1972     return $res;
1973 }
1974
1975 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1976 sub parse_net {
1977     my ($data, $disable_mac_autogen) = @_;
1978
1979     my $res = eval { parse_property_string($net_fmt, $data) };
1980     if ($@) {
1981         warn $@;
1982         return;
1983     }
1984     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1985         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1986         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1987     }
1988     return $res;
1989 }
1990
1991 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1992 sub parse_ipconfig {
1993     my ($data) = @_;
1994
1995     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1996     if ($@) {
1997         warn $@;
1998         return;
1999     }
2000
2001     if ($res->{gw} && !$res->{ip}) {
2002         warn 'gateway specified without specifying an IP address';
2003         return;
2004     }
2005     if ($res->{gw6} && !$res->{ip6}) {
2006         warn 'IPv6 gateway specified without specifying an IPv6 address';
2007         return;
2008     }
2009     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2010         warn 'gateway specified together with DHCP';
2011         return;
2012     }
2013     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2014         # gw6 + auto/dhcp
2015         warn "IPv6 gateway specified together with $res->{ip6} address";
2016         return;
2017     }
2018
2019     if (!$res->{ip} && !$res->{ip6}) {
2020         return { ip => 'dhcp', ip6 => 'dhcp' };
2021     }
2022
2023     return $res;
2024 }
2025
2026 sub print_net {
2027     my $net = shift;
2028
2029     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2030 }
2031
2032 sub add_random_macs {
2033     my ($settings) = @_;
2034
2035     foreach my $opt (keys %$settings) {
2036         next if $opt !~ m/^net(\d+)$/;
2037         my $net = parse_net($settings->{$opt});
2038         next if !$net;
2039         $settings->{$opt} = print_net($net);
2040     }
2041 }
2042
2043 sub vm_is_volid_owner {
2044     my ($storecfg, $vmid, $volid) = @_;
2045
2046     if ($volid !~  m|^/|) {
2047         my ($path, $owner);
2048         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2049         if ($owner && ($owner == $vmid)) {
2050             return 1;
2051         }
2052     }
2053
2054     return;
2055 }
2056
2057 sub vmconfig_register_unused_drive {
2058     my ($storecfg, $vmid, $conf, $drive) = @_;
2059
2060     if (drive_is_cloudinit($drive)) {
2061         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2062         warn $@ if $@;
2063         delete $conf->{cloudinit};
2064     } elsif (!drive_is_cdrom($drive)) {
2065         my $volid = $drive->{file};
2066         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2067             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2068         }
2069     }
2070 }
2071
2072 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2073 my $smbios1_fmt = {
2074     uuid => {
2075         type => 'string',
2076         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2077         format_description => 'UUID',
2078         description => "Set SMBIOS1 UUID.",
2079         optional => 1,
2080     },
2081     version => {
2082         type => 'string',
2083         pattern => '[A-Za-z0-9+\/]+={0,2}',
2084         format_description => 'Base64 encoded string',
2085         description => "Set SMBIOS1 version.",
2086         optional => 1,
2087     },
2088     serial => {
2089         type => 'string',
2090         pattern => '[A-Za-z0-9+\/]+={0,2}',
2091         format_description => 'Base64 encoded string',
2092         description => "Set SMBIOS1 serial number.",
2093         optional => 1,
2094     },
2095     manufacturer => {
2096         type => 'string',
2097         pattern => '[A-Za-z0-9+\/]+={0,2}',
2098         format_description => 'Base64 encoded string',
2099         description => "Set SMBIOS1 manufacturer.",
2100         optional => 1,
2101     },
2102     product => {
2103         type => 'string',
2104         pattern => '[A-Za-z0-9+\/]+={0,2}',
2105         format_description => 'Base64 encoded string',
2106         description => "Set SMBIOS1 product ID.",
2107         optional => 1,
2108     },
2109     sku => {
2110         type => 'string',
2111         pattern => '[A-Za-z0-9+\/]+={0,2}',
2112         format_description => 'Base64 encoded string',
2113         description => "Set SMBIOS1 SKU string.",
2114         optional => 1,
2115     },
2116     family => {
2117         type => 'string',
2118         pattern => '[A-Za-z0-9+\/]+={0,2}',
2119         format_description => 'Base64 encoded string',
2120         description => "Set SMBIOS1 family string.",
2121         optional => 1,
2122     },
2123     base64 => {
2124         type => 'boolean',
2125         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2126         optional => 1,
2127     },
2128 };
2129
2130 sub parse_smbios1 {
2131     my ($data) = @_;
2132
2133     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2134     warn $@ if $@;
2135     return $res;
2136 }
2137
2138 sub print_smbios1 {
2139     my ($smbios1) = @_;
2140     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2141 }
2142
2143 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2144
2145 sub parse_watchdog {
2146     my ($value) = @_;
2147
2148     return if !$value;
2149
2150     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2151     warn $@ if $@;
2152     return $res;
2153 }
2154
2155 sub parse_guest_agent {
2156     my ($conf) = @_;
2157
2158     return {} if !defined($conf->{agent});
2159
2160     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2161     warn $@ if $@;
2162
2163     # if the agent is disabled ignore the other potentially set properties
2164     return {} if !$res->{enabled};
2165     return $res;
2166 }
2167
2168 sub get_qga_key {
2169     my ($conf, $key) = @_;
2170     return undef if !defined($conf->{agent});
2171
2172     my $agent = parse_guest_agent($conf);
2173     return $agent->{$key};
2174 }
2175
2176 sub parse_vga {
2177     my ($value) = @_;
2178
2179     return {} if !$value;
2180     my $res = eval { parse_property_string($vga_fmt, $value) };
2181     warn $@ if $@;
2182     return $res;
2183 }
2184
2185 sub parse_rng {
2186     my ($value) = @_;
2187
2188     return if !$value;
2189
2190     my $res = eval { parse_property_string($rng_fmt, $value) };
2191     warn $@ if $@;
2192     return $res;
2193 }
2194
2195 sub parse_meta_info {
2196     my ($value) = @_;
2197
2198     return if !$value;
2199
2200     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2201     warn $@ if $@;
2202     return $res;
2203 }
2204
2205 sub new_meta_info_string {
2206     my () = @_; # for now do not allow to override any value
2207
2208     return PVE::JSONSchema::print_property_string(
2209         {
2210             'creation-qemu' => kvm_user_version(),
2211             ctime => "". int(time()),
2212         },
2213         $meta_info_fmt
2214     );
2215 }
2216
2217 sub qemu_created_version_fixups {
2218     my ($conf, $forcemachine, $kvmver) = @_;
2219
2220     my $meta = parse_meta_info($conf->{meta}) // {};
2221     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2222
2223     # check if we need to apply some handling for VMs that always use the latest machine version but
2224     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2225     # would be required (we do not want to pin machine version for non-windows OS type)
2226     if (
2227         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2228         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2229         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2230         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2231     ) {
2232         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2233         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2234             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2235             # and thus with the predictable interface naming of systemd
2236             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2237         }
2238     }
2239     return;
2240 }
2241
2242 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2243 sub verify_usb_device {
2244     my ($value, $noerr) = @_;
2245
2246     return $value if parse_usb_device($value);
2247
2248     return if $noerr;
2249
2250     die "unable to parse usb device\n";
2251 }
2252
2253 # add JSON properties for create and set function
2254 sub json_config_properties {
2255     my ($prop, $with_disk_alloc) = @_;
2256
2257     my $skip_json_config_opts = {
2258         parent => 1,
2259         snaptime => 1,
2260         vmstate => 1,
2261         runningmachine => 1,
2262         runningcpu => 1,
2263         meta => 1,
2264     };
2265
2266     foreach my $opt (keys %$confdesc) {
2267         next if $skip_json_config_opts->{$opt};
2268
2269         if ($with_disk_alloc && is_valid_drivename($opt)) {
2270             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2271         } else {
2272             $prop->{$opt} = $confdesc->{$opt};
2273         }
2274     }
2275
2276     return $prop;
2277 }
2278
2279 # Properties that we can read from an OVF file
2280 sub json_ovf_properties {
2281     my $prop = {};
2282
2283     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2284         $prop->{$device} = {
2285             type => 'string',
2286             format => 'pve-volume-id-or-absolute-path',
2287             description => "Disk image that gets imported to $device",
2288             optional => 1,
2289         };
2290     }
2291
2292     $prop->{cores} = {
2293         type => 'integer',
2294         description => "The number of CPU cores.",
2295         optional => 1,
2296     };
2297     $prop->{memory} = {
2298         type => 'integer',
2299         description => "Amount of RAM for the VM in MB.",
2300         optional => 1,
2301     };
2302     $prop->{name} = {
2303         type => 'string',
2304         description => "Name of the VM.",
2305         optional => 1,
2306     };
2307
2308     return $prop;
2309 }
2310
2311 # return copy of $confdesc_cloudinit to generate documentation
2312 sub cloudinit_config_properties {
2313
2314     return dclone($confdesc_cloudinit);
2315 }
2316
2317 sub cloudinit_pending_properties {
2318     my $p = {
2319         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2320         name => 1,
2321     };
2322     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2323     return $p;
2324 }
2325
2326 sub check_type {
2327     my ($key, $value) = @_;
2328
2329     die "unknown setting '$key'\n" if !$confdesc->{$key};
2330
2331     my $type = $confdesc->{$key}->{type};
2332
2333     if (!defined($value)) {
2334         die "got undefined value\n";
2335     }
2336
2337     if ($value =~ m/[\n\r]/) {
2338         die "property contains a line feed\n";
2339     }
2340
2341     if ($type eq 'boolean') {
2342         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2343         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2344         die "type check ('boolean') failed - got '$value'\n";
2345     } elsif ($type eq 'integer') {
2346         return int($1) if $value =~ m/^(\d+)$/;
2347         die "type check ('integer') failed - got '$value'\n";
2348     } elsif ($type eq 'number') {
2349         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2350         die "type check ('number') failed - got '$value'\n";
2351     } elsif ($type eq 'string') {
2352         if (my $fmt = $confdesc->{$key}->{format}) {
2353             PVE::JSONSchema::check_format($fmt, $value);
2354             return $value;
2355         }
2356         $value =~ s/^\"(.*)\"$/$1/;
2357         return $value;
2358     } else {
2359         die "internal error"
2360     }
2361 }
2362
2363 sub destroy_vm {
2364     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2365
2366     my $conf = PVE::QemuConfig->load_config($vmid);
2367
2368     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2369         PVE::QemuConfig->check_lock($conf);
2370     }
2371
2372     if ($conf->{template}) {
2373         # check if any base image is still used by a linked clone
2374         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2375                 my ($ds, $drive) = @_;
2376                 return if drive_is_cdrom($drive);
2377
2378                 my $volid = $drive->{file};
2379                 return if !$volid || $volid =~ m|^/|;
2380
2381                 die "base volume '$volid' is still in use by linked cloned\n"
2382                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2383
2384         });
2385     }
2386
2387     my $volids = {};
2388     my $remove_owned_drive = sub {
2389         my ($ds, $drive) = @_;
2390         return if drive_is_cdrom($drive, 1);
2391
2392         my $volid = $drive->{file};
2393         return if !$volid || $volid =~ m|^/|;
2394         return if $volids->{$volid};
2395
2396         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2397         return if !$path || !$owner || ($owner != $vmid);
2398
2399         $volids->{$volid} = 1;
2400         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2401         warn "Could not remove disk '$volid', check manually: $@" if $@;
2402     };
2403
2404     # only remove disks owned by this VM (referenced in the config)
2405     my $include_opts = {
2406         include_unused => 1,
2407         extra_keys => ['vmstate'],
2408     };
2409     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2410
2411     for my $snap (values %{$conf->{snapshots}}) {
2412         next if !defined($snap->{vmstate});
2413         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2414         next if !defined($drive);
2415         $remove_owned_drive->('vmstate', $drive);
2416     }
2417
2418     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2419
2420     if ($purge_unreferenced) { # also remove unreferenced disk
2421         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2422         PVE::Storage::foreach_volid($vmdisks, sub {
2423             my ($volid, $sid, $volname, $d) = @_;
2424             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2425             warn $@ if $@;
2426         });
2427     }
2428
2429     if (defined $replacement_conf) {
2430         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2431     } else {
2432         PVE::QemuConfig->destroy_config($vmid);
2433     }
2434 }
2435
2436 sub parse_vm_config {
2437     my ($filename, $raw, $strict) = @_;
2438
2439     return if !defined($raw);
2440
2441     my $res = {
2442         digest => Digest::SHA::sha1_hex($raw),
2443         snapshots => {},
2444         pending => {},
2445         cloudinit => {},
2446     };
2447
2448     my $handle_error = sub {
2449         my ($msg) = @_;
2450
2451         if ($strict) {
2452             die $msg;
2453         } else {
2454             warn $msg;
2455         }
2456     };
2457
2458     $filename =~ m|/qemu-server/(\d+)\.conf$|
2459         || die "got strange filename '$filename'";
2460
2461     my $vmid = $1;
2462
2463     my $conf = $res;
2464     my $descr;
2465     my $finish_description = sub {
2466         if (defined($descr)) {
2467             $descr =~ s/\s+$//;
2468             $conf->{description} = $descr;
2469         }
2470         $descr = undef;
2471     };
2472     my $section = '';
2473
2474     my @lines = split(/\n/, $raw);
2475     foreach my $line (@lines) {
2476         next if $line =~ m/^\s*$/;
2477
2478         if ($line =~ m/^\[PENDING\]\s*$/i) {
2479             $section = 'pending';
2480             $finish_description->();
2481             $conf = $res->{$section} = {};
2482             next;
2483         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2484             $section = 'cloudinit';
2485             $finish_description->();
2486             $conf = $res->{$section} = {};
2487             next;
2488
2489         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2490             $section = $1;
2491             $finish_description->();
2492             $conf = $res->{snapshots}->{$section} = {};
2493             next;
2494         }
2495
2496         if ($line =~ m/^\#(.*)$/) {
2497             $descr = '' if !defined($descr);
2498             $descr .= PVE::Tools::decode_text($1) . "\n";
2499             next;
2500         }
2501
2502         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2503             $descr = '' if !defined($descr);
2504             $descr .= PVE::Tools::decode_text($2);
2505         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2506             $conf->{snapstate} = $1;
2507         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2508             my $key = $1;
2509             my $value = $2;
2510             $conf->{$key} = $value;
2511         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2512             my $value = $1;
2513             if ($section eq 'pending') {
2514                 $conf->{delete} = $value; # we parse this later
2515             } else {
2516                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2517             }
2518         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2519             my $key = $1;
2520             my $value = $2;
2521             if ($section eq 'cloudinit') {
2522                 # ignore validation only used for informative purpose
2523                 $conf->{$key} = $value;
2524                 next;
2525             }
2526             eval { $value = check_type($key, $value); };
2527             if ($@) {
2528                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2529             } else {
2530                 $key = 'ide2' if $key eq 'cdrom';
2531                 my $fmt = $confdesc->{$key}->{format};
2532                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2533                     my $v = parse_drive($key, $value);
2534                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2535                         $v->{file} = $volid;
2536                         $value = print_drive($v);
2537                     } else {
2538                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2539                         next;
2540                     }
2541                 }
2542
2543                 $conf->{$key} = $value;
2544             }
2545         } else {
2546             $handle_error->("vm $vmid - unable to parse config: $line\n");
2547         }
2548     }
2549
2550     $finish_description->();
2551     delete $res->{snapstate}; # just to be sure
2552
2553     return $res;
2554 }
2555
2556 sub write_vm_config {
2557     my ($filename, $conf) = @_;
2558
2559     delete $conf->{snapstate}; # just to be sure
2560
2561     if ($conf->{cdrom}) {
2562         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2563         $conf->{ide2} = $conf->{cdrom};
2564         delete $conf->{cdrom};
2565     }
2566
2567     # we do not use 'smp' any longer
2568     if ($conf->{sockets}) {
2569         delete $conf->{smp};
2570     } elsif ($conf->{smp}) {
2571         $conf->{sockets} = $conf->{smp};
2572         delete $conf->{cores};
2573         delete $conf->{smp};
2574     }
2575
2576     my $used_volids = {};
2577
2578     my $cleanup_config = sub {
2579         my ($cref, $pending, $snapname) = @_;
2580
2581         foreach my $key (keys %$cref) {
2582             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2583                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2584             my $value = $cref->{$key};
2585             if ($key eq 'delete') {
2586                 die "propertry 'delete' is only allowed in [PENDING]\n"
2587                     if !$pending;
2588                 # fixme: check syntax?
2589                 next;
2590             }
2591             eval { $value = check_type($key, $value); };
2592             die "unable to parse value of '$key' - $@" if $@;
2593
2594             $cref->{$key} = $value;
2595
2596             if (!$snapname && is_valid_drivename($key)) {
2597                 my $drive = parse_drive($key, $value);
2598                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2599             }
2600         }
2601     };
2602
2603     &$cleanup_config($conf);
2604
2605     &$cleanup_config($conf->{pending}, 1);
2606
2607     foreach my $snapname (keys %{$conf->{snapshots}}) {
2608         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2609         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2610     }
2611
2612     # remove 'unusedX' settings if we re-add a volume
2613     foreach my $key (keys %$conf) {
2614         my $value = $conf->{$key};
2615         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2616             delete $conf->{$key};
2617         }
2618     }
2619
2620     my $generate_raw_config = sub {
2621         my ($conf, $pending) = @_;
2622
2623         my $raw = '';
2624
2625         # add description as comment to top of file
2626         if (defined(my $descr = $conf->{description})) {
2627             if ($descr) {
2628                 foreach my $cl (split(/\n/, $descr)) {
2629                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2630                 }
2631             } else {
2632                 $raw .= "#\n" if $pending;
2633             }
2634         }
2635
2636         foreach my $key (sort keys %$conf) {
2637             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2638             $raw .= "$key: $conf->{$key}\n";
2639         }
2640         return $raw;
2641     };
2642
2643     my $raw = &$generate_raw_config($conf);
2644
2645     if (scalar(keys %{$conf->{pending}})){
2646         $raw .= "\n[PENDING]\n";
2647         $raw .= &$generate_raw_config($conf->{pending}, 1);
2648     }
2649
2650     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2651         $raw .= "\n[special:cloudinit]\n";
2652         $raw .= &$generate_raw_config($conf->{cloudinit});
2653     }
2654
2655     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2656         $raw .= "\n[$snapname]\n";
2657         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2658     }
2659
2660     return $raw;
2661 }
2662
2663 sub load_defaults {
2664
2665     my $res = {};
2666
2667     # we use static defaults from our JSON schema configuration
2668     foreach my $key (keys %$confdesc) {
2669         if (defined(my $default = $confdesc->{$key}->{default})) {
2670             $res->{$key} = $default;
2671         }
2672     }
2673
2674     return $res;
2675 }
2676
2677 sub config_list {
2678     my $vmlist = PVE::Cluster::get_vmlist();
2679     my $res = {};
2680     return $res if !$vmlist || !$vmlist->{ids};
2681     my $ids = $vmlist->{ids};
2682     my $nodename = nodename();
2683
2684     foreach my $vmid (keys %$ids) {
2685         my $d = $ids->{$vmid};
2686         next if !$d->{node} || $d->{node} ne $nodename;
2687         next if !$d->{type} || $d->{type} ne 'qemu';
2688         $res->{$vmid}->{exists} = 1;
2689     }
2690     return $res;
2691 }
2692
2693 # test if VM uses local resources (to prevent migration)
2694 sub check_local_resources {
2695     my ($conf, $noerr) = @_;
2696
2697     my @loc_res = ();
2698
2699     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2700     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2701
2702     push @loc_res, "ivshmem" if $conf->{ivshmem};
2703
2704     foreach my $k (keys %$conf) {
2705         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2706         # sockets are safe: they will recreated be on the target side post-migrate
2707         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2708         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2709     }
2710
2711     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2712
2713     return \@loc_res;
2714 }
2715
2716 # check if used storages are available on all nodes (use by migrate)
2717 sub check_storage_availability {
2718     my ($storecfg, $conf, $node) = @_;
2719
2720     PVE::QemuConfig->foreach_volume($conf, sub {
2721         my ($ds, $drive) = @_;
2722
2723         my $volid = $drive->{file};
2724         return if !$volid;
2725
2726         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2727         return if !$sid;
2728
2729         # check if storage is available on both nodes
2730         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2731         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2732
2733         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2734
2735         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2736             if !$scfg->{content}->{$vtype};
2737    });
2738 }
2739
2740 # list nodes where all VM images are available (used by has_feature API)
2741 sub shared_nodes {
2742     my ($conf, $storecfg) = @_;
2743
2744     my $nodelist = PVE::Cluster::get_nodelist();
2745     my $nodehash = { map { $_ => 1 } @$nodelist };
2746     my $nodename = nodename();
2747
2748     PVE::QemuConfig->foreach_volume($conf, sub {
2749         my ($ds, $drive) = @_;
2750
2751         my $volid = $drive->{file};
2752         return if !$volid;
2753
2754         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2755         if ($storeid) {
2756             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2757             if ($scfg->{disable}) {
2758                 $nodehash = {};
2759             } elsif (my $avail = $scfg->{nodes}) {
2760                 foreach my $node (keys %$nodehash) {
2761                     delete $nodehash->{$node} if !$avail->{$node};
2762                 }
2763             } elsif (!$scfg->{shared}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if $node ne $nodename
2766                 }
2767             }
2768         }
2769     });
2770
2771     return $nodehash
2772 }
2773
2774 sub check_local_storage_availability {
2775     my ($conf, $storecfg) = @_;
2776
2777     my $nodelist = PVE::Cluster::get_nodelist();
2778     my $nodehash = { map { $_ => {} } @$nodelist };
2779
2780     PVE::QemuConfig->foreach_volume($conf, sub {
2781         my ($ds, $drive) = @_;
2782
2783         my $volid = $drive->{file};
2784         return if !$volid;
2785
2786         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2787         if ($storeid) {
2788             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2789
2790             if ($scfg->{disable}) {
2791                 foreach my $node (keys %$nodehash) {
2792                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2793                 }
2794             } elsif (my $avail = $scfg->{nodes}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     if (!$avail->{$node}) {
2797                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2798                     }
2799                 }
2800             }
2801         }
2802     });
2803
2804     foreach my $node (values %$nodehash) {
2805         if (my $unavail = $node->{unavailable_storages}) {
2806             $node->{unavailable_storages} = [ sort keys %$unavail ];
2807         }
2808     }
2809
2810     return $nodehash
2811 }
2812
2813 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2814 sub check_running {
2815     my ($vmid, $nocheck, $node) = @_;
2816
2817     # $nocheck is set when called during a migration, in which case the config
2818     # file might still or already reside on the *other* node
2819     # - because rename has already happened, and current node is source
2820     # - because rename hasn't happened yet, and current node is target
2821     # - because rename has happened, current node is target, but hasn't yet
2822     # processed it yet
2823     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2824     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2825 }
2826
2827 sub vzlist {
2828
2829     my $vzlist = config_list();
2830
2831     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2832
2833     while (defined(my $de = $fd->read)) {
2834         next if $de !~ m/^(\d+)\.pid$/;
2835         my $vmid = $1;
2836         next if !defined($vzlist->{$vmid});
2837         if (my $pid = check_running($vmid)) {
2838             $vzlist->{$vmid}->{pid} = $pid;
2839         }
2840     }
2841
2842     return $vzlist;
2843 }
2844
2845 our $vmstatus_return_properties = {
2846     vmid => get_standard_option('pve-vmid'),
2847     status => {
2848         description => "QEMU process status.",
2849         type => 'string',
2850         enum => ['stopped', 'running'],
2851     },
2852     maxmem => {
2853         description => "Maximum memory in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     maxdisk => {
2859         description => "Root disk size in bytes.",
2860         type => 'integer',
2861         optional => 1,
2862         renderer => 'bytes',
2863     },
2864     name => {
2865         description => "VM name.",
2866         type => 'string',
2867         optional => 1,
2868     },
2869     qmpstatus => {
2870         description => "QEMU QMP agent status.",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     pid => {
2875         description => "PID of running qemu process.",
2876         type => 'integer',
2877         optional => 1,
2878     },
2879     uptime => {
2880         description => "Uptime.",
2881         type => 'integer',
2882         optional => 1,
2883         renderer => 'duration',
2884     },
2885     cpus => {
2886         description => "Maximum usable CPUs.",
2887         type => 'number',
2888         optional => 1,
2889     },
2890     lock => {
2891         description => "The current config lock, if any.",
2892         type => 'string',
2893         optional => 1,
2894     },
2895     tags => {
2896         description  => "The current configured tags, if any",
2897         type => 'string',
2898         optional => 1,
2899     },
2900     'running-machine' => {
2901         description => "The currently running machine type (if running).",
2902         type => 'string',
2903         optional => 1,
2904     },
2905     'running-qemu' => {
2906         description => "The currently running QEMU version (if running).",
2907         type => 'string',
2908         optional => 1,
2909     },
2910 };
2911
2912 my $last_proc_pid_stat;
2913
2914 # get VM status information
2915 # This must be fast and should not block ($full == false)
2916 # We only query KVM using QMP if $full == true (this can be slow)
2917 sub vmstatus {
2918     my ($opt_vmid, $full) = @_;
2919
2920     my $res = {};
2921
2922     my $storecfg = PVE::Storage::config();
2923
2924     my $list = vzlist();
2925     my $defaults = load_defaults();
2926
2927     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2928
2929     my $cpucount = $cpuinfo->{cpus} || 1;
2930
2931     foreach my $vmid (keys %$list) {
2932         next if $opt_vmid && ($vmid ne $opt_vmid);
2933
2934         my $conf = PVE::QemuConfig->load_config($vmid);
2935
2936         my $d = { vmid => int($vmid) };
2937         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2938
2939         # fixme: better status?
2940         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2941
2942         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2943         if (defined($size)) {
2944             $d->{disk} = 0; # no info available
2945             $d->{maxdisk} = $size;
2946         } else {
2947             $d->{disk} = 0;
2948             $d->{maxdisk} = 0;
2949         }
2950
2951         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2952             * ($conf->{cores} || $defaults->{cores});
2953         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2954         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2955
2956         $d->{name} = $conf->{name} || "VM $vmid";
2957         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2958             : $defaults->{memory}*(1024*1024);
2959
2960         if ($conf->{balloon}) {
2961             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2962             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2963                 : $defaults->{shares};
2964         }
2965
2966         $d->{uptime} = 0;
2967         $d->{cpu} = 0;
2968         $d->{mem} = 0;
2969
2970         $d->{netout} = 0;
2971         $d->{netin} = 0;
2972
2973         $d->{diskread} = 0;
2974         $d->{diskwrite} = 0;
2975
2976         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2977
2978         $d->{serial} = 1 if conf_has_serial($conf);
2979         $d->{lock} = $conf->{lock} if $conf->{lock};
2980         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2981
2982         $res->{$vmid} = $d;
2983     }
2984
2985     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2986     foreach my $dev (keys %$netdev) {
2987         next if $dev !~ m/^tap([1-9]\d*)i/;
2988         my $vmid = $1;
2989         my $d = $res->{$vmid};
2990         next if !$d;
2991
2992         $d->{netout} += $netdev->{$dev}->{receive};
2993         $d->{netin} += $netdev->{$dev}->{transmit};
2994
2995         if ($full) {
2996             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2997             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2998         }
2999
3000     }
3001
3002     my $ctime = gettimeofday;
3003
3004     foreach my $vmid (keys %$list) {
3005
3006         my $d = $res->{$vmid};
3007         my $pid = $d->{pid};
3008         next if !$pid;
3009
3010         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3011         next if !$pstat; # not running
3012
3013         my $used = $pstat->{utime} + $pstat->{stime};
3014
3015         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3016
3017         if ($pstat->{vsize}) {
3018             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3019         }
3020
3021         my $old = $last_proc_pid_stat->{$pid};
3022         if (!$old) {
3023             $last_proc_pid_stat->{$pid} = {
3024                 time => $ctime,
3025                 used => $used,
3026                 cpu => 0,
3027             };
3028             next;
3029         }
3030
3031         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3032
3033         if ($dtime > 1000) {
3034             my $dutime = $used -  $old->{used};
3035
3036             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3037             $last_proc_pid_stat->{$pid} = {
3038                 time => $ctime,
3039                 used => $used,
3040                 cpu => $d->{cpu},
3041             };
3042         } else {
3043             $d->{cpu} = $old->{cpu};
3044         }
3045     }
3046
3047     return $res if !$full;
3048
3049     my $qmpclient = PVE::QMPClient->new();
3050
3051     my $ballooncb = sub {
3052         my ($vmid, $resp) = @_;
3053
3054         my $info = $resp->{'return'};
3055         return if !$info->{max_mem};
3056
3057         my $d = $res->{$vmid};
3058
3059         # use memory assigned to VM
3060         $d->{maxmem} = $info->{max_mem};
3061         $d->{balloon} = $info->{actual};
3062
3063         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3064             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3065             $d->{freemem} = $info->{free_mem};
3066         }
3067
3068         $d->{ballooninfo} = $info;
3069     };
3070
3071     my $blockstatscb = sub {
3072         my ($vmid, $resp) = @_;
3073         my $data = $resp->{'return'} || [];
3074         my $totalrdbytes = 0;
3075         my $totalwrbytes = 0;
3076
3077         for my $blockstat (@$data) {
3078             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3079             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3080
3081             $blockstat->{device} =~ s/drive-//;
3082             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3083         }
3084         $res->{$vmid}->{diskread} = $totalrdbytes;
3085         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3086     };
3087
3088     my $machinecb = sub {
3089         my ($vmid, $resp) = @_;
3090         my $data = $resp->{'return'} || [];
3091
3092         $res->{$vmid}->{'running-machine'} =
3093             PVE::QemuServer::Machine::current_from_query_machines($data);
3094     };
3095
3096     my $versioncb = sub {
3097         my ($vmid, $resp) = @_;
3098         my $data = $resp->{'return'} // {};
3099         my $version = 'unknown';
3100
3101         if (my $v = $data->{qemu}) {
3102             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3103         }
3104
3105         $res->{$vmid}->{'running-qemu'} = $version;
3106     };
3107
3108     my $statuscb = sub {
3109         my ($vmid, $resp) = @_;
3110
3111         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3112         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3113         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3114         # this fails if ballon driver is not loaded, so this must be
3115         # the last commnand (following command are aborted if this fails).
3116         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3117
3118         my $status = 'unknown';
3119         if (!defined($status = $resp->{'return'}->{status})) {
3120             warn "unable to get VM status\n";
3121             return;
3122         }
3123
3124         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3125     };
3126
3127     foreach my $vmid (keys %$list) {
3128         next if $opt_vmid && ($vmid ne $opt_vmid);
3129         next if !$res->{$vmid}->{pid}; # not running
3130         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3131     }
3132
3133     $qmpclient->queue_execute(undef, 2);
3134
3135     foreach my $vmid (keys %$list) {
3136         next if $opt_vmid && ($vmid ne $opt_vmid);
3137         next if !$res->{$vmid}->{pid}; #not running
3138
3139         # we can't use the $qmpclient since it might have already aborted on
3140         # 'query-balloon', but this might also fail for older versions...
3141         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3142         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3143     }
3144
3145     foreach my $vmid (keys %$list) {
3146         next if $opt_vmid && ($vmid ne $opt_vmid);
3147         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3148     }
3149
3150     return $res;
3151 }
3152
3153 sub conf_has_serial {
3154     my ($conf) = @_;
3155
3156     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3157         if ($conf->{"serial$i"}) {
3158             return 1;
3159         }
3160     }
3161
3162     return 0;
3163 }
3164
3165 sub conf_has_audio {
3166     my ($conf, $id) = @_;
3167
3168     $id //= 0;
3169     my $audio = $conf->{"audio$id"};
3170     return if !defined($audio);
3171
3172     my $audioproperties = parse_property_string($audio_fmt, $audio);
3173     my $audiodriver = $audioproperties->{driver} // 'spice';
3174
3175     return {
3176         dev => $audioproperties->{device},
3177         dev_id => "audiodev$id",
3178         backend => $audiodriver,
3179         backend_id => "$audiodriver-backend${id}",
3180     };
3181 }
3182
3183 sub audio_devs {
3184     my ($audio, $audiopciaddr, $machine_version) = @_;
3185
3186     my $devs = [];
3187
3188     my $id = $audio->{dev_id};
3189     my $audiodev = "";
3190     if (min_version($machine_version, 4, 2)) {
3191         $audiodev = ",audiodev=$audio->{backend_id}";
3192     }
3193
3194     if ($audio->{dev} eq 'AC97') {
3195         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3196     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3197         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3198         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3199         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3200     } else {
3201         die "unkown audio device '$audio->{dev}', implement me!";
3202     }
3203
3204     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3205
3206     return $devs;
3207 }
3208
3209 sub get_tpm_paths {
3210     my ($vmid) = @_;
3211     return {
3212         socket => "/var/run/qemu-server/$vmid.swtpm",
3213         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3214     };
3215 }
3216
3217 sub add_tpm_device {
3218     my ($vmid, $devices, $conf) = @_;
3219
3220     return if !$conf->{tpmstate0};
3221
3222     my $paths = get_tpm_paths($vmid);
3223
3224     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3225     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3226     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3227 }
3228
3229 sub start_swtpm {
3230     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3231
3232     return if !$tpmdrive;
3233
3234     my $state;
3235     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3236     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3237     if ($storeid) {
3238         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3239     } else {
3240         $state = $tpm->{file};
3241     }
3242
3243     my $paths = get_tpm_paths($vmid);
3244
3245     # during migration, we will get state from remote
3246     #
3247     if (!$migration) {
3248         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3249         my $setup_cmd = [
3250             "swtpm_setup",
3251             "--tpmstate",
3252             "file://$state",
3253             "--createek",
3254             "--create-ek-cert",
3255             "--create-platform-cert",
3256             "--lock-nvram",
3257             "--config",
3258             "/etc/swtpm_setup.conf", # do not use XDG configs
3259             "--runas",
3260             "0", # force creation as root, error if not possible
3261             "--not-overwrite", # ignore existing state, do not modify
3262         ];
3263
3264         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3265         # TPM 2.0 supports ECC crypto, use if possible
3266         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3267
3268         run_command($setup_cmd, outfunc => sub {
3269             print "swtpm_setup: $1\n";
3270         });
3271     }
3272
3273     # Used to distinguish different invocations in the log.
3274     my $log_prefix = "[id=" . int(time()) . "] ";
3275
3276     my $emulator_cmd = [
3277         "swtpm",
3278         "socket",
3279         "--tpmstate",
3280         "backend-uri=file://$state,mode=0600",
3281         "--ctrl",
3282         "type=unixio,path=$paths->{socket},mode=0600",
3283         "--pid",
3284         "file=$paths->{pid}",
3285         "--terminate", # terminate on QEMU disconnect
3286         "--daemon",
3287         "--log",
3288         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3289     ];
3290     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3291     run_command($emulator_cmd, outfunc => sub { print $1; });
3292
3293     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3294     while (! -e $paths->{pid}) {
3295         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3296         usleep(50_000);
3297     }
3298
3299     # return untainted PID of swtpm daemon so it can be killed on error
3300     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3301     return $1;
3302 }
3303
3304 sub vga_conf_has_spice {
3305     my ($vga) = @_;
3306
3307     my $vgaconf = parse_vga($vga);
3308     my $vgatype = $vgaconf->{type};
3309     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3310
3311     return $1 || 1;
3312 }
3313
3314 sub is_native($) {
3315     my ($arch) = @_;
3316     return get_host_arch() eq $arch;
3317 }
3318
3319 sub get_vm_arch {
3320     my ($conf) = @_;
3321     return $conf->{arch} // get_host_arch();
3322 }
3323
3324 my $default_machines = {
3325     x86_64 => 'pc',
3326     aarch64 => 'virt',
3327 };
3328
3329 sub get_installed_machine_version {
3330     my ($kvmversion) = @_;
3331     $kvmversion = kvm_user_version() if !defined($kvmversion);
3332     $kvmversion =~ m/^(\d+\.\d+)/;
3333     return $1;
3334 }
3335
3336 sub windows_get_pinned_machine_version {
3337     my ($machine, $base_version, $kvmversion) = @_;
3338
3339     my $pin_version = $base_version;
3340     if (!defined($base_version) ||
3341         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3342     ) {
3343         $pin_version = get_installed_machine_version($kvmversion);
3344     }
3345     if (!$machine || $machine eq 'pc') {
3346         $machine = "pc-i440fx-$pin_version";
3347     } elsif ($machine eq 'q35') {
3348         $machine = "pc-q35-$pin_version";
3349     } elsif ($machine eq 'virt') {
3350         $machine = "virt-$pin_version";
3351     } else {
3352         warn "unknown machine type '$machine', not touching that!\n";
3353     }
3354
3355     return $machine;
3356 }
3357
3358 sub get_vm_machine {
3359     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3360
3361     my $machine = $forcemachine || $conf->{machine};
3362
3363     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3364         $kvmversion //= kvm_user_version();
3365         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3366         # layout which confuses windows quite a bit and may result in various regressions..
3367         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3368         if (windows_version($conf->{ostype})) {
3369             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3370         }
3371         $arch //= 'x86_64';
3372         $machine ||= $default_machines->{$arch};
3373         if ($add_pve_version) {
3374             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3375             $machine .= "+pve$pvever";
3376         }
3377     }
3378
3379     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3380         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3381         $machine = $1 if $is_pxe;
3382
3383         # for version-pinned machines that do not include a pve-version (e.g.
3384         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3385         $machine .= '+pve0';
3386
3387         $machine .= '.pxe' if $is_pxe;
3388     }
3389
3390     return $machine;
3391 }
3392
3393 sub get_ovmf_files($$$) {
3394     my ($arch, $efidisk, $smm) = @_;
3395
3396     my $types = $OVMF->{$arch}
3397         or die "no OVMF images known for architecture '$arch'\n";
3398
3399     my $type = 'default';
3400     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3401         $type = $smm ? "4m" : "4m-no-smm";
3402         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3403     }
3404
3405     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3406     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3407     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3408
3409     return ($ovmf_code, $ovmf_vars);
3410 }
3411
3412 my $Arch2Qemu = {
3413     aarch64 => '/usr/bin/qemu-system-aarch64',
3414     x86_64 => '/usr/bin/qemu-system-x86_64',
3415 };
3416 sub get_command_for_arch($) {
3417     my ($arch) = @_;
3418     return '/usr/bin/kvm' if is_native($arch);
3419
3420     my $cmd = $Arch2Qemu->{$arch}
3421         or die "don't know how to emulate architecture '$arch'\n";
3422     return $cmd;
3423 }
3424
3425 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3426 # to use in a QEMU command line (-cpu element), first array_intersect the result
3427 # of query_supported_ with query_understood_. This is necessary because:
3428 #
3429 # a) query_understood_ returns flags the host cannot use and
3430 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3431 #    flags, but CPU settings - with most of them being flags. Those settings
3432 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3433 #
3434 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3435 # expensive. If you need the value returned from this, you can get it much
3436 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3437 # $accel being 'kvm' or 'tcg'.
3438 #
3439 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3440 # changes, automatically populating pmxcfs.
3441 #
3442 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3443 # since kvm and tcg machines support different flags
3444 #
3445 sub query_supported_cpu_flags {
3446     my ($arch) = @_;
3447
3448     $arch //= get_host_arch();
3449     my $default_machine = $default_machines->{$arch};
3450
3451     my $flags = {};
3452
3453     # FIXME: Once this is merged, the code below should work for ARM as well:
3454     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3455     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3456         $arch eq "aarch64";
3457
3458     my $kvm_supported = defined(kvm_version());
3459     my $qemu_cmd = get_command_for_arch($arch);
3460     my $fakevmid = -1;
3461     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3462
3463     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3464     my $query_supported_run_qemu = sub {
3465         my ($kvm) = @_;
3466
3467         my $flags = {};
3468         my $cmd = [
3469             $qemu_cmd,
3470             '-machine', $default_machine,
3471             '-display', 'none',
3472             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3473             '-mon', 'chardev=qmp,mode=control',
3474             '-pidfile', $pidfile,
3475             '-S', '-daemonize'
3476         ];
3477
3478         if (!$kvm) {
3479             push @$cmd, '-accel', 'tcg';
3480         }
3481
3482         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3483         die "QEMU flag querying VM exited with code " . $rc if $rc;
3484
3485         eval {
3486             my $cmd_result = mon_cmd(
3487                 $fakevmid,
3488                 'query-cpu-model-expansion',
3489                 type => 'full',
3490                 model => { name => 'host' }
3491             );
3492
3493             my $props = $cmd_result->{model}->{props};
3494             foreach my $prop (keys %$props) {
3495                 next if $props->{$prop} ne '1';
3496                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3497                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3498                 # We only keep those with underscores, to match /proc/cpuinfo
3499                 $prop =~ s/\.|-/_/g;
3500                 $flags->{$prop} = 1;
3501             }
3502         };
3503         my $err = $@;
3504
3505         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3506         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3507
3508         die $err if $err;
3509
3510         return [ sort keys %$flags ];
3511     };
3512
3513     # We need to query QEMU twice, since KVM and TCG have different supported flags
3514     PVE::QemuConfig->lock_config($fakevmid, sub {
3515         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3516         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3517
3518         if ($kvm_supported) {
3519             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3520             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3521         }
3522     });
3523
3524     return $flags;
3525 }
3526
3527 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3528 my $understood_cpu_flag_dir = "/usr/share/kvm";
3529 sub query_understood_cpu_flags {
3530     my $arch = get_host_arch();
3531     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3532
3533     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3534         if ! -e $filepath;
3535
3536     my $raw = file_get_contents($filepath);
3537     $raw =~ s/^\s+|\s+$//g;
3538     my @flags = split(/\s+/, $raw);
3539
3540     return \@flags;
3541 }
3542
3543 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3544 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3545 my sub should_disable_smm {
3546     my ($conf, $vga) = @_;
3547
3548     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3549         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3550 }
3551
3552 my sub print_ovmf_drive_commandlines {
3553     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3554
3555     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3556
3557     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3558
3559     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3560     if ($d) {
3561         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3562         my ($path, $format) = $d->@{'file', 'format'};
3563         if ($storeid) {
3564             $path = PVE::Storage::path($storecfg, $d->{file});
3565             if (!defined($format)) {
3566                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3567                 $format = qemu_img_format($scfg, $volname);
3568             }
3569         } elsif (!defined($format)) {
3570             die "efidisk format must be specified\n";
3571         }
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         if ($path =~ m/^rbd:/) {
3574             $var_drive_str .= ',cache=writeback';
3575             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3576         }
3577         $var_drive_str .= ",format=$format,file=$path";
3578
3579         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3580         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3581     } else {
3582         log_warn("no efidisk configured! Using temporary efivars disk.");
3583         my $path = "/tmp/$vmid-ovmf.fd";
3584         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3585         $var_drive_str .= ",format=raw,file=$path";
3586         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3587     }
3588
3589     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3590 }
3591
3592 sub config_to_command {
3593     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3594         $pbs_backing) = @_;
3595
3596     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3597     my $devices = [];
3598     my $bridges = {};
3599     my $ostype = $conf->{ostype};
3600     my $winversion = windows_version($ostype);
3601     my $kvm = $conf->{kvm};
3602     my $nodename = nodename();
3603
3604     my $arch = get_vm_arch($conf);
3605     my $kvm_binary = get_command_for_arch($arch);
3606     my $kvmver = kvm_user_version($kvm_binary);
3607
3608     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3609         $kvmver //= "undefined";
3610         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3611     }
3612
3613     my $add_pve_version = min_version($kvmver, 4, 1);
3614
3615     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3616     my $machine_version = extract_version($machine_type, $kvmver);
3617     $kvm //= 1 if is_native($arch);
3618
3619     $machine_version =~ m/(\d+)\.(\d+)/;
3620     my ($machine_major, $machine_minor) = ($1, $2);
3621
3622     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3623         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3624     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3625         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3626             ." please upgrade node '$nodename'\n"
3627     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3628         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3629         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3630             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3631             ." node '$nodename'\n";
3632     }
3633
3634     # if a specific +pve version is required for a feature, use $version_guard
3635     # instead of min_version to allow machines to be run with the minimum
3636     # required version
3637     my $required_pve_version = 0;
3638     my $version_guard = sub {
3639         my ($major, $minor, $pve) = @_;
3640         return 0 if !min_version($machine_version, $major, $minor, $pve);
3641         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3642         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3643         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3644         return 1;
3645     };
3646
3647     if ($kvm && !defined kvm_version()) {
3648         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3649             ." or enable in BIOS.\n";
3650     }
3651
3652     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3653     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3654     my $use_old_bios_files = undef;
3655     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3656
3657     my $cmd = [];
3658     if ($conf->{affinity}) {
3659         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3660     }
3661
3662     push @$cmd, $kvm_binary;
3663
3664     push @$cmd, '-id', $vmid;
3665
3666     my $vmname = $conf->{name} || "vm$vmid";
3667
3668     push @$cmd, '-name', "$vmname,debug-threads=on";
3669
3670     push @$cmd, '-no-shutdown';
3671
3672     my $use_virtio = 0;
3673
3674     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3675     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3676     push @$cmd, '-mon', "chardev=qmp,mode=control";
3677
3678     if (min_version($machine_version, 2, 12)) {
3679         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3680         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3681     }
3682
3683     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3684
3685     push @$cmd, '-daemonize';
3686
3687     if ($conf->{smbios1}) {
3688         my $smbios_conf = parse_smbios1($conf->{smbios1});
3689         if ($smbios_conf->{base64}) {
3690             # Do not pass base64 flag to qemu
3691             delete $smbios_conf->{base64};
3692             my $smbios_string = "";
3693             foreach my $key (keys %$smbios_conf) {
3694                 my $value;
3695                 if ($key eq "uuid") {
3696                     $value = $smbios_conf->{uuid}
3697                 } else {
3698                     $value = decode_base64($smbios_conf->{$key});
3699                 }
3700                 # qemu accepts any binary data, only commas need escaping by double comma
3701                 $value =~ s/,/,,/g;
3702                 $smbios_string .= "," . $key . "=" . $value if $value;
3703             }
3704             push @$cmd, '-smbios', "type=1" . $smbios_string;
3705         } else {
3706             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3707         }
3708     }
3709
3710     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3711         my ($code_drive_str, $var_drive_str) =
3712             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3713         push $cmd->@*, '-drive', $code_drive_str;
3714         push $cmd->@*, '-drive', $var_drive_str;
3715     }
3716
3717     if ($q35) { # tell QEMU to load q35 config early
3718         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3719         if (min_version($machine_version, 4, 0)) {
3720             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3721         } else {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3723         }
3724     }
3725
3726     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3727         push @$cmd, $fixups->@*;
3728     }
3729
3730     if ($conf->{vmgenid}) {
3731         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3732     }
3733
3734     # add usb controllers
3735     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3736         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3737     push @$devices, @usbcontrollers if @usbcontrollers;
3738     my $vga = parse_vga($conf->{vga});
3739
3740     my $qxlnum = vga_conf_has_spice($conf->{vga});
3741     $vga->{type} = 'qxl' if $qxlnum;
3742
3743     if (!$vga->{type}) {
3744         if ($arch eq 'aarch64') {
3745             $vga->{type} = 'virtio';
3746         } elsif (min_version($machine_version, 2, 9)) {
3747             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3748         } else {
3749             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3750         }
3751     }
3752
3753     # enable absolute mouse coordinates (needed by vnc)
3754     my $tablet = $conf->{tablet};
3755     if (!defined($tablet)) {
3756         $tablet = $defaults->{tablet};
3757         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3758         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3759     }
3760
3761     if ($tablet) {
3762         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3763         my $kbd = print_keyboarddevice_full($conf, $arch);
3764         push @$devices, '-device', $kbd if defined($kbd);
3765     }
3766
3767     my $bootorder = device_bootorder($conf);
3768
3769     # host pci device passthrough
3770     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3771         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3772
3773     # usb devices
3774     my $usb_dev_features = {};
3775     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3776
3777     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3778         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3779     push @$devices, @usbdevices if @usbdevices;
3780
3781     # serial devices
3782     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3783         my $path = $conf->{"serial$i"} or next;
3784         if ($path eq 'socket') {
3785             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3786             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3787             # On aarch64, serial0 is the UART device. QEMU only allows
3788             # connecting UART devices via the '-serial' command line, as
3789             # the device has a fixed slot on the hardware...
3790             if ($arch eq 'aarch64' && $i == 0) {
3791                 push @$devices, '-serial', "chardev:serial$i";
3792             } else {
3793                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3794             }
3795         } else {
3796             die "no such serial device\n" if ! -c $path;
3797             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3798             push @$devices, '-device', "isa-serial,chardev=serial$i";
3799         }
3800     }
3801
3802     # parallel devices
3803     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3804         if (my $path = $conf->{"parallel$i"}) {
3805             die "no such parallel device\n" if ! -c $path;
3806             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3807             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3808             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3809         }
3810     }
3811
3812     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3813         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3814         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3815         push @$devices, @$audio_devs;
3816     }
3817
3818     add_tpm_device($vmid, $devices, $conf);
3819
3820     my $sockets = 1;
3821     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3822     $sockets = $conf->{sockets} if  $conf->{sockets};
3823
3824     my $cores = $conf->{cores} || 1;
3825
3826     my $maxcpus = $sockets * $cores;
3827
3828     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3829
3830     my $allowed_vcpus = $cpuinfo->{cpus};
3831
3832     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3833
3834     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3835         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3836         for (my $i = 2; $i <= $vcpus; $i++)  {
3837             my $cpustr = print_cpu_device($conf,$i);
3838             push @$cmd, '-device', $cpustr;
3839         }
3840
3841     } else {
3842
3843         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3844     }
3845     push @$cmd, '-nodefaults';
3846
3847     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3848
3849     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3850
3851     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3852
3853     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3854         push @$devices, '-device', print_vga_device(
3855             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3856
3857         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3858
3859         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3860         push @$cmd,  '-vnc', "unix:$socket,password=on";
3861     } else {
3862         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3863         push @$cmd, '-nographic';
3864     }
3865
3866     # time drift fix
3867     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3868     my $useLocaltime = $conf->{localtime};
3869
3870     if ($winversion >= 5) { # windows
3871         $useLocaltime = 1 if !defined($conf->{localtime});
3872
3873         # use time drift fix when acpi is enabled
3874         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3875             $tdf = 1 if !defined($conf->{tdf});
3876         }
3877     }
3878
3879     if ($winversion >= 6) {
3880         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3881         push @$cmd, '-no-hpet';
3882     }
3883
3884     push @$rtcFlags, 'driftfix=slew' if $tdf;
3885
3886     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3887         push @$rtcFlags, "base=$conf->{startdate}";
3888     } elsif ($useLocaltime) {
3889         push @$rtcFlags, 'base=localtime';
3890     }
3891
3892     if ($forcecpu) {
3893         push @$cmd, '-cpu', $forcecpu;
3894     } else {
3895         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3896     }
3897
3898     PVE::QemuServer::Memory::config(
3899         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3900
3901     push @$cmd, '-S' if $conf->{freeze};
3902
3903     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3904
3905     my $guest_agent = parse_guest_agent($conf);
3906
3907     if ($guest_agent->{enabled}) {
3908         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3909         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3910
3911         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3912             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3913             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3914             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3915         } elsif ($guest_agent->{type} eq 'isa') {
3916             push @$devices, '-device', "isa-serial,chardev=qga0";
3917         }
3918     }
3919
3920     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3921     if ($rng && $version_guard->(4, 1, 2)) {
3922         check_rng_source($rng->{source});
3923
3924         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3925         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3926         my $limiter_str = "";
3927         if ($max_bytes) {
3928             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3929         }
3930
3931         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3932         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3933         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3934     }
3935
3936     my $spice_port;
3937
3938     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3939         if ($qxlnum > 1) {
3940             if ($winversion){
3941                 for (my $i = 1; $i < $qxlnum; $i++){
3942                     push @$devices, '-device', print_vga_device(
3943                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3944                 }
3945             } else {
3946                 # assume other OS works like Linux
3947                 my ($ram, $vram) = ("134217728", "67108864");
3948                 if ($vga->{memory}) {
3949                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3950                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3951                 }
3952                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3953                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3954             }
3955         }
3956
3957         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3958
3959         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3960         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3961         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3962
3963         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3964         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3965         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3966
3967         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3968         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3969
3970         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3971         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3972         if ($spice_enhancement->{foldersharing}) {
3973             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3974             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3975         }
3976
3977         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3978         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3979             if $spice_enhancement->{videostreaming};
3980
3981         push @$devices, '-spice', "$spice_opts";
3982     }
3983
3984     # enable balloon by default, unless explicitly disabled
3985     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3986         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3987         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3988         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3989         push @$devices, '-device', $ballooncmd;
3990     }
3991
3992     if ($conf->{watchdog}) {
3993         my $wdopts = parse_watchdog($conf->{watchdog});
3994         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3995         my $watchdog = $wdopts->{model} || 'i6300esb';
3996         push @$devices, '-device', "$watchdog$pciaddr";
3997         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3998     }
3999
4000     my $vollist = [];
4001     my $scsicontroller = {};
4002     my $ahcicontroller = {};
4003     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4004
4005     # Add iscsi initiator name if available
4006     if (my $initiator = get_initiator_name()) {
4007         push @$devices, '-iscsi', "initiator-name=$initiator";
4008     }
4009
4010     PVE::QemuConfig->foreach_volume($conf, sub {
4011         my ($ds, $drive) = @_;
4012
4013         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4014             check_volume_storage_type($storecfg, $drive->{file});
4015             push @$vollist, $drive->{file};
4016         }
4017
4018         # ignore efidisk here, already added in bios/fw handling code above
4019         return if $drive->{interface} eq 'efidisk';
4020         # similar for TPM
4021         return if $drive->{interface} eq 'tpmstate';
4022
4023         $use_virtio = 1 if $ds =~ m/^virtio/;
4024
4025         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4026
4027         if ($drive->{interface} eq 'virtio'){
4028            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4029         }
4030
4031         if ($drive->{interface} eq 'scsi') {
4032
4033             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4034
4035             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4036                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4037
4038             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4039             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4040
4041             my $iothread = '';
4042             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4043                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4044                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4045             } elsif ($drive->{iothread}) {
4046                 log_warn(
4047                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4048                 );
4049             }
4050
4051             my $queues = '';
4052             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4053                 $queues = ",num_queues=$drive->{queues}";
4054             }
4055
4056             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4057                 if !$scsicontroller->{$controller};
4058             $scsicontroller->{$controller}=1;
4059         }
4060
4061         if ($drive->{interface} eq 'sata') {
4062             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4063             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4064             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4065                 if !$ahcicontroller->{$controller};
4066             $ahcicontroller->{$controller}=1;
4067         }
4068
4069         my $pbs_conf = $pbs_backing->{$ds};
4070         my $pbs_name = undef;
4071         if ($pbs_conf) {
4072             $pbs_name = "drive-$ds-pbs";
4073             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4074         }
4075
4076         my $drive_cmd = print_drive_commandline_full(
4077             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4078
4079         # extra protection for templates, but SATA and IDE don't support it..
4080         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4081
4082         push @$devices, '-drive',$drive_cmd;
4083         push @$devices, '-device', print_drivedevice_full(
4084             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4085     });
4086
4087     for (my $i = 0; $i < $MAX_NETS; $i++) {
4088         my $netname = "net$i";
4089
4090         next if !$conf->{$netname};
4091         my $d = parse_net($conf->{$netname});
4092         next if !$d;
4093         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4094
4095         $use_virtio = 1 if $d->{model} eq 'virtio';
4096
4097         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4098
4099         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4100         push @$devices, '-netdev', $netdevfull;
4101
4102         my $netdevicefull = print_netdevice_full(
4103             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4104
4105         push @$devices, '-device', $netdevicefull;
4106     }
4107
4108     if ($conf->{ivshmem}) {
4109         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4110
4111         my $bus;
4112         if ($q35) {
4113             $bus = print_pcie_addr("ivshmem");
4114         } else {
4115             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4116         }
4117
4118         my $ivshmem_name = $ivshmem->{name} // $vmid;
4119         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4120
4121         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4122         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4123             .",size=$ivshmem->{size}M";
4124     }
4125
4126     # pci.4 is nested in pci.1
4127     $bridges->{1} = 1 if $bridges->{4};
4128
4129     if (!$q35) { # add pci bridges
4130         if (min_version($machine_version, 2, 3)) {
4131            $bridges->{1} = 1;
4132            $bridges->{2} = 1;
4133         }
4134         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4135     }
4136
4137     for my $k (sort {$b cmp $a} keys %$bridges) {
4138         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4139
4140         my $k_name = $k;
4141         if ($k == 2 && $legacy_igd) {
4142             $k_name = "$k-igd";
4143         }
4144         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4145         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4146
4147         if ($q35) { # add after -readconfig pve-q35.cfg
4148             splice @$devices, 2, 0, '-device', $devstr;
4149         } else {
4150             unshift @$devices, '-device', $devstr if $k > 0;
4151         }
4152     }
4153
4154     if (!$kvm) {
4155         push @$machineFlags, 'accel=tcg';
4156     }
4157
4158     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4159
4160     my $machine_type_min = $machine_type;
4161     if ($add_pve_version) {
4162         $machine_type_min =~ s/\+pve\d+$//;
4163         $machine_type_min .= "+pve$required_pve_version";
4164     }
4165     push @$machineFlags, "type=${machine_type_min}";
4166
4167     push @$cmd, @$devices;
4168     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4169     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4170     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4171
4172     if (my $vmstate = $conf->{vmstate}) {
4173         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4174         push @$vollist, $vmstate;
4175         push @$cmd, '-loadstate', $statepath;
4176         print "activating and using '$vmstate' as vmstate\n";
4177     }
4178
4179     if (PVE::QemuConfig->is_template($conf)) {
4180         # needed to workaround base volumes being read-only
4181         push @$cmd, '-snapshot';
4182     }
4183
4184     # add custom args
4185     if ($conf->{args}) {
4186         my $aa = PVE::Tools::split_args($conf->{args});
4187         push @$cmd, @$aa;
4188     }
4189
4190     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4191 }
4192
4193 sub check_rng_source {
4194     my ($source) = @_;
4195
4196     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4197     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4198         if ! -e $source;
4199
4200     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4201     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4202         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4203         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4204         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4205             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4206             ." to the host.\n";
4207     }
4208 }
4209
4210 sub spice_port {
4211     my ($vmid) = @_;
4212
4213     my $res = mon_cmd($vmid, 'query-spice');
4214
4215     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4216 }
4217
4218 sub vm_devices_list {
4219     my ($vmid) = @_;
4220
4221     my $res = mon_cmd($vmid, 'query-pci');
4222     my $devices_to_check = [];
4223     my $devices = {};
4224     foreach my $pcibus (@$res) {
4225         push @$devices_to_check, @{$pcibus->{devices}},
4226     }
4227
4228     while (@$devices_to_check) {
4229         my $to_check = [];
4230         for my $d (@$devices_to_check) {
4231             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4232             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4233
4234             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4235             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4236         }
4237         $devices_to_check = $to_check;
4238     }
4239
4240     my $resblock = mon_cmd($vmid, 'query-block');
4241     foreach my $block (@$resblock) {
4242         if($block->{device} =~ m/^drive-(\S+)/){
4243                 $devices->{$1} = 1;
4244         }
4245     }
4246
4247     my $resmice = mon_cmd($vmid, 'query-mice');
4248     foreach my $mice (@$resmice) {
4249         if ($mice->{name} eq 'QEMU HID Tablet') {
4250             $devices->{tablet} = 1;
4251             last;
4252         }
4253     }
4254
4255     # for usb devices there is no query-usb
4256     # but we can iterate over the entries in
4257     # qom-list path=/machine/peripheral
4258     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4259     foreach my $per (@$resperipheral) {
4260         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4261             $devices->{$per->{name}} = 1;
4262         }
4263     }
4264
4265     return $devices;
4266 }
4267
4268 sub vm_deviceplug {
4269     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4270
4271     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4272
4273     my $devices_list = vm_devices_list($vmid);
4274     return 1 if defined($devices_list->{$deviceid});
4275
4276     # add PCI bridge if we need it for the device
4277     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4278
4279     if ($deviceid eq 'tablet') {
4280         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4281     } elsif ($deviceid eq 'keyboard') {
4282         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4283     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4284         my $id = $1;
4285         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4286         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4287     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4288         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4289     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4290         qemu_iothread_add($vmid, $deviceid, $device);
4291
4292         qemu_driveadd($storecfg, $vmid, $device);
4293         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4294
4295         qemu_deviceadd($vmid, $devicefull);
4296         eval { qemu_deviceaddverify($vmid, $deviceid); };
4297         if (my $err = $@) {
4298             eval { qemu_drivedel($vmid, $deviceid); };
4299             warn $@ if $@;
4300             die $err;
4301         }
4302     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4303         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4304         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4305         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4306
4307         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4308
4309         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4310             qemu_iothread_add($vmid, $deviceid, $device);
4311             $devicefull .= ",iothread=iothread-$deviceid";
4312         }
4313
4314         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4315             $devicefull .= ",num_queues=$device->{queues}";
4316         }
4317
4318         qemu_deviceadd($vmid, $devicefull);
4319         qemu_deviceaddverify($vmid, $deviceid);
4320     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4321         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4322         qemu_driveadd($storecfg, $vmid, $device);
4323
4324         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4325         eval { qemu_deviceadd($vmid, $devicefull); };
4326         if (my $err = $@) {
4327             eval { qemu_drivedel($vmid, $deviceid); };
4328             warn $@ if $@;
4329             die $err;
4330         }
4331     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4332         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4333
4334         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4335         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4336         my $use_old_bios_files = undef;
4337         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4338
4339         my $netdevicefull = print_netdevice_full(
4340             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4341         qemu_deviceadd($vmid, $netdevicefull);
4342         eval {
4343             qemu_deviceaddverify($vmid, $deviceid);
4344             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4345         };
4346         if (my $err = $@) {
4347             eval { qemu_netdevdel($vmid, $deviceid); };
4348             warn $@ if $@;
4349             die $err;
4350         }
4351     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4352         my $bridgeid = $2;
4353         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4354         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4355
4356         qemu_deviceadd($vmid, $devicefull);
4357         qemu_deviceaddverify($vmid, $deviceid);
4358     } else {
4359         die "can't hotplug device '$deviceid'\n";
4360     }
4361
4362     return 1;
4363 }
4364
4365 # fixme: this should raise exceptions on error!
4366 sub vm_deviceunplug {
4367     my ($vmid, $conf, $deviceid) = @_;
4368
4369     my $devices_list = vm_devices_list($vmid);
4370     return 1 if !defined($devices_list->{$deviceid});
4371
4372     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4373     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4374
4375     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4376         qemu_devicedel($vmid, $deviceid);
4377     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4378         qemu_devicedel($vmid, $deviceid);
4379         qemu_devicedelverify($vmid, $deviceid);
4380     } elsif ($deviceid =~ m/^usb\d+$/) {
4381         qemu_devicedel($vmid, $deviceid);
4382         qemu_devicedelverify($vmid, $deviceid);
4383     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4384         my $device = parse_drive($deviceid, $conf->{$deviceid});
4385
4386         qemu_devicedel($vmid, $deviceid);
4387         qemu_devicedelverify($vmid, $deviceid);
4388         qemu_drivedel($vmid, $deviceid);
4389         qemu_iothread_del($vmid, $deviceid, $device);
4390     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4391         qemu_devicedel($vmid, $deviceid);
4392         qemu_devicedelverify($vmid, $deviceid);
4393     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4394         my $device = parse_drive($deviceid, $conf->{$deviceid});
4395
4396         qemu_devicedel($vmid, $deviceid);
4397         qemu_devicedelverify($vmid, $deviceid);
4398         qemu_drivedel($vmid, $deviceid);
4399         qemu_deletescsihw($conf, $vmid, $deviceid);
4400
4401         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4402             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4403     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406         qemu_netdevdel($vmid, $deviceid);
4407     } else {
4408         die "can't unplug device '$deviceid'\n";
4409     }
4410
4411     return 1;
4412 }
4413
4414 sub qemu_spice_usbredir_chardev_add {
4415     my ($vmid, $id) = @_;
4416
4417     mon_cmd($vmid, "chardev-add" , (
4418         id => $id,
4419         backend => {
4420             type => 'spicevmc',
4421             data => {
4422                 type => "usbredir",
4423             },
4424         },
4425     ));
4426 }
4427
4428 sub qemu_deviceadd {
4429     my ($vmid, $devicefull) = @_;
4430
4431     $devicefull = "driver=".$devicefull;
4432     my %options =  split(/[=,]/, $devicefull);
4433
4434     mon_cmd($vmid, "device_add" , %options);
4435 }
4436
4437 sub qemu_devicedel {
4438     my ($vmid, $deviceid) = @_;
4439
4440     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4441 }
4442
4443 sub qemu_iothread_add {
4444     my ($vmid, $deviceid, $device) = @_;
4445
4446     if ($device->{iothread}) {
4447         my $iothreads = vm_iothreads_list($vmid);
4448         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4449     }
4450 }
4451
4452 sub qemu_iothread_del {
4453     my ($vmid, $deviceid, $device) = @_;
4454
4455     if ($device->{iothread}) {
4456         my $iothreads = vm_iothreads_list($vmid);
4457         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4458     }
4459 }
4460
4461 sub qemu_objectadd {
4462     my ($vmid, $objectid, $qomtype) = @_;
4463
4464     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4465
4466     return 1;
4467 }
4468
4469 sub qemu_objectdel {
4470     my ($vmid, $objectid) = @_;
4471
4472     mon_cmd($vmid, "object-del", id => $objectid);
4473
4474     return 1;
4475 }
4476
4477 sub qemu_driveadd {
4478     my ($storecfg, $vmid, $device) = @_;
4479
4480     my $kvmver = get_running_qemu_version($vmid);
4481     my $io_uring = min_version($kvmver, 6, 0);
4482     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4483     $drive =~ s/\\/\\\\/g;
4484     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4485
4486     # If the command succeeds qemu prints: "OK"
4487     return 1 if $ret =~ m/OK/s;
4488
4489     die "adding drive failed: $ret\n";
4490 }
4491
4492 sub qemu_drivedel {
4493     my ($vmid, $deviceid) = @_;
4494
4495     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4496     $ret =~ s/^\s+//;
4497
4498     return 1 if $ret eq "";
4499
4500     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4501     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4502
4503     die "deleting drive $deviceid failed : $ret\n";
4504 }
4505
4506 sub qemu_deviceaddverify {
4507     my ($vmid, $deviceid) = @_;
4508
4509     for (my $i = 0; $i <= 5; $i++) {
4510          my $devices_list = vm_devices_list($vmid);
4511          return 1 if defined($devices_list->{$deviceid});
4512          sleep 1;
4513     }
4514
4515     die "error on hotplug device '$deviceid'\n";
4516 }
4517
4518
4519 sub qemu_devicedelverify {
4520     my ($vmid, $deviceid) = @_;
4521
4522     # need to verify that the device is correctly removed as device_del
4523     # is async and empty return is not reliable
4524
4525     for (my $i = 0; $i <= 5; $i++) {
4526          my $devices_list = vm_devices_list($vmid);
4527          return 1 if !defined($devices_list->{$deviceid});
4528          sleep 1;
4529     }
4530
4531     die "error on hot-unplugging device '$deviceid'\n";
4532 }
4533
4534 sub qemu_findorcreatescsihw {
4535     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4536
4537     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4538
4539     my $scsihwid="$controller_prefix$controller";
4540     my $devices_list = vm_devices_list($vmid);
4541
4542     if (!defined($devices_list->{$scsihwid})) {
4543         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4544     }
4545
4546     return 1;
4547 }
4548
4549 sub qemu_deletescsihw {
4550     my ($conf, $vmid, $opt) = @_;
4551
4552     my $device = parse_drive($opt, $conf->{$opt});
4553
4554     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4555         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4556         return 1;
4557     }
4558
4559     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4560
4561     my $devices_list = vm_devices_list($vmid);
4562     foreach my $opt (keys %{$devices_list}) {
4563         if (is_valid_drivename($opt)) {
4564             my $drive = parse_drive($opt, $conf->{$opt});
4565             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4566                 return 1;
4567             }
4568         }
4569     }
4570
4571     my $scsihwid="scsihw$controller";
4572
4573     vm_deviceunplug($vmid, $conf, $scsihwid);
4574
4575     return 1;
4576 }
4577
4578 sub qemu_add_pci_bridge {
4579     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4580
4581     my $bridges = {};
4582
4583     my $bridgeid;
4584
4585     print_pci_addr($device, $bridges, $arch, $machine_type);
4586
4587     while (my ($k, $v) = each %$bridges) {
4588         $bridgeid = $k;
4589     }
4590     return 1 if !defined($bridgeid) || $bridgeid < 1;
4591
4592     my $bridge = "pci.$bridgeid";
4593     my $devices_list = vm_devices_list($vmid);
4594
4595     if (!defined($devices_list->{$bridge})) {
4596         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4597     }
4598
4599     return 1;
4600 }
4601
4602 sub qemu_set_link_status {
4603     my ($vmid, $device, $up) = @_;
4604
4605     mon_cmd($vmid, "set_link", name => $device,
4606                up => $up ? JSON::true : JSON::false);
4607 }
4608
4609 sub qemu_netdevadd {
4610     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4611
4612     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4613     my %options =  split(/[=,]/, $netdev);
4614
4615     if (defined(my $vhost = $options{vhost})) {
4616         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4617     }
4618
4619     if (defined(my $queues = $options{queues})) {
4620         $options{queues} = $queues + 0;
4621     }
4622
4623     mon_cmd($vmid, "netdev_add",  %options);
4624     return 1;
4625 }
4626
4627 sub qemu_netdevdel {
4628     my ($vmid, $deviceid) = @_;
4629
4630     mon_cmd($vmid, "netdev_del", id => $deviceid);
4631 }
4632
4633 sub qemu_usb_hotplug {
4634     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4635
4636     return if !$device;
4637
4638     # remove the old one first
4639     vm_deviceunplug($vmid, $conf, $deviceid);
4640
4641     # check if xhci controller is necessary and available
4642     my $devicelist = vm_devices_list($vmid);
4643
4644     if (!$devicelist->{xhci}) {
4645         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4646         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4647     }
4648
4649     # print_usbdevice_full expects the parsed device
4650     my $d = parse_usb_device($device->{host});
4651     $d->{usb3} = $device->{usb3};
4652
4653     # add the new one
4654     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4655 }
4656
4657 sub qemu_cpu_hotplug {
4658     my ($vmid, $conf, $vcpus) = @_;
4659
4660     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4661
4662     my $sockets = 1;
4663     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4664     $sockets = $conf->{sockets} if  $conf->{sockets};
4665     my $cores = $conf->{cores} || 1;
4666     my $maxcpus = $sockets * $cores;
4667
4668     $vcpus = $maxcpus if !$vcpus;
4669
4670     die "you can't add more vcpus than maxcpus\n"
4671         if $vcpus > $maxcpus;
4672
4673     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4674
4675     if ($vcpus < $currentvcpus) {
4676
4677         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4678
4679             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4680                 qemu_devicedel($vmid, "cpu$i");
4681                 my $retry = 0;
4682                 my $currentrunningvcpus = undef;
4683                 while (1) {
4684                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4685                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4686                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4687                     $retry++;
4688                     sleep 1;
4689                 }
4690                 #update conf after each succesfull cpu unplug
4691                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4692                 PVE::QemuConfig->write_config($vmid, $conf);
4693             }
4694         } else {
4695             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4696         }
4697
4698         return;
4699     }
4700
4701     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4702     die "vcpus in running vm does not match its configuration\n"
4703         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4704
4705     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4706
4707         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4708             my $cpustr = print_cpu_device($conf, $i);
4709             qemu_deviceadd($vmid, $cpustr);
4710
4711             my $retry = 0;
4712             my $currentrunningvcpus = undef;
4713             while (1) {
4714                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4715                 last if scalar(@{$currentrunningvcpus}) == $i;
4716                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4717                 sleep 1;
4718                 $retry++;
4719             }
4720             #update conf after each succesfull cpu hotplug
4721             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4722             PVE::QemuConfig->write_config($vmid, $conf);
4723         }
4724     } else {
4725
4726         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4727             mon_cmd($vmid, "cpu-add", id => int($i));
4728         }
4729     }
4730 }
4731
4732 sub qemu_block_set_io_throttle {
4733     my ($vmid, $deviceid,
4734         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4735         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4736         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4737         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4738
4739     return if !check_running($vmid) ;
4740
4741     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4742         bps => int($bps),
4743         bps_rd => int($bps_rd),
4744         bps_wr => int($bps_wr),
4745         iops => int($iops),
4746         iops_rd => int($iops_rd),
4747         iops_wr => int($iops_wr),
4748         bps_max => int($bps_max),
4749         bps_rd_max => int($bps_rd_max),
4750         bps_wr_max => int($bps_wr_max),
4751         iops_max => int($iops_max),
4752         iops_rd_max => int($iops_rd_max),
4753         iops_wr_max => int($iops_wr_max),
4754         bps_max_length => int($bps_max_length),
4755         bps_rd_max_length => int($bps_rd_max_length),
4756         bps_wr_max_length => int($bps_wr_max_length),
4757         iops_max_length => int($iops_max_length),
4758         iops_rd_max_length => int($iops_rd_max_length),
4759         iops_wr_max_length => int($iops_wr_max_length),
4760     );
4761
4762 }
4763
4764 sub qemu_block_resize {
4765     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4766
4767     my $running = check_running($vmid);
4768
4769     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4770
4771     return if !$running;
4772
4773     my $padding = (1024 - $size % 1024) % 1024;
4774     $size = $size + $padding;
4775
4776     mon_cmd(
4777         $vmid,
4778         "block_resize",
4779         device => $deviceid,
4780         size => int($size),
4781         timeout => 60,
4782     );
4783 }
4784
4785 sub qemu_volume_snapshot {
4786     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4787
4788     my $running = check_running($vmid);
4789
4790     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4791         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4792     } else {
4793         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4794     }
4795 }
4796
4797 sub qemu_volume_snapshot_delete {
4798     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4799
4800     my $running = check_running($vmid);
4801
4802     if($running) {
4803
4804         $running = undef;
4805         my $conf = PVE::QemuConfig->load_config($vmid);
4806         PVE::QemuConfig->foreach_volume($conf, sub {
4807             my ($ds, $drive) = @_;
4808             $running = 1 if $drive->{file} eq $volid;
4809         });
4810     }
4811
4812     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4813         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4814     } else {
4815         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4816     }
4817 }
4818
4819 sub set_migration_caps {
4820     my ($vmid, $savevm) = @_;
4821
4822     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4823
4824     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4825     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4826
4827     my $cap_ref = [];
4828
4829     my $enabled_cap = {
4830         "auto-converge" => 1,
4831         "xbzrle" => 1,
4832         "x-rdma-pin-all" => 0,
4833         "zero-blocks" => 0,
4834         "compress" => 0,
4835         "dirty-bitmaps" => $dirty_bitmaps,
4836     };
4837
4838     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4839
4840     for my $supported_capability (@$supported_capabilities) {
4841         push @$cap_ref, {
4842             capability => $supported_capability->{capability},
4843             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4844         };
4845     }
4846
4847     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4848 }
4849
4850 sub foreach_volid {
4851     my ($conf, $func, @param) = @_;
4852
4853     my $volhash = {};
4854
4855     my $test_volid = sub {
4856         my ($key, $drive, $snapname) = @_;
4857
4858         my $volid = $drive->{file};
4859         return if !$volid;
4860
4861         $volhash->{$volid}->{cdrom} //= 1;
4862         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4863
4864         my $replicate = $drive->{replicate} // 1;
4865         $volhash->{$volid}->{replicate} //= 0;
4866         $volhash->{$volid}->{replicate} = 1 if $replicate;
4867
4868         $volhash->{$volid}->{shared} //= 0;
4869         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4870
4871         $volhash->{$volid}->{referenced_in_config} //= 0;
4872         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4873
4874         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4875             if defined($snapname);
4876
4877         my $size = $drive->{size};
4878         $volhash->{$volid}->{size} //= $size if $size;
4879
4880         $volhash->{$volid}->{is_vmstate} //= 0;
4881         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4882
4883         $volhash->{$volid}->{is_tpmstate} //= 0;
4884         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4885
4886         $volhash->{$volid}->{is_unused} //= 0;
4887         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4888
4889         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4890     };
4891
4892     my $include_opts = {
4893         extra_keys => ['vmstate'],
4894         include_unused => 1,
4895     };
4896
4897     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4898     foreach my $snapname (keys %{$conf->{snapshots}}) {
4899         my $snap = $conf->{snapshots}->{$snapname};
4900         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4901     }
4902
4903     foreach my $volid (keys %$volhash) {
4904         &$func($volid, $volhash->{$volid}, @param);
4905     }
4906 }
4907
4908 my $fast_plug_option = {
4909     'lock' => 1,
4910     'name' => 1,
4911     'onboot' => 1,
4912     'shares' => 1,
4913     'startup' => 1,
4914     'description' => 1,
4915     'protection' => 1,
4916     'vmstatestorage' => 1,
4917     'hookscript' => 1,
4918     'tags' => 1,
4919 };
4920
4921 for my $opt (keys %$confdesc_cloudinit) {
4922     $fast_plug_option->{$opt} = 1;
4923 };
4924
4925 # hotplug changes in [PENDING]
4926 # $selection hash can be used to only apply specified options, for
4927 # example: { cores => 1 } (only apply changed 'cores')
4928 # $errors ref is used to return error messages
4929 sub vmconfig_hotplug_pending {
4930     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4931
4932     my $defaults = load_defaults();
4933     my $arch = get_vm_arch($conf);
4934     my $machine_type = get_vm_machine($conf, undef, $arch);
4935
4936     # commit values which do not have any impact on running VM first
4937     # Note: those option cannot raise errors, we we do not care about
4938     # $selection and always apply them.
4939
4940     my $add_error = sub {
4941         my ($opt, $msg) = @_;
4942         $errors->{$opt} = "hotplug problem - $msg";
4943     };
4944
4945     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4946
4947     my $cloudinit_record_changed = sub {
4948         my ($conf, $opt, $old, $new) = @_;
4949         return if !$cloudinit_pending_properties->{$opt};
4950
4951         my $ci = ($conf->{cloudinit} //= {});
4952
4953         my $recorded = $ci->{$opt};
4954         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4955
4956         if (defined($new)) {
4957             if (defined($old)) {
4958                 # an existing value is being modified
4959                 if (defined($recorded)) {
4960                     # the value was already not in sync
4961                     if ($new eq $recorded) {
4962                         # a value is being reverted to the cloud-init state:
4963                         delete $ci->{$opt};
4964                         delete $added{$opt};
4965                     } else {
4966                         # the value was changed multiple times, do nothing
4967                     }
4968                 } elsif ($added{$opt}) {
4969                     # the value had been marked as added and is being changed, do nothing
4970                 } else {
4971                     # the value is new, record it:
4972                     $ci->{$opt} = $old;
4973                 }
4974             } else {
4975                 # a new value is being added
4976                 if (defined($recorded)) {
4977                     # it was already not in sync
4978                     if ($new eq $recorded) {
4979                         # a value is being reverted to the cloud-init state:
4980                         delete $ci->{$opt};
4981                         delete $added{$opt};
4982                     } else {
4983                         # the value had temporarily been removed, do nothing
4984                     }
4985                 } elsif ($added{$opt}) {
4986                     # the value had been marked as added already, do nothing
4987                 } else {
4988                     # the value is new, add it
4989                     $added{$opt} = 1;
4990                 }
4991             }
4992         } elsif (!defined($old)) {
4993             # a non-existent value is being removed? ignore...
4994         } else {
4995             # a value is being deleted
4996             if (defined($recorded)) {
4997                 # a value was already recorded, just keep it
4998             } elsif ($added{$opt}) {
4999                 # the value was marked as added, remove it
5000                 delete $added{$opt};
5001             } else {
5002                 # a previously unrecorded value is being removed, record the old value:
5003                 $ci->{$opt} = $old;
5004             }
5005         }
5006
5007         my $added = join(',', sort keys %added);
5008         $ci->{added} = $added if length($added);
5009     };
5010
5011     my $changes = 0;
5012     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5013         if ($fast_plug_option->{$opt}) {
5014             my $new = delete $conf->{pending}->{$opt};
5015             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5016             $conf->{$opt} = $new;
5017             $changes = 1;
5018         }
5019     }
5020
5021     if ($changes) {
5022         PVE::QemuConfig->write_config($vmid, $conf);
5023     }
5024
5025     my $ostype = $conf->{ostype};
5026     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5027     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5028     my $usb_hotplug = $hotplug_features->{usb}
5029         && min_version($version, 7, 1)
5030         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5031
5032     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5033     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5034
5035     foreach my $opt (sort keys %$pending_delete_hash) {
5036         next if $selection && !$selection->{$opt};
5037         my $force = $pending_delete_hash->{$opt}->{force};
5038         eval {
5039             if ($opt eq 'hotplug') {
5040                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5041             } elsif ($opt eq 'tablet') {
5042                 die "skip\n" if !$hotplug_features->{usb};
5043                 if ($defaults->{tablet}) {
5044                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5045                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5046                         if $arch eq 'aarch64';
5047                 } else {
5048                     vm_deviceunplug($vmid, $conf, 'tablet');
5049                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5050                 }
5051             } elsif ($opt =~ m/^usb(\d+)$/) {
5052                 my $index = $1;
5053                 die "skip\n" if !$usb_hotplug;
5054                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5055                 vm_deviceunplug($vmid, $conf, $opt);
5056             } elsif ($opt eq 'vcpus') {
5057                 die "skip\n" if !$hotplug_features->{cpu};
5058                 qemu_cpu_hotplug($vmid, $conf, undef);
5059             } elsif ($opt eq 'balloon') {
5060                 # enable balloon device is not hotpluggable
5061                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5062                 # here we reset the ballooning value to memory
5063                 my $balloon = $conf->{memory} || $defaults->{memory};
5064                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5065             } elsif ($fast_plug_option->{$opt}) {
5066                 # do nothing
5067             } elsif ($opt =~ m/^net(\d+)$/) {
5068                 die "skip\n" if !$hotplug_features->{network};
5069                 vm_deviceunplug($vmid, $conf, $opt);
5070             } elsif (is_valid_drivename($opt)) {
5071                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5072                 vm_deviceunplug($vmid, $conf, $opt);
5073                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5074             } elsif ($opt =~ m/^memory$/) {
5075                 die "skip\n" if !$hotplug_features->{memory};
5076                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5077             } elsif ($opt eq 'cpuunits') {
5078                 $cgroup->change_cpu_shares(undef);
5079             } elsif ($opt eq 'cpulimit') {
5080                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5081             } else {
5082                 die "skip\n";
5083             }
5084         };
5085         if (my $err = $@) {
5086             &$add_error($opt, $err) if $err ne "skip\n";
5087         } else {
5088             my $old = delete $conf->{$opt};
5089             $cloudinit_record_changed->($conf, $opt, $old, undef);
5090             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5091         }
5092     }
5093
5094     my $cloudinit_opt;
5095     foreach my $opt (keys %{$conf->{pending}}) {
5096         next if $selection && !$selection->{$opt};
5097         my $value = $conf->{pending}->{$opt};
5098         eval {
5099             if ($opt eq 'hotplug') {
5100                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5101             } elsif ($opt eq 'tablet') {
5102                 die "skip\n" if !$hotplug_features->{usb};
5103                 if ($value == 1) {
5104                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5105                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5106                         if $arch eq 'aarch64';
5107                 } elsif ($value == 0) {
5108                     vm_deviceunplug($vmid, $conf, 'tablet');
5109                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5110                 }
5111             } elsif ($opt =~ m/^usb(\d+)$/) {
5112                 my $index = $1;
5113                 die "skip\n" if !$usb_hotplug;
5114                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5115                 my $id = $opt;
5116                 if ($d->{host} eq 'spice')  {
5117                     $id = "usbredirdev$index";
5118                 }
5119                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5120             } elsif ($opt eq 'vcpus') {
5121                 die "skip\n" if !$hotplug_features->{cpu};
5122                 qemu_cpu_hotplug($vmid, $conf, $value);
5123             } elsif ($opt eq 'balloon') {
5124                 # enable/disable balloning device is not hotpluggable
5125                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5126                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5127                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5128
5129                 # allow manual ballooning if shares is set to zero
5130                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5131                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5132                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5133                 }
5134             } elsif ($opt =~ m/^net(\d+)$/) {
5135                 # some changes can be done without hotplug
5136                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5137                                     $vmid, $opt, $value, $arch, $machine_type);
5138             } elsif (is_valid_drivename($opt)) {
5139                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5140                 # some changes can be done without hotplug
5141                 my $drive = parse_drive($opt, $value);
5142                 if (drive_is_cloudinit($drive)) {
5143                     $cloudinit_opt = [$opt, $drive];
5144                     # apply all the other changes first, then generate the cloudinit disk
5145                     die "skip\n";
5146                 }
5147                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5148                                      $vmid, $opt, $value, $arch, $machine_type);
5149             } elsif ($opt =~ m/^memory$/) { #dimms
5150                 die "skip\n" if !$hotplug_features->{memory};
5151                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5152             } elsif ($opt eq 'cpuunits') {
5153                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5154                 $cgroup->change_cpu_shares($new_cpuunits);
5155             } elsif ($opt eq 'cpulimit') {
5156                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5157                 $cgroup->change_cpu_quota($cpulimit, 100000);
5158             } elsif ($opt eq 'agent') {
5159                 vmconfig_update_agent($conf, $opt, $value);
5160             } else {
5161                 die "skip\n";  # skip non-hot-pluggable options
5162             }
5163         };
5164         if (my $err = $@) {
5165             &$add_error($opt, $err) if $err ne "skip\n";
5166         } else {
5167             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5168             $conf->{$opt} = $value;
5169             delete $conf->{pending}->{$opt};
5170         }
5171     }
5172
5173     if (defined($cloudinit_opt)) {
5174         my ($opt, $drive) = @$cloudinit_opt;
5175         my $value = $conf->{pending}->{$opt};
5176         eval {
5177             my $temp = {%$conf, $opt => $value};
5178             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5179             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5180                                  $vmid, $opt, $value, $arch, $machine_type);
5181         };
5182         if (my $err = $@) {
5183             &$add_error($opt, $err) if $err ne "skip\n";
5184         } else {
5185             $conf->{$opt} = $value;
5186             delete $conf->{pending}->{$opt};
5187         }
5188     }
5189
5190     # unplug xhci controller if no usb device is left
5191     if ($usb_hotplug) {
5192         my $has_usb = 0;
5193         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5194             next if !defined($conf->{"usb$i"});
5195             $has_usb = 1;
5196             last;
5197         }
5198         if (!$has_usb) {
5199             vm_deviceunplug($vmid, $conf, 'xhci');
5200         }
5201     }
5202
5203     PVE::QemuConfig->write_config($vmid, $conf);
5204
5205     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5206         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5207     }
5208 }
5209
5210 sub try_deallocate_drive {
5211     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5212
5213     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5214         my $volid = $drive->{file};
5215         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5216             my $sid = PVE::Storage::parse_volume_id($volid);
5217             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5218
5219             # check if the disk is really unused
5220             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5221                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5222             PVE::Storage::vdisk_free($storecfg, $volid);
5223             return 1;
5224         } else {
5225             # If vm is not owner of this disk remove from config
5226             return 1;
5227         }
5228     }
5229
5230     return;
5231 }
5232
5233 sub vmconfig_delete_or_detach_drive {
5234     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5235
5236     my $drive = parse_drive($opt, $conf->{$opt});
5237
5238     my $rpcenv = PVE::RPCEnvironment::get();
5239     my $authuser = $rpcenv->get_user();
5240
5241     if ($force) {
5242         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5243         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5244     } else {
5245         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5246     }
5247 }
5248
5249
5250
5251 sub vmconfig_apply_pending {
5252     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5253
5254     return if !scalar(keys %{$conf->{pending}});
5255
5256     my $add_apply_error = sub {
5257         my ($opt, $msg) = @_;
5258         my $err_msg = "unable to apply pending change $opt : $msg";
5259         $errors->{$opt} = $err_msg;
5260         warn $err_msg;
5261     };
5262
5263     # cold plug
5264
5265     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5266     foreach my $opt (sort keys %$pending_delete_hash) {
5267         my $force = $pending_delete_hash->{$opt}->{force};
5268         eval {
5269             if ($opt =~ m/^unused/) {
5270                 die "internal error";
5271             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5272                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5273             }
5274         };
5275         if (my $err = $@) {
5276             $add_apply_error->($opt, $err);
5277         } else {
5278             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5279             delete $conf->{$opt};
5280         }
5281     }
5282
5283     PVE::QemuConfig->cleanup_pending($conf);
5284
5285     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5286
5287     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5288         next if $opt eq 'delete'; # just to be sure
5289         eval {
5290             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5291                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5292             }
5293         };
5294         if (my $err = $@) {
5295             $add_apply_error->($opt, $err);
5296         } else {
5297
5298             if (is_valid_drivename($opt)) {
5299                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5300                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5301             }
5302
5303             $conf->{$opt} = delete $conf->{pending}->{$opt};
5304         }
5305     }
5306
5307     # write all changes at once to avoid unnecessary i/o
5308     PVE::QemuConfig->write_config($vmid, $conf);
5309     if ($generate_cloudinit) {
5310         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5311             # After successful generation and if there were changes to be applied, update the
5312             # config to drop the {cloudinit} entry.
5313             PVE::QemuConfig->write_config($vmid, $conf);
5314         }
5315     }
5316 }
5317
5318 sub vmconfig_update_net {
5319     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5320
5321     my $newnet = parse_net($value);
5322
5323     if ($conf->{$opt}) {
5324         my $oldnet = parse_net($conf->{$opt});
5325
5326         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5327             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5328             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5329             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5330             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5331
5332             # for non online change, we try to hot-unplug
5333             die "skip\n" if !$hotplug;
5334             vm_deviceunplug($vmid, $conf, $opt);
5335         } else {
5336
5337             die "internal error" if $opt !~ m/net(\d+)/;
5338             my $iface = "tap${vmid}i$1";
5339
5340             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5341                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5342                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5343                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5344                 PVE::Network::tap_unplug($iface);
5345
5346                 if ($have_sdn) {
5347                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5348                 } else {
5349                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5350                 }
5351             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5352                 # Rate can be applied on its own but any change above needs to
5353                 # include the rate in tap_plug since OVS resets everything.
5354                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5355             }
5356
5357             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5358                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5359             }
5360
5361             return 1;
5362         }
5363     }
5364
5365     if ($hotplug) {
5366         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5367     } else {
5368         die "skip\n";
5369     }
5370 }
5371
5372 sub vmconfig_update_agent {
5373     my ($conf, $opt, $value) = @_;
5374
5375     die "skip\n" if !$conf->{$opt};
5376
5377     my $hotplug_options = { fstrim_cloned_disks => 1 };
5378
5379     my $old_agent = parse_guest_agent($conf);
5380     my $agent = parse_guest_agent({$opt => $value});
5381
5382     for my $option (keys %$agent) { # added/changed options
5383         next if defined($hotplug_options->{$option});
5384         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5385     }
5386
5387     for my $option (keys %$old_agent) { # removed options
5388         next if defined($hotplug_options->{$option});
5389         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5390     }
5391
5392     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5393 }
5394
5395 sub vmconfig_update_disk {
5396     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5397
5398     my $drive = parse_drive($opt, $value);
5399
5400     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5401         my $media = $drive->{media} || 'disk';
5402         my $oldmedia = $old_drive->{media} || 'disk';
5403         die "unable to change media type\n" if $media ne $oldmedia;
5404
5405         if (!drive_is_cdrom($old_drive)) {
5406
5407             if ($drive->{file} ne $old_drive->{file}) {
5408
5409                 die "skip\n" if !$hotplug;
5410
5411                 # unplug and register as unused
5412                 vm_deviceunplug($vmid, $conf, $opt);
5413                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5414
5415             } else {
5416                 # update existing disk
5417
5418                 # skip non hotpluggable value
5419                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5420                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5421                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5422                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5423                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5424                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5425                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5426                     die "skip\n";
5427                 }
5428
5429                 # apply throttle
5430                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5431                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5432                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5433                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5434                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5435                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5436                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5437                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5438                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5439                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5440                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5441                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5442                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5443                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5444                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5445                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5446                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5447                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5448
5449                     qemu_block_set_io_throttle(
5450                         $vmid,"drive-$opt",
5451                         ($drive->{mbps} || 0)*1024*1024,
5452                         ($drive->{mbps_rd} || 0)*1024*1024,
5453                         ($drive->{mbps_wr} || 0)*1024*1024,
5454                          $drive->{iops} || 0,
5455                          $drive->{iops_rd} || 0,
5456                          $drive->{iops_wr} || 0,
5457                         ($drive->{mbps_max} || 0)*1024*1024,
5458                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5459                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5460                          $drive->{iops_max} || 0,
5461                          $drive->{iops_rd_max} || 0,
5462                          $drive->{iops_wr_max} || 0,
5463                          $drive->{bps_max_length} || 1,
5464                          $drive->{bps_rd_max_length} || 1,
5465                          $drive->{bps_wr_max_length} || 1,
5466                          $drive->{iops_max_length} || 1,
5467                          $drive->{iops_rd_max_length} || 1,
5468                          $drive->{iops_wr_max_length} || 1,
5469                     );
5470
5471                 }
5472
5473                 return 1;
5474             }
5475
5476         } else { # cdrom
5477
5478             if ($drive->{file} eq 'none') {
5479                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5480                 if (drive_is_cloudinit($old_drive)) {
5481                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5482                 }
5483             } else {
5484                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5485
5486                 # force eject if locked
5487                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5488
5489                 if ($path) {
5490                     mon_cmd($vmid, "blockdev-change-medium",
5491                         id => "$opt", filename => "$path");
5492                 }
5493             }
5494
5495             return 1;
5496         }
5497     }
5498
5499     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5500     # hotplug new disks
5501     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5502     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5503 }
5504
5505 sub vmconfig_update_cloudinit_drive {
5506     my ($storecfg, $conf, $vmid) = @_;
5507
5508     my $cloudinit_ds = undef;
5509     my $cloudinit_drive = undef;
5510
5511     PVE::QemuConfig->foreach_volume($conf, sub {
5512         my ($ds, $drive) = @_;
5513         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5514             $cloudinit_ds = $ds;
5515             $cloudinit_drive = $drive;
5516         }
5517     });
5518
5519     return if !$cloudinit_drive;
5520
5521     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5522         PVE::QemuConfig->write_config($vmid, $conf);
5523     }
5524
5525     my $running = PVE::QemuServer::check_running($vmid);
5526
5527     if ($running) {
5528         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5529         if ($path) {
5530             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5531             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5532         }
5533     }
5534 }
5535
5536 # called in locked context by incoming migration
5537 sub vm_migrate_get_nbd_disks {
5538     my ($storecfg, $conf, $replicated_volumes) = @_;
5539
5540     my $local_volumes = {};
5541     PVE::QemuConfig->foreach_volume($conf, sub {
5542         my ($ds, $drive) = @_;
5543
5544         return if drive_is_cdrom($drive);
5545         return if $ds eq 'tpmstate0';
5546
5547         my $volid = $drive->{file};
5548
5549         return if !$volid;
5550
5551         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5552
5553         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5554         return if $scfg->{shared};
5555
5556         # replicated disks re-use existing state via bitmap
5557         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5558         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5559     });
5560     return $local_volumes;
5561 }
5562
5563 # called in locked context by incoming migration
5564 sub vm_migrate_alloc_nbd_disks {
5565     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5566
5567     my $nbd = {};
5568     foreach my $opt (sort keys %$source_volumes) {
5569         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5570
5571         if ($use_existing) {
5572             $nbd->{$opt}->{drivestr} = print_drive($drive);
5573             $nbd->{$opt}->{volid} = $volid;
5574             $nbd->{$opt}->{replicated} = 1;
5575             next;
5576         }
5577
5578         # storage mapping + volname = regular migration
5579         # storage mapping + format = remote migration
5580         # order of precedence, filtered by whether storage supports it:
5581         # 1. explicit requested format
5582         # 2. format of current volume
5583         # 3. default format of storage
5584         if (!$storagemap->{identity}) {
5585             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5586             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5587             if (!$format || !grep { $format eq $_ } @$validFormats) {
5588                 if ($volname) {
5589                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5590                     my $fileFormat = qemu_img_format($scfg, $volname);
5591                     $format = $fileFormat
5592                         if grep { $fileFormat eq $_ } @$validFormats;
5593                 }
5594                 $format //= $defFormat;
5595             }
5596         } else {
5597             # can't happen for remote migration, so $volname is always defined
5598             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5599             $format = qemu_img_format($scfg, $volname);
5600         }
5601
5602         my $size = $drive->{size} / 1024;
5603         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5604         my $newdrive = $drive;
5605         $newdrive->{format} = $format;
5606         $newdrive->{file} = $newvolid;
5607         my $drivestr = print_drive($newdrive);
5608         $nbd->{$opt}->{drivestr} = $drivestr;
5609         $nbd->{$opt}->{volid} = $newvolid;
5610     }
5611
5612     return $nbd;
5613 }
5614
5615 # see vm_start_nolock for parameters, additionally:
5616 # migrate_opts:
5617 #   storagemap = parsed storage map for allocating NBD disks
5618 sub vm_start {
5619     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5620
5621     return PVE::QemuConfig->lock_config($vmid, sub {
5622         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5623
5624         die "you can't start a vm if it's a template\n"
5625             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5626
5627         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5628         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5629
5630         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5631
5632         if ($has_backup_lock && $running) {
5633             # a backup is currently running, attempt to start the guest in the
5634             # existing QEMU instance
5635             return vm_resume($vmid);
5636         }
5637
5638         PVE::QemuConfig->check_lock($conf)
5639             if !($params->{skiplock} || $has_suspended_lock);
5640
5641         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5642
5643         die "VM $vmid already running\n" if $running;
5644
5645         if (my $storagemap = $migrate_opts->{storagemap}) {
5646             my $replicated = $migrate_opts->{replicated_volumes};
5647             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5648             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5649
5650             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5651                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5652             }
5653         }
5654
5655         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5656     });
5657 }
5658
5659
5660 # params:
5661 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5662 #   skiplock => 0/1, skip checking for config lock
5663 #   skiptemplate => 0/1, skip checking whether VM is template
5664 #   forcemachine => to force QEMU machine (rollback/migration)
5665 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5666 #   timeout => in seconds
5667 #   paused => start VM in paused state (backup)
5668 #   resume => resume from hibernation
5669 #   pbs-backing => {
5670 #      sata0 => {
5671 #         repository
5672 #         snapshot
5673 #         keyfile
5674 #         archive
5675 #      },
5676 #      virtio2 => ...
5677 #   }
5678 # migrate_opts:
5679 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5680 #   migratedfrom => source node
5681 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5682 #   network => CIDR of migration network
5683 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5684 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5685 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5686 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5687 #       contained in config
5688 sub vm_start_nolock {
5689     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5690
5691     my $statefile = $params->{statefile};
5692     my $resume = $params->{resume};
5693
5694     my $migratedfrom = $migrate_opts->{migratedfrom};
5695     my $migration_type = $migrate_opts->{type};
5696
5697     my $res = {};
5698
5699     # clean up leftover reboot request files
5700     eval { clear_reboot_request($vmid); };
5701     warn $@ if $@;
5702
5703     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5704         vmconfig_apply_pending($vmid, $conf, $storecfg);
5705         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5706     }
5707
5708     # don't regenerate the ISO if the VM is started as part of a live migration
5709     # this way we can reuse the old ISO with the correct config
5710     if (!$migratedfrom) {
5711         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5712             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5713             # $conf->{cloudinit}, so we could just not do this?
5714             # But we do it above, so for now let's be consistent.
5715             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5716         }
5717     }
5718
5719     # override offline migrated volumes, conf is out of date still
5720     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5721         for my $key (sort keys $offline_volumes->%*) {
5722             my $parsed = parse_drive($key, $conf->{$key});
5723             $parsed->{file} = $offline_volumes->{$key};
5724             $conf->{$key} = print_drive($parsed);
5725         }
5726     }
5727
5728     my $defaults = load_defaults();
5729
5730     # set environment variable useful inside network script
5731     # for remote migration the config is available on the target node!
5732     if (!$migrate_opts->{remote_node}) {
5733         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5734     }
5735
5736     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5737
5738     my $forcemachine = $params->{forcemachine};
5739     my $forcecpu = $params->{forcecpu};
5740     if ($resume) {
5741         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5742         $forcemachine = $conf->{runningmachine};
5743         $forcecpu = $conf->{runningcpu};
5744         print "Resuming suspended VM\n";
5745     }
5746
5747     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5748         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5749
5750     my $migration_ip;
5751     my $get_migration_ip = sub {
5752         my ($nodename) = @_;
5753
5754         return $migration_ip if defined($migration_ip);
5755
5756         my $cidr = $migrate_opts->{network};
5757
5758         if (!defined($cidr)) {
5759             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5760             $cidr = $dc_conf->{migration}->{network};
5761         }
5762
5763         if (defined($cidr)) {
5764             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5765
5766             die "could not get IP: no address configured on local " .
5767                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5768
5769             die "could not get IP: multiple addresses configured on local " .
5770                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5771
5772             $migration_ip = @$ips[0];
5773         }
5774
5775         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5776             if !defined($migration_ip);
5777
5778         return $migration_ip;
5779     };
5780
5781     if ($statefile) {
5782         if ($statefile eq 'tcp') {
5783             my $migrate = $res->{migrate} = { proto => 'tcp' };
5784             $migrate->{addr} = "localhost";
5785             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5786             my $nodename = nodename();
5787
5788             if (!defined($migration_type)) {
5789                 if (defined($datacenterconf->{migration}->{type})) {
5790                     $migration_type = $datacenterconf->{migration}->{type};
5791                 } else {
5792                     $migration_type = 'secure';
5793                 }
5794             }
5795
5796             if ($migration_type eq 'insecure') {
5797                 $migrate->{addr} = $get_migration_ip->($nodename);
5798                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5799             }
5800
5801             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5802             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5803             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5804             push @$cmd, '-incoming', $migrate->{uri};
5805             push @$cmd, '-S';
5806
5807         } elsif ($statefile eq 'unix') {
5808             # should be default for secure migrations as a ssh TCP forward
5809             # tunnel is not deterministic reliable ready and fails regurarly
5810             # to set up in time, so use UNIX socket forwards
5811             my $migrate = $res->{migrate} = { proto => 'unix' };
5812             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5813             unlink $migrate->{addr};
5814
5815             $migrate->{uri} = "unix:$migrate->{addr}";
5816             push @$cmd, '-incoming', $migrate->{uri};
5817             push @$cmd, '-S';
5818
5819         } elsif (-e $statefile) {
5820             push @$cmd, '-loadstate', $statefile;
5821         } else {
5822             my $statepath = PVE::Storage::path($storecfg, $statefile);
5823             push @$vollist, $statefile;
5824             push @$cmd, '-loadstate', $statepath;
5825         }
5826     } elsif ($params->{paused}) {
5827         push @$cmd, '-S';
5828     }
5829
5830     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5831
5832     my $pci_devices = {}; # host pci devices
5833     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5834         my $dev = $conf->{"hostpci$i"} or next;
5835         $pci_devices->{$i} = parse_hostpci($dev);
5836     }
5837
5838     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5839     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5840
5841     # map to a flat list of pci ids
5842     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5843
5844     # reserve all PCI IDs before actually doing anything with them
5845     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5846
5847     eval {
5848         my $uuid;
5849         for my $id (sort keys %$pci_devices) {
5850             my $d = $pci_devices->{$id};
5851             for my $dev ($d->{pciid}->@*) {
5852                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5853
5854                 # nvidia grid needs the qemu parameter '-uuid' set
5855                 # use smbios uuid or mdev uuid as fallback for that
5856                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5857                     if (defined($conf->{smbios1})) {
5858                         my $smbios_conf = parse_smbios1($conf->{smbios1});
5859                         $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5860                     }
5861                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id) if !defined($uuid);
5862                 }
5863             }
5864         }
5865         push @$cmd, '-uuid', $uuid if defined($uuid);
5866     };
5867     if (my $err = $@) {
5868         eval { cleanup_pci_devices($vmid, $conf) };
5869         warn $@ if $@;
5870         die $err;
5871     }
5872
5873     PVE::Storage::activate_volumes($storecfg, $vollist);
5874
5875     eval {
5876         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5877     };
5878     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5879     # timeout should be more than enough here...
5880     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5881
5882     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5883
5884     my %run_params = (
5885         timeout => $statefile ? undef : $start_timeout,
5886         umask => 0077,
5887         noerr => 1,
5888     );
5889
5890     # when migrating, prefix QEMU output so other side can pick up any
5891     # errors that might occur and show the user
5892     if ($migratedfrom) {
5893         $run_params{quiet} = 1;
5894         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5895     }
5896
5897     my %systemd_properties = (
5898         Slice => 'qemu.slice',
5899         KillMode => 'process',
5900         SendSIGKILL => 0,
5901         TimeoutStopUSec => ULONG_MAX, # infinity
5902     );
5903
5904     if (PVE::CGroup::cgroup_mode() == 2) {
5905         $systemd_properties{CPUWeight} = $cpuunits;
5906     } else {
5907         $systemd_properties{CPUShares} = $cpuunits;
5908     }
5909
5910     if (my $cpulimit = $conf->{cpulimit}) {
5911         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5912     }
5913     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5914
5915     my $run_qemu = sub {
5916         PVE::Tools::run_fork sub {
5917             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5918
5919             my $tpmpid;
5920             if (my $tpm = $conf->{tpmstate0}) {
5921                 # start the TPM emulator so QEMU can connect on start
5922                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5923             }
5924
5925             my $exitcode = run_command($cmd, %run_params);
5926             if ($exitcode) {
5927                 if ($tpmpid) {
5928                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5929                     kill 'TERM', $tpmpid;
5930                 }
5931                 die "QEMU exited with code $exitcode\n";
5932             }
5933         };
5934     };
5935
5936     if ($conf->{hugepages}) {
5937
5938         my $code = sub {
5939             my $hotplug_features =
5940                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5941             my $hugepages_topology =
5942                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5943
5944             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5945
5946             PVE::QemuServer::Memory::hugepages_mount();
5947             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5948
5949             eval { $run_qemu->() };
5950             if (my $err = $@) {
5951                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5952                     if !$conf->{keephugepages};
5953                 die $err;
5954             }
5955
5956             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5957                 if !$conf->{keephugepages};
5958         };
5959         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5960
5961     } else {
5962         eval { $run_qemu->() };
5963     }
5964
5965     if (my $err = $@) {
5966         # deactivate volumes if start fails
5967         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5968         warn $@ if $@;
5969         eval { cleanup_pci_devices($vmid, $conf) };
5970         warn $@ if $@;
5971
5972         die "start failed: $err";
5973     }
5974
5975     # re-reserve all PCI IDs now that we can know the actual VM PID
5976     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5977     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5978     warn $@ if $@;
5979
5980     if (defined($res->{migrate})) {
5981         print "migration listens on $res->{migrate}->{uri}\n";
5982     } elsif ($statefile) {
5983         eval { mon_cmd($vmid, "cont"); };
5984         warn $@ if $@;
5985     }
5986
5987     #start nbd server for storage migration
5988     if (my $nbd = $migrate_opts->{nbd}) {
5989         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5990
5991         my $migrate_storage_uri;
5992         # nbd_protocol_version > 0 for unix socket support
5993         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5994             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5995             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5996             $migrate_storage_uri = "nbd:unix:$socket_path";
5997             $res->{migrate}->{unix_sockets} = [$socket_path];
5998         } else {
5999             my $nodename = nodename();
6000             my $localip = $get_migration_ip->($nodename);
6001             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6002             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6003
6004             mon_cmd($vmid, "nbd-server-start", addr => {
6005                 type => 'inet',
6006                 data => {
6007                     host => "${localip}",
6008                     port => "${storage_migrate_port}",
6009                 },
6010             });
6011             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6012             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6013         }
6014
6015         my $block_info = mon_cmd($vmid, "query-block");
6016         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6017
6018         foreach my $opt (sort keys %$nbd) {
6019             my $drivestr = $nbd->{$opt}->{drivestr};
6020             my $volid = $nbd->{$opt}->{volid};
6021
6022             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6023
6024             mon_cmd(
6025                 $vmid,
6026                 "block-export-add",
6027                 id => "drive-$opt",
6028                 'node-name' => $block_node,
6029                 writable => JSON::true,
6030                 type => "nbd",
6031                 name => "drive-$opt", # NBD export name
6032             );
6033
6034             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6035             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6036             print "re-using replicated volume: $opt - $volid\n"
6037                 if $nbd->{$opt}->{replicated};
6038
6039             $res->{drives}->{$opt} = $nbd->{$opt};
6040             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6041         }
6042     }
6043
6044     if ($migratedfrom) {
6045         eval {
6046             set_migration_caps($vmid);
6047         };
6048         warn $@ if $@;
6049
6050         if ($spice_port) {
6051             print "spice listens on port $spice_port\n";
6052             $res->{spice_port} = $spice_port;
6053             if ($migrate_opts->{spice_ticket}) {
6054                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6055                     $migrate_opts->{spice_ticket});
6056                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6057             }
6058         }
6059
6060     } else {
6061         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6062             if !$statefile && $conf->{balloon};
6063
6064         foreach my $opt (keys %$conf) {
6065             next if $opt !~  m/^net\d+$/;
6066             my $nicconf = parse_net($conf->{$opt});
6067             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6068         }
6069         add_nets_bridge_fdb($conf, $vmid);
6070     }
6071
6072    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6073         eval {
6074             mon_cmd(
6075                 $vmid,
6076                 'qom-set',
6077                 path => "machine/peripheral/balloon0",
6078                 property => "guest-stats-polling-interval",
6079                 value => 2
6080             );
6081         };
6082         log_warn("could not set polling interval for ballooning - $@") if $@;
6083     }
6084
6085     if ($resume) {
6086         print "Resumed VM, removing state\n";
6087         if (my $vmstate = $conf->{vmstate}) {
6088             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6089             PVE::Storage::vdisk_free($storecfg, $vmstate);
6090         }
6091         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6092         PVE::QemuConfig->write_config($vmid, $conf);
6093     }
6094
6095     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6096
6097     return $res;
6098 }
6099
6100 sub vm_commandline {
6101     my ($storecfg, $vmid, $snapname) = @_;
6102
6103     my $conf = PVE::QemuConfig->load_config($vmid);
6104
6105     my ($forcemachine, $forcecpu);
6106     if ($snapname) {
6107         my $snapshot = $conf->{snapshots}->{$snapname};
6108         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6109
6110         # check for machine or CPU overrides in snapshot
6111         $forcemachine = $snapshot->{runningmachine};
6112         $forcecpu = $snapshot->{runningcpu};
6113
6114         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6115
6116         $conf = $snapshot;
6117     }
6118
6119     my $defaults = load_defaults();
6120
6121     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6122
6123     return PVE::Tools::cmd2string($cmd);
6124 }
6125
6126 sub vm_reset {
6127     my ($vmid, $skiplock) = @_;
6128
6129     PVE::QemuConfig->lock_config($vmid, sub {
6130
6131         my $conf = PVE::QemuConfig->load_config($vmid);
6132
6133         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6134
6135         mon_cmd($vmid, "system_reset");
6136     });
6137 }
6138
6139 sub get_vm_volumes {
6140     my ($conf) = @_;
6141
6142     my $vollist = [];
6143     foreach_volid($conf, sub {
6144         my ($volid, $attr) = @_;
6145
6146         return if $volid =~ m|^/|;
6147
6148         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6149         return if !$sid;
6150
6151         push @$vollist, $volid;
6152     });
6153
6154     return $vollist;
6155 }
6156
6157 sub cleanup_pci_devices {
6158     my ($vmid, $conf) = @_;
6159
6160     foreach my $key (keys %$conf) {
6161         next if $key !~ m/^hostpci(\d+)$/;
6162         my $hostpciindex = $1;
6163         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6164         my $d = parse_hostpci($conf->{$key});
6165         if ($d->{mdev}) {
6166             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6167             # don't want to break ABI just for this two liner
6168             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6169
6170             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6171             # out when we do it first. so wait for 10 seconds and then try it
6172             my $pciid = $d->{pciid}->[0]->{id};
6173             my $info = PVE::SysFSTools::pci_device_info("$pciid");
6174             if ($info->{vendor} eq '10de') {
6175                 sleep 10;
6176             }
6177
6178             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6179         }
6180     }
6181     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6182 }
6183
6184 sub vm_stop_cleanup {
6185     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6186
6187     eval {
6188
6189         if (!$keepActive) {
6190             my $vollist = get_vm_volumes($conf);
6191             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6192
6193             if (my $tpmdrive = $conf->{tpmstate0}) {
6194                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6195                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6196                 if ($storeid) {
6197                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6198                 }
6199             }
6200         }
6201
6202         foreach my $ext (qw(mon qmp pid vnc qga)) {
6203             unlink "/var/run/qemu-server/${vmid}.$ext";
6204         }
6205
6206         if ($conf->{ivshmem}) {
6207             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6208             # just delete it for now, VMs which have this already open do not
6209             # are affected, but new VMs will get a separated one. If this
6210             # becomes an issue we either add some sort of ref-counting or just
6211             # add a "don't delete on stop" flag to the ivshmem format.
6212             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6213         }
6214
6215         cleanup_pci_devices($vmid, $conf);
6216
6217         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6218     };
6219     warn $@ if $@; # avoid errors - just warn
6220 }
6221
6222 # call only in locked context
6223 sub _do_vm_stop {
6224     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6225
6226     my $pid = check_running($vmid, $nocheck);
6227     return if !$pid;
6228
6229     my $conf;
6230     if (!$nocheck) {
6231         $conf = PVE::QemuConfig->load_config($vmid);
6232         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6233         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6234             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6235             $timeout = $opts->{down} if $opts->{down};
6236         }
6237         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6238     }
6239
6240     eval {
6241         if ($shutdown) {
6242             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6243                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6244             } else {
6245                 mon_cmd($vmid, "system_powerdown");
6246             }
6247         } else {
6248             mon_cmd($vmid, "quit");
6249         }
6250     };
6251     my $err = $@;
6252
6253     if (!$err) {
6254         $timeout = 60 if !defined($timeout);
6255
6256         my $count = 0;
6257         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6258             $count++;
6259             sleep 1;
6260         }
6261
6262         if ($count >= $timeout) {
6263             if ($force) {
6264                 warn "VM still running - terminating now with SIGTERM\n";
6265                 kill 15, $pid;
6266             } else {
6267                 die "VM quit/powerdown failed - got timeout\n";
6268             }
6269         } else {
6270             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6271             return;
6272         }
6273     } else {
6274        if (!check_running($vmid, $nocheck)) {
6275             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6276             return;
6277        }
6278        if ($force) {
6279             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6280             kill 15, $pid;
6281         } else {
6282             die "VM quit/powerdown failed\n";
6283         }
6284     }
6285
6286     # wait again
6287     $timeout = 10;
6288
6289     my $count = 0;
6290     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6291         $count++;
6292         sleep 1;
6293     }
6294
6295     if ($count >= $timeout) {
6296         warn "VM still running - terminating now with SIGKILL\n";
6297         kill 9, $pid;
6298         sleep 1;
6299     }
6300
6301     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6302 }
6303
6304 # Note: use $nocheck to skip tests if VM configuration file exists.
6305 # We need that when migration VMs to other nodes (files already moved)
6306 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6307 sub vm_stop {
6308     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6309
6310     $force = 1 if !defined($force) && !$shutdown;
6311
6312     if ($migratedfrom){
6313         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6314         kill 15, $pid if $pid;
6315         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6316         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6317         return;
6318     }
6319
6320     PVE::QemuConfig->lock_config($vmid, sub {
6321         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6322    });
6323 }
6324
6325 sub vm_reboot {
6326     my ($vmid, $timeout) = @_;
6327
6328     PVE::QemuConfig->lock_config($vmid, sub {
6329         eval {
6330
6331             # only reboot if running, as qmeventd starts it again on a stop event
6332             return if !check_running($vmid);
6333
6334             create_reboot_request($vmid);
6335
6336             my $storecfg = PVE::Storage::config();
6337             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6338
6339         };
6340         if (my $err = $@) {
6341             # avoid that the next normal shutdown will be confused for a reboot
6342             clear_reboot_request($vmid);
6343             die $err;
6344         }
6345    });
6346 }
6347
6348 # note: if using the statestorage parameter, the caller has to check privileges
6349 sub vm_suspend {
6350     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6351
6352     my $conf;
6353     my $path;
6354     my $storecfg;
6355     my $vmstate;
6356
6357     PVE::QemuConfig->lock_config($vmid, sub {
6358
6359         $conf = PVE::QemuConfig->load_config($vmid);
6360
6361         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6362         PVE::QemuConfig->check_lock($conf)
6363             if !($skiplock || $is_backing_up);
6364
6365         die "cannot suspend to disk during backup\n"
6366             if $is_backing_up && $includestate;
6367
6368         if ($includestate) {
6369             $conf->{lock} = 'suspending';
6370             my $date = strftime("%Y-%m-%d", localtime(time()));
6371             $storecfg = PVE::Storage::config();
6372             if (!$statestorage) {
6373                 $statestorage = find_vmstate_storage($conf, $storecfg);
6374                 # check permissions for the storage
6375                 my $rpcenv = PVE::RPCEnvironment::get();
6376                 if ($rpcenv->{type} ne 'cli') {
6377                     my $authuser = $rpcenv->get_user();
6378                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6379                 }
6380             }
6381
6382
6383             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6384                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6385             $path = PVE::Storage::path($storecfg, $vmstate);
6386             PVE::QemuConfig->write_config($vmid, $conf);
6387         } else {
6388             mon_cmd($vmid, "stop");
6389         }
6390     });
6391
6392     if ($includestate) {
6393         # save vm state
6394         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6395
6396         eval {
6397             set_migration_caps($vmid, 1);
6398             mon_cmd($vmid, "savevm-start", statefile => $path);
6399             for(;;) {
6400                 my $state = mon_cmd($vmid, "query-savevm");
6401                 if (!$state->{status}) {
6402                     die "savevm not active\n";
6403                 } elsif ($state->{status} eq 'active') {
6404                     sleep(1);
6405                     next;
6406                 } elsif ($state->{status} eq 'completed') {
6407                     print "State saved, quitting\n";
6408                     last;
6409                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6410                     die "query-savevm failed with error '$state->{error}'\n"
6411                 } else {
6412                     die "query-savevm returned status '$state->{status}'\n";
6413                 }
6414             }
6415         };
6416         my $err = $@;
6417
6418         PVE::QemuConfig->lock_config($vmid, sub {
6419             $conf = PVE::QemuConfig->load_config($vmid);
6420             if ($err) {
6421                 # cleanup, but leave suspending lock, to indicate something went wrong
6422                 eval {
6423                     mon_cmd($vmid, "savevm-end");
6424                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6425                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6426                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6427                     PVE::QemuConfig->write_config($vmid, $conf);
6428                 };
6429                 warn $@ if $@;
6430                 die $err;
6431             }
6432
6433             die "lock changed unexpectedly\n"
6434                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6435
6436             mon_cmd($vmid, "quit");
6437             $conf->{lock} = 'suspended';
6438             PVE::QemuConfig->write_config($vmid, $conf);
6439         });
6440     }
6441 }
6442
6443 # $nocheck is set when called as part of a migration - in this context the
6444 # location of the config file (source or target node) is not deterministic,
6445 # since migration cannot wait for pmxcfs to process the rename
6446 sub vm_resume {
6447     my ($vmid, $skiplock, $nocheck) = @_;
6448
6449     PVE::QemuConfig->lock_config($vmid, sub {
6450         my $res = mon_cmd($vmid, 'query-status');
6451         my $resume_cmd = 'cont';
6452         my $reset = 0;
6453         my $conf;
6454         if ($nocheck) {
6455             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6456             if ($@) {
6457                 my $vmlist = PVE::Cluster::get_vmlist();
6458                 if (exists($vmlist->{ids}->{$vmid})) {
6459                     my $node = $vmlist->{ids}->{$vmid}->{node};
6460                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6461                 }
6462                 if (!$conf) {
6463                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6464                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6465                 }
6466             }
6467         } else {
6468             $conf = PVE::QemuConfig->load_config($vmid);
6469         }
6470
6471         if ($res->{status}) {
6472             return if $res->{status} eq 'running'; # job done, go home
6473             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6474             $reset = 1 if $res->{status} eq 'shutdown';
6475         }
6476
6477         if (!$nocheck) {
6478             PVE::QemuConfig->check_lock($conf)
6479                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6480         }
6481
6482         if ($reset) {
6483             # required if a VM shuts down during a backup and we get a resume
6484             # request before the backup finishes for example
6485             mon_cmd($vmid, "system_reset");
6486         }
6487
6488         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6489
6490         mon_cmd($vmid, $resume_cmd);
6491     });
6492 }
6493
6494 sub vm_sendkey {
6495     my ($vmid, $skiplock, $key) = @_;
6496
6497     PVE::QemuConfig->lock_config($vmid, sub {
6498
6499         my $conf = PVE::QemuConfig->load_config($vmid);
6500
6501         # there is no qmp command, so we use the human monitor command
6502         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6503         die $res if $res ne '';
6504     });
6505 }
6506
6507 # vzdump restore implementaion
6508
6509 sub tar_archive_read_firstfile {
6510     my $archive = shift;
6511
6512     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6513
6514     # try to detect archive type first
6515     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6516         die "unable to open file '$archive'\n";
6517     my $firstfile = <$fh>;
6518     kill 15, $pid;
6519     close $fh;
6520
6521     die "ERROR: archive contaions no data\n" if !$firstfile;
6522     chomp $firstfile;
6523
6524     return $firstfile;
6525 }
6526
6527 sub tar_restore_cleanup {
6528     my ($storecfg, $statfile) = @_;
6529
6530     print STDERR "starting cleanup\n";
6531
6532     if (my $fd = IO::File->new($statfile, "r")) {
6533         while (defined(my $line = <$fd>)) {
6534             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6535                 my $volid = $2;
6536                 eval {
6537                     if ($volid =~ m|^/|) {
6538                         unlink $volid || die 'unlink failed\n';
6539                     } else {
6540                         PVE::Storage::vdisk_free($storecfg, $volid);
6541                     }
6542                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6543                 };
6544                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6545             } else {
6546                 print STDERR "unable to parse line in statfile - $line";
6547             }
6548         }
6549         $fd->close();
6550     }
6551 }
6552
6553 sub restore_file_archive {
6554     my ($archive, $vmid, $user, $opts) = @_;
6555
6556     return restore_vma_archive($archive, $vmid, $user, $opts)
6557         if $archive eq '-';
6558
6559     my $info = PVE::Storage::archive_info($archive);
6560     my $format = $opts->{format} // $info->{format};
6561     my $comp = $info->{compression};
6562
6563     # try to detect archive format
6564     if ($format eq 'tar') {
6565         return restore_tar_archive($archive, $vmid, $user, $opts);
6566     } else {
6567         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6568     }
6569 }
6570
6571 # hepler to remove disks that will not be used after restore
6572 my $restore_cleanup_oldconf = sub {
6573     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6574
6575     my $kept_disks = {};
6576
6577     PVE::QemuConfig->foreach_volume($oldconf, sub {
6578         my ($ds, $drive) = @_;
6579
6580         return if drive_is_cdrom($drive, 1);
6581
6582         my $volid = $drive->{file};
6583         return if !$volid || $volid =~ m|^/|;
6584
6585         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6586         return if !$path || !$owner || ($owner != $vmid);
6587
6588         # Note: only delete disk we want to restore
6589         # other volumes will become unused
6590         if ($virtdev_hash->{$ds}) {
6591             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6592             if (my $err = $@) {
6593                 warn $err;
6594             }
6595         } else {
6596             $kept_disks->{$volid} = 1;
6597         }
6598     });
6599
6600     # after the restore we have no snapshots anymore
6601     for my $snapname (keys $oldconf->{snapshots}->%*) {
6602         my $snap = $oldconf->{snapshots}->{$snapname};
6603         if ($snap->{vmstate}) {
6604             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6605             if (my $err = $@) {
6606                 warn $err;
6607             }
6608         }
6609
6610         for my $volid (keys $kept_disks->%*) {
6611             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6612             warn $@ if $@;
6613         }
6614     }
6615 };
6616
6617 # Helper to parse vzdump backup device hints
6618 #
6619 # $rpcenv: Environment, used to ckeck storage permissions
6620 # $user: User ID, to check storage permissions
6621 # $storecfg: Storage configuration
6622 # $fh: the file handle for reading the configuration
6623 # $devinfo: should contain device sizes for all backu-up'ed devices
6624 # $options: backup options (pool, default storage)
6625 #
6626 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6627 my $parse_backup_hints = sub {
6628     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6629
6630     my $check_storage = sub { # assert if an image can be allocate
6631         my ($storeid, $scfg) = @_;
6632         die "Content type 'images' is not available on storage '$storeid'\n"
6633             if !$scfg->{content}->{images};
6634         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6635             if $user ne 'root@pam';
6636     };
6637
6638     my $virtdev_hash = {};
6639     while (defined(my $line = <$fh>)) {
6640         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6641             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6642             die "archive does not contain data for drive '$virtdev'\n"
6643                 if !$devinfo->{$devname};
6644
6645             if (defined($options->{storage})) {
6646                 $storeid = $options->{storage} || 'local';
6647             } elsif (!$storeid) {
6648                 $storeid = 'local';
6649             }
6650             $format = 'raw' if !$format;
6651             $devinfo->{$devname}->{devname} = $devname;
6652             $devinfo->{$devname}->{virtdev} = $virtdev;
6653             $devinfo->{$devname}->{format} = $format;
6654             $devinfo->{$devname}->{storeid} = $storeid;
6655
6656             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6657             $check_storage->($storeid, $scfg); # permission and content type check
6658
6659             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6660         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6661             my $virtdev = $1;
6662             my $drive = parse_drive($virtdev, $2);
6663
6664             if (drive_is_cloudinit($drive)) {
6665                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6666                 $storeid = $options->{storage} if defined ($options->{storage});
6667                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6668                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6669
6670                 $check_storage->($storeid, $scfg); # permission and content type check
6671
6672                 $virtdev_hash->{$virtdev} = {
6673                     format => $format,
6674                     storeid => $storeid,
6675                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6676                     is_cloudinit => 1,
6677                 };
6678             }
6679         }
6680     }
6681
6682     return $virtdev_hash;
6683 };
6684
6685 # Helper to allocate and activate all volumes required for a restore
6686 #
6687 # $storecfg: Storage configuration
6688 # $virtdev_hash: as returned by parse_backup_hints()
6689 #
6690 # Returns: { $virtdev => $volid }
6691 my $restore_allocate_devices = sub {
6692     my ($storecfg, $virtdev_hash, $vmid) = @_;
6693
6694     my $map = {};
6695     foreach my $virtdev (sort keys %$virtdev_hash) {
6696         my $d = $virtdev_hash->{$virtdev};
6697         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6698         my $storeid = $d->{storeid};
6699         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6700
6701         # test if requested format is supported
6702         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6703         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6704         $d->{format} = $defFormat if !$supported;
6705
6706         my $name;
6707         if ($d->{is_cloudinit}) {
6708             $name = "vm-$vmid-cloudinit";
6709             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6710             if ($scfg->{path}) {
6711                 $name .= ".$d->{format}";
6712             }
6713         }
6714
6715         my $volid = PVE::Storage::vdisk_alloc(
6716             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6717
6718         print STDERR "new volume ID is '$volid'\n";
6719         $d->{volid} = $volid;
6720
6721         PVE::Storage::activate_volumes($storecfg, [$volid]);
6722
6723         $map->{$virtdev} = $volid;
6724     }
6725
6726     return $map;
6727 };
6728
6729 sub restore_update_config_line {
6730     my ($cookie, $map, $line, $unique) = @_;
6731
6732     return '' if $line =~ m/^\#qmdump\#/;
6733     return '' if $line =~ m/^\#vzdump\#/;
6734     return '' if $line =~ m/^lock:/;
6735     return '' if $line =~ m/^unused\d+:/;
6736     return '' if $line =~ m/^parent:/;
6737
6738     my $res = '';
6739
6740     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6741     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6742         # try to convert old 1.X settings
6743         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6744         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6745             my ($model, $macaddr) = split(/\=/, $devconfig);
6746             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6747             my $net = {
6748                 model => $model,
6749                 bridge => "vmbr$ind",
6750                 macaddr => $macaddr,
6751             };
6752             my $netstr = print_net($net);
6753
6754             $res .= "net$cookie->{netcount}: $netstr\n";
6755             $cookie->{netcount}++;
6756         }
6757     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6758         my ($id, $netstr) = ($1, $2);
6759         my $net = parse_net($netstr);
6760         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6761         $netstr = print_net($net);
6762         $res .= "$id: $netstr\n";
6763     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6764         my $virtdev = $1;
6765         my $value = $3;
6766         my $di = parse_drive($virtdev, $value);
6767         if (defined($di->{backup}) && !$di->{backup}) {
6768             $res .= "#$line";
6769         } elsif ($map->{$virtdev}) {
6770             delete $di->{format}; # format can change on restore
6771             $di->{file} = $map->{$virtdev};
6772             $value = print_drive($di);
6773             $res .= "$virtdev: $value\n";
6774         } else {
6775             $res .= $line;
6776         }
6777     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6778         my $vmgenid = $1;
6779         if ($vmgenid ne '0') {
6780             # always generate a new vmgenid if there was a valid one setup
6781             $vmgenid = generate_uuid();
6782         }
6783         $res .= "vmgenid: $vmgenid\n";
6784     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6785         my ($uuid, $uuid_str);
6786         UUID::generate($uuid);
6787         UUID::unparse($uuid, $uuid_str);
6788         my $smbios1 = parse_smbios1($2);
6789         $smbios1->{uuid} = $uuid_str;
6790         $res .= $1.print_smbios1($smbios1)."\n";
6791     } else {
6792         $res .= $line;
6793     }
6794
6795     return $res;
6796 }
6797
6798 my $restore_deactivate_volumes = sub {
6799     my ($storecfg, $virtdev_hash) = @_;
6800
6801     my $vollist = [];
6802     for my $dev (values $virtdev_hash->%*) {
6803         push $vollist->@*, $dev->{volid} if $dev->{volid};
6804     }
6805
6806     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6807     print STDERR $@ if $@;
6808 };
6809
6810 my $restore_destroy_volumes = sub {
6811     my ($storecfg, $virtdev_hash) = @_;
6812
6813     for my $dev (values $virtdev_hash->%*) {
6814         my $volid = $dev->{volid} or next;
6815         eval {
6816             PVE::Storage::vdisk_free($storecfg, $volid);
6817             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6818         };
6819         print STDERR "unable to cleanup '$volid' - $@" if $@;
6820     }
6821 };
6822
6823 my $restore_merge_config = sub {
6824     my ($filename, $backup_conf_raw, $override_conf) = @_;
6825
6826     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6827     for my $key (keys $override_conf->%*) {
6828         $backup_conf->{$key} = $override_conf->{$key};
6829     }
6830
6831     return $backup_conf;
6832 };
6833
6834 sub scan_volids {
6835     my ($cfg, $vmid) = @_;
6836
6837     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6838
6839     my $volid_hash = {};
6840     foreach my $storeid (keys %$info) {
6841         foreach my $item (@{$info->{$storeid}}) {
6842             next if !($item->{volid} && $item->{size});
6843             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6844             $volid_hash->{$item->{volid}} = $item;
6845         }
6846     }
6847
6848     return $volid_hash;
6849 }
6850
6851 sub update_disk_config {
6852     my ($vmid, $conf, $volid_hash) = @_;
6853
6854     my $changes;
6855     my $prefix = "VM $vmid";
6856
6857     # used and unused disks
6858     my $referenced = {};
6859
6860     # Note: it is allowed to define multiple storages with same path (alias), so
6861     # we need to check both 'volid' and real 'path' (two different volid can point
6862     # to the same path).
6863
6864     my $referencedpath = {};
6865
6866     # update size info
6867     PVE::QemuConfig->foreach_volume($conf, sub {
6868         my ($opt, $drive) = @_;
6869
6870         my $volid = $drive->{file};
6871         return if !$volid;
6872         my $volume = $volid_hash->{$volid};
6873
6874         # mark volid as "in-use" for next step
6875         $referenced->{$volid} = 1;
6876         if ($volume && (my $path = $volume->{path})) {
6877             $referencedpath->{$path} = 1;
6878         }
6879
6880         return if drive_is_cdrom($drive);
6881         return if !$volume;
6882
6883         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6884         if (defined($updated)) {
6885             $changes = 1;
6886             $conf->{$opt} = print_drive($updated);
6887             print "$prefix ($opt): $msg\n";
6888         }
6889     });
6890
6891     # remove 'unusedX' entry if volume is used
6892     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6893         my ($opt, $drive) = @_;
6894
6895         my $volid = $drive->{file};
6896         return if !$volid;
6897
6898         my $path;
6899         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6900         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6901             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6902             $changes = 1;
6903             delete $conf->{$opt};
6904         }
6905
6906         $referenced->{$volid} = 1;
6907         $referencedpath->{$path} = 1 if $path;
6908     });
6909
6910     foreach my $volid (sort keys %$volid_hash) {
6911         next if $volid =~ m/vm-$vmid-state-/;
6912         next if $referenced->{$volid};
6913         my $path = $volid_hash->{$volid}->{path};
6914         next if !$path; # just to be sure
6915         next if $referencedpath->{$path};
6916         $changes = 1;
6917         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6918         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6919         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6920     }
6921
6922     return $changes;
6923 }
6924
6925 sub rescan {
6926     my ($vmid, $nolock, $dryrun) = @_;
6927
6928     my $cfg = PVE::Storage::config();
6929
6930     print "rescan volumes...\n";
6931     my $volid_hash = scan_volids($cfg, $vmid);
6932
6933     my $updatefn =  sub {
6934         my ($vmid) = @_;
6935
6936         my $conf = PVE::QemuConfig->load_config($vmid);
6937
6938         PVE::QemuConfig->check_lock($conf);
6939
6940         my $vm_volids = {};
6941         foreach my $volid (keys %$volid_hash) {
6942             my $info = $volid_hash->{$volid};
6943             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6944         }
6945
6946         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6947
6948         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6949     };
6950
6951     if (defined($vmid)) {
6952         if ($nolock) {
6953             &$updatefn($vmid);
6954         } else {
6955             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6956         }
6957     } else {
6958         my $vmlist = config_list();
6959         foreach my $vmid (keys %$vmlist) {
6960             if ($nolock) {
6961                 &$updatefn($vmid);
6962             } else {
6963                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6964             }
6965         }
6966     }
6967 }
6968
6969 sub restore_proxmox_backup_archive {
6970     my ($archive, $vmid, $user, $options) = @_;
6971
6972     my $storecfg = PVE::Storage::config();
6973
6974     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6975     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6976
6977     my $fingerprint = $scfg->{fingerprint};
6978     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6979
6980     my $repo = PVE::PBSClient::get_repository($scfg);
6981     my $namespace = $scfg->{namespace};
6982
6983     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6984     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6985     local $ENV{PBS_PASSWORD} = $password;
6986     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6987
6988     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6989         PVE::Storage::parse_volname($storecfg, $archive);
6990
6991     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6992
6993     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6994
6995     my $tmpdir = "/var/tmp/vzdumptmp$$";
6996     rmtree $tmpdir;
6997     mkpath $tmpdir;
6998
6999     my $conffile = PVE::QemuConfig->config_file($vmid);
7000      # disable interrupts (always do cleanups)
7001     local $SIG{INT} =
7002         local $SIG{TERM} =
7003         local $SIG{QUIT} =
7004         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7005
7006     # Note: $oldconf is undef if VM does not exists
7007     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7008     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7009     my $new_conf_raw = '';
7010
7011     my $rpcenv = PVE::RPCEnvironment::get();
7012     my $devinfo = {}; # info about drives included in backup
7013     my $virtdev_hash = {}; # info about allocated drives
7014
7015     eval {
7016         # enable interrupts
7017         local $SIG{INT} =
7018             local $SIG{TERM} =
7019             local $SIG{QUIT} =
7020             local $SIG{HUP} =
7021             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7022
7023         my $cfgfn = "$tmpdir/qemu-server.conf";
7024         my $firewall_config_fn = "$tmpdir/fw.conf";
7025         my $index_fn = "$tmpdir/index.json";
7026
7027         my $cmd = "restore";
7028
7029         my $param = [$pbs_backup_name, "index.json", $index_fn];
7030         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7031         my $index = PVE::Tools::file_get_contents($index_fn);
7032         $index = decode_json($index);
7033
7034         foreach my $info (@{$index->{files}}) {
7035             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7036                 my $devname = $1;
7037                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7038                     $devinfo->{$devname}->{size} = $1;
7039                 } else {
7040                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7041                 }
7042             }
7043         }
7044
7045         my $is_qemu_server_backup = scalar(
7046             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7047         );
7048         if (!$is_qemu_server_backup) {
7049             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7050         }
7051         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7052
7053         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7054         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7055
7056         if ($has_firewall_config) {
7057             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7058             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7059
7060             my $pve_firewall_dir = '/etc/pve/firewall';
7061             mkdir $pve_firewall_dir; # make sure the dir exists
7062             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7063         }
7064
7065         my $fh = IO::File->new($cfgfn, "r") ||
7066             die "unable to read qemu-server.conf - $!\n";
7067
7068         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7069
7070         # fixme: rate limit?
7071
7072         # create empty/temp config
7073         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7074
7075         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7076
7077         # allocate volumes
7078         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7079
7080         foreach my $virtdev (sort keys %$virtdev_hash) {
7081             my $d = $virtdev_hash->{$virtdev};
7082             next if $d->{is_cloudinit}; # no need to restore cloudinit
7083
7084             # this fails if storage is unavailable
7085             my $volid = $d->{volid};
7086             my $path = PVE::Storage::path($storecfg, $volid);
7087
7088             # for live-restore we only want to preload the efidisk and TPM state
7089             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7090
7091             my @ns_arg;
7092             if (defined(my $ns = $scfg->{namespace})) {
7093                 @ns_arg = ('--ns', $ns);
7094             }
7095
7096             my $pbs_restore_cmd = [
7097                 '/usr/bin/pbs-restore',
7098                 '--repository', $repo,
7099                 @ns_arg,
7100                 $pbs_backup_name,
7101                 "$d->{devname}.img.fidx",
7102                 $path,
7103                 '--verbose',
7104                 ];
7105
7106             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7107             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7108
7109             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7110                 push @$pbs_restore_cmd, '--skip-zero';
7111             }
7112
7113             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7114             print "restore proxmox backup image: $dbg_cmdstring\n";
7115             run_command($pbs_restore_cmd);
7116         }
7117
7118         $fh->seek(0, 0) || die "seek failed - $!\n";
7119
7120         my $cookie = { netcount => 0 };
7121         while (defined(my $line = <$fh>)) {
7122             $new_conf_raw .= restore_update_config_line(
7123                 $cookie,
7124                 $map,
7125                 $line,
7126                 $options->{unique},
7127             );
7128         }
7129
7130         $fh->close();
7131     };
7132     my $err = $@;
7133
7134     if ($err || !$options->{live}) {
7135         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7136     }
7137
7138     rmtree $tmpdir;
7139
7140     if ($err) {
7141         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7142         die $err;
7143     }
7144
7145     if ($options->{live}) {
7146         # keep lock during live-restore
7147         $new_conf_raw .= "\nlock: create";
7148     }
7149
7150     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7151     PVE::QemuConfig->write_config($vmid, $new_conf);
7152
7153     eval { rescan($vmid, 1); };
7154     warn $@ if $@;
7155
7156     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7157
7158     if ($options->{live}) {
7159         # enable interrupts
7160         local $SIG{INT} =
7161             local $SIG{TERM} =
7162             local $SIG{QUIT} =
7163             local $SIG{HUP} =
7164             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7165
7166         my $conf = PVE::QemuConfig->load_config($vmid);
7167         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7168
7169         # these special drives are already restored before start
7170         delete $devinfo->{'drive-efidisk0'};
7171         delete $devinfo->{'drive-tpmstate0-backup'};
7172
7173         my $pbs_opts = {
7174             repo => $repo,
7175             keyfile => $keyfile,
7176             snapshot => $pbs_backup_name,
7177             namespace => $namespace,
7178         };
7179         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7180
7181         PVE::QemuConfig->remove_lock($vmid, "create");
7182     }
7183 }
7184
7185 sub pbs_live_restore {
7186     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7187
7188     print "starting VM for live-restore\n";
7189     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7190
7191     my $pbs_backing = {};
7192     for my $ds (keys %$restored_disks) {
7193         $ds =~ m/^drive-(.*)$/;
7194         my $confname = $1;
7195         $pbs_backing->{$confname} = {
7196             repository => $opts->{repo},
7197             snapshot => $opts->{snapshot},
7198             archive => "$ds.img.fidx",
7199         };
7200         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7201         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7202
7203         my $drive = parse_drive($confname, $conf->{$confname});
7204         print "restoring '$ds' to '$drive->{file}'\n";
7205     }
7206
7207     my $drives_streamed = 0;
7208     eval {
7209         # make sure HA doesn't interrupt our restore by stopping the VM
7210         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7211             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7212         }
7213
7214         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7215         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7216         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7217
7218         my $qmeventd_fd = register_qmeventd_handle($vmid);
7219
7220         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7221         # this will effectively collapse the backing image chain consisting of
7222         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7223         # removes itself once all backing images vanish with 'auto-remove=on')
7224         my $jobs = {};
7225         for my $ds (sort keys %$restored_disks) {
7226             my $job_id = "restore-$ds";
7227             mon_cmd($vmid, 'block-stream',
7228                 'job-id' => $job_id,
7229                 device => "$ds",
7230             );
7231             $jobs->{$job_id} = {};
7232         }
7233
7234         mon_cmd($vmid, 'cont');
7235         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7236
7237         print "restore-drive jobs finished successfully, removing all tracking block devices"
7238             ." to disconnect from Proxmox Backup Server\n";
7239
7240         for my $ds (sort keys %$restored_disks) {
7241             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7242         }
7243
7244         close($qmeventd_fd);
7245     };
7246
7247     my $err = $@;
7248
7249     if ($err) {
7250         warn "An error occurred during live-restore: $err\n";
7251         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7252         die "live-restore failed\n";
7253     }
7254 }
7255
7256 sub restore_vma_archive {
7257     my ($archive, $vmid, $user, $opts, $comp) = @_;
7258
7259     my $readfrom = $archive;
7260
7261     my $cfg = PVE::Storage::config();
7262     my $commands = [];
7263     my $bwlimit = $opts->{bwlimit};
7264
7265     my $dbg_cmdstring = '';
7266     my $add_pipe = sub {
7267         my ($cmd) = @_;
7268         push @$commands, $cmd;
7269         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7270         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7271         $readfrom = '-';
7272     };
7273
7274     my $input = undef;
7275     if ($archive eq '-') {
7276         $input = '<&STDIN';
7277     } else {
7278         # If we use a backup from a PVE defined storage we also consider that
7279         # storage's rate limit:
7280         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7281         if (defined($volid)) {
7282             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7283             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7284             if ($readlimit) {
7285                 print STDERR "applying read rate limit: $readlimit\n";
7286                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7287                 $add_pipe->($cstream);
7288             }
7289         }
7290     }
7291
7292     if ($comp) {
7293         my $info = PVE::Storage::decompressor_info('vma', $comp);
7294         my $cmd = $info->{decompressor};
7295         push @$cmd, $readfrom;
7296         $add_pipe->($cmd);
7297     }
7298
7299     my $tmpdir = "/var/tmp/vzdumptmp$$";
7300     rmtree $tmpdir;
7301
7302     # disable interrupts (always do cleanups)
7303     local $SIG{INT} =
7304         local $SIG{TERM} =
7305         local $SIG{QUIT} =
7306         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7307
7308     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7309     POSIX::mkfifo($mapfifo, 0600);
7310     my $fifofh;
7311     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7312
7313     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7314
7315     my $oldtimeout;
7316     my $timeout = 5;
7317
7318     my $devinfo = {}; # info about drives included in backup
7319     my $virtdev_hash = {}; # info about allocated drives
7320
7321     my $rpcenv = PVE::RPCEnvironment::get();
7322
7323     my $conffile = PVE::QemuConfig->config_file($vmid);
7324
7325     # Note: $oldconf is undef if VM does not exist
7326     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7327     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7328     my $new_conf_raw = '';
7329
7330     my %storage_limits;
7331
7332     my $print_devmap = sub {
7333         my $cfgfn = "$tmpdir/qemu-server.conf";
7334
7335         # we can read the config - that is already extracted
7336         my $fh = IO::File->new($cfgfn, "r") ||
7337             die "unable to read qemu-server.conf - $!\n";
7338
7339         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7340         if (-f $fwcfgfn) {
7341             my $pve_firewall_dir = '/etc/pve/firewall';
7342             mkdir $pve_firewall_dir; # make sure the dir exists
7343             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7344         }
7345
7346         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7347
7348         foreach my $info (values %{$virtdev_hash}) {
7349             my $storeid = $info->{storeid};
7350             next if defined($storage_limits{$storeid});
7351
7352             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7353             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7354             $storage_limits{$storeid} = $limit * 1024;
7355         }
7356
7357         foreach my $devname (keys %$devinfo) {
7358             die "found no device mapping information for device '$devname'\n"
7359                 if !$devinfo->{$devname}->{virtdev};
7360         }
7361
7362         # create empty/temp config
7363         if ($oldconf) {
7364             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7365             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7366         }
7367
7368         # allocate volumes
7369         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7370
7371         # print restore information to $fifofh
7372         foreach my $virtdev (sort keys %$virtdev_hash) {
7373             my $d = $virtdev_hash->{$virtdev};
7374             next if $d->{is_cloudinit}; # no need to restore cloudinit
7375
7376             my $storeid = $d->{storeid};
7377             my $volid = $d->{volid};
7378
7379             my $map_opts = '';
7380             if (my $limit = $storage_limits{$storeid}) {
7381                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7382             }
7383
7384             my $write_zeros = 1;
7385             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7386                 $write_zeros = 0;
7387             }
7388
7389             my $path = PVE::Storage::path($cfg, $volid);
7390
7391             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7392
7393             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7394         }
7395
7396         $fh->seek(0, 0) || die "seek failed - $!\n";
7397
7398         my $cookie = { netcount => 0 };
7399         while (defined(my $line = <$fh>)) {
7400             $new_conf_raw .= restore_update_config_line(
7401                 $cookie,
7402                 $map,
7403                 $line,
7404                 $opts->{unique},
7405             );
7406         }
7407
7408         $fh->close();
7409     };
7410
7411     eval {
7412         # enable interrupts
7413         local $SIG{INT} =
7414             local $SIG{TERM} =
7415             local $SIG{QUIT} =
7416             local $SIG{HUP} =
7417             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7418         local $SIG{ALRM} = sub { die "got timeout\n"; };
7419
7420         $oldtimeout = alarm($timeout);
7421
7422         my $parser = sub {
7423             my $line = shift;
7424
7425             print "$line\n";
7426
7427             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7428                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7429                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7430             } elsif ($line =~ m/^CTIME: /) {
7431                 # we correctly received the vma config, so we can disable
7432                 # the timeout now for disk allocation (set to 10 minutes, so
7433                 # that we always timeout if something goes wrong)
7434                 alarm(600);
7435                 &$print_devmap();
7436                 print $fifofh "done\n";
7437                 my $tmp = $oldtimeout || 0;
7438                 $oldtimeout = undef;
7439                 alarm($tmp);
7440                 close($fifofh);
7441                 $fifofh = undef;
7442             }
7443         };
7444
7445         print "restore vma archive: $dbg_cmdstring\n";
7446         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7447     };
7448     my $err = $@;
7449
7450     alarm($oldtimeout) if $oldtimeout;
7451
7452     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7453
7454     close($fifofh) if $fifofh;
7455     unlink $mapfifo;
7456     rmtree $tmpdir;
7457
7458     if ($err) {
7459         $restore_destroy_volumes->($cfg, $virtdev_hash);
7460         die $err;
7461     }
7462
7463     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7464     PVE::QemuConfig->write_config($vmid, $new_conf);
7465
7466     eval { rescan($vmid, 1); };
7467     warn $@ if $@;
7468
7469     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7470 }
7471
7472 sub restore_tar_archive {
7473     my ($archive, $vmid, $user, $opts) = @_;
7474
7475     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7476         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7477         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7478     }
7479
7480     if ($archive ne '-') {
7481         my $firstfile = tar_archive_read_firstfile($archive);
7482         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7483             if $firstfile ne 'qemu-server.conf';
7484     }
7485
7486     my $storecfg = PVE::Storage::config();
7487
7488     # avoid zombie disks when restoring over an existing VM -> cleanup first
7489     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7490     # skiplock=1 because qmrestore has set the 'create' lock itself already
7491     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7492     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7493
7494     my $tocmd = "/usr/lib/qemu-server/qmextract";
7495
7496     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7497     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7498     $tocmd .= ' --prealloc' if $opts->{prealloc};
7499     $tocmd .= ' --info' if $opts->{info};
7500
7501     # tar option "xf" does not autodetect compression when read from STDIN,
7502     # so we pipe to zcat
7503     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7504         PVE::Tools::shellquote("--to-command=$tocmd");
7505
7506     my $tmpdir = "/var/tmp/vzdumptmp$$";
7507     mkpath $tmpdir;
7508
7509     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7510     local $ENV{VZDUMP_VMID} = $vmid;
7511     local $ENV{VZDUMP_USER} = $user;
7512
7513     my $conffile = PVE::QemuConfig->config_file($vmid);
7514     my $new_conf_raw = '';
7515
7516     # disable interrupts (always do cleanups)
7517     local $SIG{INT} =
7518         local $SIG{TERM} =
7519         local $SIG{QUIT} =
7520         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7521
7522     eval {
7523         # enable interrupts
7524         local $SIG{INT} =
7525             local $SIG{TERM} =
7526             local $SIG{QUIT} =
7527             local $SIG{HUP} =
7528             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7529
7530         if ($archive eq '-') {
7531             print "extracting archive from STDIN\n";
7532             run_command($cmd, input => "<&STDIN");
7533         } else {
7534             print "extracting archive '$archive'\n";
7535             run_command($cmd);
7536         }
7537
7538         return if $opts->{info};
7539
7540         # read new mapping
7541         my $map = {};
7542         my $statfile = "$tmpdir/qmrestore.stat";
7543         if (my $fd = IO::File->new($statfile, "r")) {
7544             while (defined (my $line = <$fd>)) {
7545                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7546                     $map->{$1} = $2 if $1;
7547                 } else {
7548                     print STDERR "unable to parse line in statfile - $line\n";
7549                 }
7550             }
7551             $fd->close();
7552         }
7553
7554         my $confsrc = "$tmpdir/qemu-server.conf";
7555
7556         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7557
7558         my $cookie = { netcount => 0 };
7559         while (defined (my $line = <$srcfd>)) {
7560             $new_conf_raw .= restore_update_config_line(
7561                 $cookie,
7562                 $map,
7563                 $line,
7564                 $opts->{unique},
7565             );
7566         }
7567
7568         $srcfd->close();
7569     };
7570     if (my $err = $@) {
7571         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7572         die $err;
7573     }
7574
7575     rmtree $tmpdir;
7576
7577     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7578
7579     PVE::Cluster::cfs_update(); # make sure we read new file
7580
7581     eval { rescan($vmid, 1); };
7582     warn $@ if $@;
7583 };
7584
7585 sub foreach_storage_used_by_vm {
7586     my ($conf, $func) = @_;
7587
7588     my $sidhash = {};
7589
7590     PVE::QemuConfig->foreach_volume($conf, sub {
7591         my ($ds, $drive) = @_;
7592         return if drive_is_cdrom($drive);
7593
7594         my $volid = $drive->{file};
7595
7596         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7597         $sidhash->{$sid} = $sid if $sid;
7598     });
7599
7600     foreach my $sid (sort keys %$sidhash) {
7601         &$func($sid);
7602     }
7603 }
7604
7605 my $qemu_snap_storage = {
7606     rbd => 1,
7607 };
7608 sub do_snapshots_with_qemu {
7609     my ($storecfg, $volid, $deviceid) = @_;
7610
7611     return if $deviceid =~ m/tpmstate0/;
7612
7613     my $storage_name = PVE::Storage::parse_volume_id($volid);
7614     my $scfg = $storecfg->{ids}->{$storage_name};
7615     die "could not find storage '$storage_name'\n" if !defined($scfg);
7616
7617     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7618         return 1;
7619     }
7620
7621     if ($volid =~ m/\.(qcow2|qed)$/){
7622         return 1;
7623     }
7624
7625     return;
7626 }
7627
7628 sub qga_check_running {
7629     my ($vmid, $nowarn) = @_;
7630
7631     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7632     if ($@) {
7633         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7634         return 0;
7635     }
7636     return 1;
7637 }
7638
7639 sub template_create {
7640     my ($vmid, $conf, $disk) = @_;
7641
7642     my $storecfg = PVE::Storage::config();
7643
7644     PVE::QemuConfig->foreach_volume($conf, sub {
7645         my ($ds, $drive) = @_;
7646
7647         return if drive_is_cdrom($drive);
7648         return if $disk && $ds ne $disk;
7649
7650         my $volid = $drive->{file};
7651         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7652
7653         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7654         $drive->{file} = $voliddst;
7655         $conf->{$ds} = print_drive($drive);
7656         PVE::QemuConfig->write_config($vmid, $conf);
7657     });
7658 }
7659
7660 sub convert_iscsi_path {
7661     my ($path) = @_;
7662
7663     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7664         my $portal = $1;
7665         my $target = $2;
7666         my $lun = $3;
7667
7668         my $initiator_name = get_initiator_name();
7669
7670         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7671                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7672     }
7673
7674     die "cannot convert iscsi path '$path', unkown format\n";
7675 }
7676
7677 sub qemu_img_convert {
7678     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7679
7680     my $storecfg = PVE::Storage::config();
7681     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7682     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7683
7684     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7685
7686     my $cachemode;
7687     my $src_path;
7688     my $src_is_iscsi = 0;
7689     my $src_format;
7690
7691     if ($src_storeid) {
7692         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7693         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7694         $src_format = qemu_img_format($src_scfg, $src_volname);
7695         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7696         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7697         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7698     } elsif (-f $src_volid || -b $src_volid) {
7699         $src_path = $src_volid;
7700         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7701             $src_format = $1;
7702         }
7703     }
7704
7705     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7706
7707     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7708     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7709     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7710     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7711
7712     my $cmd = [];
7713     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7714     push @$cmd, '-l', "snapshot.name=$snapname"
7715         if $snapname && $src_format && $src_format eq "qcow2";
7716     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7717     push @$cmd, '-T', $cachemode if defined($cachemode);
7718     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7719
7720     if ($src_is_iscsi) {
7721         push @$cmd, '--image-opts';
7722         $src_path = convert_iscsi_path($src_path);
7723     } elsif ($src_format) {
7724         push @$cmd, '-f', $src_format;
7725     }
7726
7727     if ($dst_is_iscsi) {
7728         push @$cmd, '--target-image-opts';
7729         $dst_path = convert_iscsi_path($dst_path);
7730     } else {
7731         push @$cmd, '-O', $dst_format;
7732     }
7733
7734     push @$cmd, $src_path;
7735
7736     if (!$dst_is_iscsi && $is_zero_initialized) {
7737         push @$cmd, "zeroinit:$dst_path";
7738     } else {
7739         push @$cmd, $dst_path;
7740     }
7741
7742     my $parser = sub {
7743         my $line = shift;
7744         if($line =~ m/\((\S+)\/100\%\)/){
7745             my $percent = $1;
7746             my $transferred = int($size * $percent / 100);
7747             my $total_h = render_bytes($size, 1);
7748             my $transferred_h = render_bytes($transferred, 1);
7749
7750             print "transferred $transferred_h of $total_h ($percent%)\n";
7751         }
7752
7753     };
7754
7755     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7756     my $err = $@;
7757     die "copy failed: $err" if $err;
7758 }
7759
7760 sub qemu_img_format {
7761     my ($scfg, $volname) = @_;
7762
7763     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7764         return $1;
7765     } else {
7766         return "raw";
7767     }
7768 }
7769
7770 sub qemu_drive_mirror {
7771     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7772
7773     $jobs = {} if !$jobs;
7774
7775     my $qemu_target;
7776     my $format;
7777     $jobs->{"drive-$drive"} = {};
7778
7779     if ($dst_volid =~ /^nbd:/) {
7780         $qemu_target = $dst_volid;
7781         $format = "nbd";
7782     } else {
7783         my $storecfg = PVE::Storage::config();
7784         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7785
7786         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7787
7788         $format = qemu_img_format($dst_scfg, $dst_volname);
7789
7790         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7791
7792         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7793     }
7794
7795     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7796     $opts->{format} = $format if $format;
7797
7798     if (defined($src_bitmap)) {
7799         $opts->{sync} = 'incremental';
7800         $opts->{bitmap} = $src_bitmap;
7801         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7802     }
7803
7804     if (defined($bwlimit)) {
7805         $opts->{speed} = $bwlimit * 1024;
7806         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7807     } else {
7808         print "drive mirror is starting for drive-$drive\n";
7809     }
7810
7811     # if a job already runs for this device we get an error, catch it for cleanup
7812     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7813     if (my $err = $@) {
7814         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7815         warn "$@\n" if $@;
7816         die "mirroring error: $err\n";
7817     }
7818
7819     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7820 }
7821
7822 # $completion can be either
7823 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7824 # 'cancel': wait until all jobs are ready, block-job-cancel them
7825 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7826 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7827 sub qemu_drive_mirror_monitor {
7828     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7829
7830     $completion //= 'complete';
7831     $op //= "mirror";
7832
7833     eval {
7834         my $err_complete = 0;
7835
7836         my $starttime = time ();
7837         while (1) {
7838             die "block job ('$op') timed out\n" if $err_complete > 300;
7839
7840             my $stats = mon_cmd($vmid, "query-block-jobs");
7841             my $ctime = time();
7842
7843             my $running_jobs = {};
7844             for my $stat (@$stats) {
7845                 next if $stat->{type} ne $op;
7846                 $running_jobs->{$stat->{device}} = $stat;
7847             }
7848
7849             my $readycounter = 0;
7850
7851             for my $job_id (sort keys %$jobs) {
7852                 my $job = $running_jobs->{$job_id};
7853
7854                 my $vanished = !defined($job);
7855                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7856                 if($complete || ($vanished && $completion eq 'auto')) {
7857                     print "$job_id: $op-job finished\n";
7858                     delete $jobs->{$job_id};
7859                     next;
7860                 }
7861
7862                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7863
7864                 my $busy = $job->{busy};
7865                 my $ready = $job->{ready};
7866                 if (my $total = $job->{len}) {
7867                     my $transferred = $job->{offset} || 0;
7868                     my $remaining = $total - $transferred;
7869                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7870
7871                     my $duration = $ctime - $starttime;
7872                     my $total_h = render_bytes($total, 1);
7873                     my $transferred_h = render_bytes($transferred, 1);
7874
7875                     my $status = sprintf(
7876                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7877                         render_duration($duration),
7878                     );
7879
7880                     if ($ready) {
7881                         if ($busy) {
7882                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7883                         } else {
7884                             $status .= ", ready";
7885                         }
7886                     }
7887                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7888                     $jobs->{$job_id}->{ready} = $ready;
7889                 }
7890
7891                 $readycounter++ if $job->{ready};
7892             }
7893
7894             last if scalar(keys %$jobs) == 0;
7895
7896             if ($readycounter == scalar(keys %$jobs)) {
7897                 print "all '$op' jobs are ready\n";
7898
7899                 # do the complete later (or has already been done)
7900                 last if $completion eq 'skip' || $completion eq 'auto';
7901
7902                 if ($vmiddst && $vmiddst != $vmid) {
7903                     my $agent_running = $qga && qga_check_running($vmid);
7904                     if ($agent_running) {
7905                         print "freeze filesystem\n";
7906                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7907                         warn $@ if $@;
7908                     } else {
7909                         print "suspend vm\n";
7910                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7911                         warn $@ if $@;
7912                     }
7913
7914                     # if we clone a disk for a new target vm, we don't switch the disk
7915                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7916
7917                     if ($agent_running) {
7918                         print "unfreeze filesystem\n";
7919                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7920                         warn $@ if $@;
7921                     } else {
7922                         print "resume vm\n";
7923                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7924                         warn $@ if $@;
7925                     }
7926
7927                     last;
7928                 } else {
7929
7930                     for my $job_id (sort keys %$jobs) {
7931                         # try to switch the disk if source and destination are on the same guest
7932                         print "$job_id: Completing block job_id...\n";
7933
7934                         my $op;
7935                         if ($completion eq 'complete') {
7936                             $op = 'block-job-complete';
7937                         } elsif ($completion eq 'cancel') {
7938                             $op = 'block-job-cancel';
7939                         } else {
7940                             die "invalid completion value: $completion\n";
7941                         }
7942                         eval { mon_cmd($vmid, $op, device => $job_id) };
7943                         if ($@ =~ m/cannot be completed/) {
7944                             print "$job_id: block job cannot be completed, trying again.\n";
7945                             $err_complete++;
7946                         }else {
7947                             print "$job_id: Completed successfully.\n";
7948                             $jobs->{$job_id}->{complete} = 1;
7949                         }
7950                     }
7951                 }
7952             }
7953             sleep 1;
7954         }
7955     };
7956     my $err = $@;
7957
7958     if ($err) {
7959         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7960         die "block job ($op) error: $err";
7961     }
7962 }
7963
7964 sub qemu_blockjobs_cancel {
7965     my ($vmid, $jobs) = @_;
7966
7967     foreach my $job (keys %$jobs) {
7968         print "$job: Cancelling block job\n";
7969         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7970         $jobs->{$job}->{cancel} = 1;
7971     }
7972
7973     while (1) {
7974         my $stats = mon_cmd($vmid, "query-block-jobs");
7975
7976         my $running_jobs = {};
7977         foreach my $stat (@$stats) {
7978             $running_jobs->{$stat->{device}} = $stat;
7979         }
7980
7981         foreach my $job (keys %$jobs) {
7982
7983             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7984                 print "$job: Done.\n";
7985                 delete $jobs->{$job};
7986             }
7987         }
7988
7989         last if scalar(keys %$jobs) == 0;
7990
7991         sleep 1;
7992     }
7993 }
7994
7995 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7996 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7997 my sub clone_disk_check_io_uring {
7998     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7999
8000     return if !$use_drive_mirror;
8001
8002     # Don't complain when not changing storage.
8003     # Assume if it works for the source, it'll work for the target too.
8004     return if $src_storeid eq $dst_storeid;
8005
8006     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8007     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8008
8009     my $cache_direct = drive_uses_cache_direct($src_drive);
8010
8011     my $src_uses_io_uring;
8012     if ($src_drive->{aio}) {
8013         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8014     } else {
8015         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8016     }
8017
8018     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8019         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8020 }
8021
8022 sub clone_disk {
8023     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8024
8025     my ($vmid, $running) = $source->@{qw(vmid running)};
8026     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8027
8028     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8029     my ($storage, $format) = $dest->@{qw(storage format)};
8030
8031     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8032
8033     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8034         die "cloning from/to EFI disk requires EFI disk\n"
8035             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8036         die "cloning from/to TPM state requires TPM state\n"
8037             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8038
8039         # This would lead to two device nodes in QEMU pointing to the same backing image!
8040         die "cannot change drive name when cloning disk from/to the same VM\n"
8041             if $use_drive_mirror && $vmid == $newvmid;
8042     }
8043
8044     die "cannot move TPM state while VM is running\n"
8045         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8046
8047     my $newvolid;
8048
8049     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8050     print "$src_drivename " if $src_drivename;
8051     print "($drive->{file})\n";
8052
8053     if (!$full) {
8054         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8055         push @$newvollist, $newvolid;
8056     } else {
8057         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8058         my $storeid = $storage || $src_storeid;
8059
8060         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8061
8062         my $name = undef;
8063         my $size = undef;
8064         if (drive_is_cloudinit($drive)) {
8065             $name = "vm-$newvmid-cloudinit";
8066             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8067             if ($scfg->{path}) {
8068                 $name .= ".$dst_format";
8069             }
8070             $snapname = undef;
8071             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8072         } elsif ($dst_drivename eq 'efidisk0') {
8073             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8074         } elsif ($dst_drivename eq 'tpmstate0') {
8075             $dst_format = 'raw';
8076             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8077         } else {
8078             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8079
8080             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8081         }
8082         $newvolid = PVE::Storage::vdisk_alloc(
8083             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8084         );
8085         push @$newvollist, $newvolid;
8086
8087         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8088
8089         if (drive_is_cloudinit($drive)) {
8090             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8091             # if this is the case, we have to complete any block-jobs still there from
8092             # previous drive-mirrors
8093             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8094                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8095             }
8096             goto no_data_clone;
8097         }
8098
8099         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8100         if ($use_drive_mirror) {
8101             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8102                 $completion, $qga, $bwlimit);
8103         } else {
8104             if ($dst_drivename eq 'efidisk0') {
8105                 # the relevant data on the efidisk may be smaller than the source
8106                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8107                 # that is given by the OVMF_VARS.fd
8108                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8109                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8110
8111                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8112
8113                 # better for Ceph if block size is not too small, see bug #3324
8114                 my $bs = 1024*1024;
8115
8116                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8117
8118                 if ($src_format eq 'qcow2' && $snapname) {
8119                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8120                         if !min_version(kvm_user_version(), 6, 2);
8121                     push $cmd->@*, '-l', $snapname;
8122                 }
8123                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8124                 run_command($cmd);
8125             } else {
8126                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8127             }
8128         }
8129     }
8130
8131 no_data_clone:
8132     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8133
8134     my $disk = dclone($drive);
8135     delete $disk->{format};
8136     $disk->{file} = $newvolid;
8137     $disk->{size} = $size if defined($size);
8138
8139     return $disk;
8140 }
8141
8142 sub get_running_qemu_version {
8143     my ($vmid) = @_;
8144     my $res = mon_cmd($vmid, "query-version");
8145     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8146 }
8147
8148 sub qemu_use_old_bios_files {
8149     my ($machine_type) = @_;
8150
8151     return if !$machine_type;
8152
8153     my $use_old_bios_files = undef;
8154
8155     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8156         $machine_type = $1;
8157         $use_old_bios_files = 1;
8158     } else {
8159         my $version = extract_version($machine_type, kvm_user_version());
8160         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8161         # load new efi bios files on migration. So this hack is required to allow
8162         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8163         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8164         $use_old_bios_files = !min_version($version, 2, 4);
8165     }
8166
8167     return ($use_old_bios_files, $machine_type);
8168 }
8169
8170 sub get_efivars_size {
8171     my ($conf, $efidisk) = @_;
8172
8173     my $arch = get_vm_arch($conf);
8174     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8175     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8176     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8177     return -s $ovmf_vars;
8178 }
8179
8180 sub update_efidisk_size {
8181     my ($conf) = @_;
8182
8183     return if !defined($conf->{efidisk0});
8184
8185     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8186     $disk->{size} = get_efivars_size($conf);
8187     $conf->{efidisk0} = print_drive($disk);
8188
8189     return;
8190 }
8191
8192 sub update_tpmstate_size {
8193     my ($conf) = @_;
8194
8195     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8196     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8197     $conf->{tpmstate0} = print_drive($disk);
8198 }
8199
8200 sub create_efidisk($$$$$$$) {
8201     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8202
8203     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8204
8205     my $vars_size_b = -s $ovmf_vars;
8206     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8207     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8208     PVE::Storage::activate_volumes($storecfg, [$volid]);
8209
8210     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8211     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8212
8213     return ($volid, $size/1024);
8214 }
8215
8216 sub vm_iothreads_list {
8217     my ($vmid) = @_;
8218
8219     my $res = mon_cmd($vmid, 'query-iothreads');
8220
8221     my $iothreads = {};
8222     foreach my $iothread (@$res) {
8223         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8224     }
8225
8226     return $iothreads;
8227 }
8228
8229 sub scsihw_infos {
8230     my ($conf, $drive) = @_;
8231
8232     my $maxdev = 0;
8233
8234     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8235         $maxdev = 7;
8236     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8237         $maxdev = 1;
8238     } else {
8239         $maxdev = 256;
8240     }
8241
8242     my $controller = int($drive->{index} / $maxdev);
8243     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8244         ? "virtioscsi"
8245         : "scsihw";
8246
8247     return ($maxdev, $controller, $controller_prefix);
8248 }
8249
8250 sub resolve_dst_disk_format {
8251         my ($storecfg, $storeid, $src_volname, $format) = @_;
8252         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8253
8254         if (!$format) {
8255             # if no target format is specified, use the source disk format as hint
8256             if ($src_volname) {
8257                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8258                 $format = qemu_img_format($scfg, $src_volname);
8259             } else {
8260                 return $defFormat;
8261             }
8262         }
8263
8264         # test if requested format is supported - else use default
8265         my $supported = grep { $_ eq $format } @$validFormats;
8266         $format = $defFormat if !$supported;
8267         return $format;
8268 }
8269
8270 # NOTE: if this logic changes, please update docs & possibly gui logic
8271 sub find_vmstate_storage {
8272     my ($conf, $storecfg) = @_;
8273
8274     # first, return storage from conf if set
8275     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8276
8277     my ($target, $shared, $local);
8278
8279     foreach_storage_used_by_vm($conf, sub {
8280         my ($sid) = @_;
8281         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8282         my $dst = $scfg->{shared} ? \$shared : \$local;
8283         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8284     });
8285
8286     # second, use shared storage where VM has at least one disk
8287     # third, use local storage where VM has at least one disk
8288     # fall back to local storage
8289     $target = $shared // $local // 'local';
8290
8291     return $target;
8292 }
8293
8294 sub generate_uuid {
8295     my ($uuid, $uuid_str);
8296     UUID::generate($uuid);
8297     UUID::unparse($uuid, $uuid_str);
8298     return $uuid_str;
8299 }
8300
8301 sub generate_smbios1_uuid {
8302     return "uuid=".generate_uuid();
8303 }
8304
8305 sub nbd_stop {
8306     my ($vmid) = @_;
8307
8308     mon_cmd($vmid, 'nbd-server-stop');
8309 }
8310
8311 sub create_reboot_request {
8312     my ($vmid) = @_;
8313     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8314         or die "failed to create reboot trigger file: $!\n";
8315     close($fh);
8316 }
8317
8318 sub clear_reboot_request {
8319     my ($vmid) = @_;
8320     my $path = "/run/qemu-server/$vmid.reboot";
8321     my $res = 0;
8322
8323     $res = unlink($path);
8324     die "could not remove reboot request for $vmid: $!"
8325         if !$res && $! != POSIX::ENOENT;
8326
8327     return $res;
8328 }
8329
8330 sub bootorder_from_legacy {
8331     my ($conf, $bootcfg) = @_;
8332
8333     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8334     my $bootindex_hash = {};
8335     my $i = 1;
8336     foreach my $o (split(//, $boot)) {
8337         $bootindex_hash->{$o} = $i*100;
8338         $i++;
8339     }
8340
8341     my $bootorder = {};
8342
8343     PVE::QemuConfig->foreach_volume($conf, sub {
8344         my ($ds, $drive) = @_;
8345
8346         if (drive_is_cdrom ($drive, 1)) {
8347             if ($bootindex_hash->{d}) {
8348                 $bootorder->{$ds} = $bootindex_hash->{d};
8349                 $bootindex_hash->{d} += 1;
8350             }
8351         } elsif ($bootindex_hash->{c}) {
8352             $bootorder->{$ds} = $bootindex_hash->{c}
8353                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8354             $bootindex_hash->{c} += 1;
8355         }
8356     });
8357
8358     if ($bootindex_hash->{n}) {
8359         for (my $i = 0; $i < $MAX_NETS; $i++) {
8360             my $netname = "net$i";
8361             next if !$conf->{$netname};
8362             $bootorder->{$netname} = $bootindex_hash->{n};
8363             $bootindex_hash->{n} += 1;
8364         }
8365     }
8366
8367     return $bootorder;
8368 }
8369
8370 # Generate default device list for 'boot: order=' property. Matches legacy
8371 # default boot order, but with explicit device names. This is important, since
8372 # the fallback for when neither 'order' nor the old format is specified relies
8373 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8374 sub get_default_bootdevices {
8375     my ($conf) = @_;
8376
8377     my @ret = ();
8378
8379     # harddisk
8380     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8381     push @ret, $first if $first;
8382
8383     # cdrom
8384     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8385     push @ret, $first if $first;
8386
8387     # network
8388     for (my $i = 0; $i < $MAX_NETS; $i++) {
8389         my $netname = "net$i";
8390         next if !$conf->{$netname};
8391         push @ret, $netname;
8392         last;
8393     }
8394
8395     return \@ret;
8396 }
8397
8398 sub device_bootorder {
8399     my ($conf) = @_;
8400
8401     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8402
8403     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8404
8405     my $bootorder = {};
8406     if (!defined($boot) || $boot->{legacy}) {
8407         $bootorder = bootorder_from_legacy($conf, $boot);
8408     } elsif ($boot->{order}) {
8409         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8410         for my $dev (PVE::Tools::split_list($boot->{order})) {
8411             $bootorder->{$dev} = $i++;
8412         }
8413     }
8414
8415     return $bootorder;
8416 }
8417
8418 sub register_qmeventd_handle {
8419     my ($vmid) = @_;
8420
8421     my $fh;
8422     my $peer = "/var/run/qmeventd.sock";
8423     my $count = 0;
8424
8425     for (;;) {
8426         $count++;
8427         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8428         last if $fh;
8429         if ($! != EINTR && $! != EAGAIN) {
8430             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8431         }
8432         if ($count > 4) {
8433             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8434               . "after $count retries\n";
8435         }
8436         usleep(25000);
8437     }
8438
8439     # send handshake to mark VM as backing up
8440     print $fh to_json({vzdump => {vmid => "$vmid"}});
8441
8442     # return handle to be closed later when inhibit is no longer required
8443     return $fh;
8444 }
8445
8446 # bash completion helper
8447
8448 sub complete_backup_archives {
8449     my ($cmdname, $pname, $cvalue) = @_;
8450
8451     my $cfg = PVE::Storage::config();
8452
8453     my $storeid;
8454
8455     if ($cvalue =~ m/^([^:]+):/) {
8456         $storeid = $1;
8457     }
8458
8459     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8460
8461     my $res = [];
8462     foreach my $id (keys %$data) {
8463         foreach my $item (@{$data->{$id}}) {
8464             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8465             push @$res, $item->{volid} if defined($item->{volid});
8466         }
8467     }
8468
8469     return $res;
8470 }
8471
8472 my $complete_vmid_full = sub {
8473     my ($running) = @_;
8474
8475     my $idlist = vmstatus();
8476
8477     my $res = [];
8478
8479     foreach my $id (keys %$idlist) {
8480         my $d = $idlist->{$id};
8481         if (defined($running)) {
8482             next if $d->{template};
8483             next if $running && $d->{status} ne 'running';
8484             next if !$running && $d->{status} eq 'running';
8485         }
8486         push @$res, $id;
8487
8488     }
8489     return $res;
8490 };
8491
8492 sub complete_vmid {
8493     return &$complete_vmid_full();
8494 }
8495
8496 sub complete_vmid_stopped {
8497     return &$complete_vmid_full(0);
8498 }
8499
8500 sub complete_vmid_running {
8501     return &$complete_vmid_full(1);
8502 }
8503
8504 sub complete_storage {
8505
8506     my $cfg = PVE::Storage::config();
8507     my $ids = $cfg->{ids};
8508
8509     my $res = [];
8510     foreach my $sid (keys %$ids) {
8511         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8512         next if !$ids->{$sid}->{content}->{images};
8513         push @$res, $sid;
8514     }
8515
8516     return $res;
8517 }
8518
8519 sub complete_migration_storage {
8520     my ($cmd, $param, $current_value, $all_args) = @_;
8521
8522     my $targetnode = @$all_args[1];
8523
8524     my $cfg = PVE::Storage::config();
8525     my $ids = $cfg->{ids};
8526
8527     my $res = [];
8528     foreach my $sid (keys %$ids) {
8529         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8530         next if !$ids->{$sid}->{content}->{images};
8531         push @$res, $sid;
8532     }
8533
8534     return $res;
8535 }
8536
8537 sub vm_is_paused {
8538     my ($vmid) = @_;
8539     my $qmpstatus = eval {
8540         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8541         mon_cmd($vmid, "query-status");
8542     };
8543     warn "$@\n" if $@;
8544     return $qmpstatus && $qmpstatus->{status} eq "paused";
8545 }
8546
8547 sub check_volume_storage_type {
8548     my ($storecfg, $vol) = @_;
8549
8550     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8551     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8552     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8553
8554     die "storage '$storeid' does not support content-type '$vtype'\n"
8555         if !$scfg->{content}->{$vtype};
8556
8557     return 1;
8558 }
8559
8560 sub add_nets_bridge_fdb {
8561     my ($conf, $vmid) = @_;
8562
8563     for my $opt (keys %$conf) {
8564         next if $opt !~ m/^net(\d+)$/;
8565         my $iface = "tap${vmid}i$1";
8566         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8567         my $net = parse_net($conf->{$opt}, 1) or next;
8568
8569         my $mac = $net->{macaddr};
8570         if (!$mac) {
8571             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8572                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8573             next;
8574         }
8575
8576         my $bridge = $net->{bridge};
8577         if ($have_sdn) {
8578             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8579         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8580             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8581         }
8582     }
8583 }
8584
8585 sub del_nets_bridge_fdb {
8586     my ($conf, $vmid) = @_;
8587
8588     for my $opt (keys %$conf) {
8589         next if $opt !~ m/^net(\d+)$/;
8590         my $iface = "tap${vmid}i$1";
8591
8592         my $net = parse_net($conf->{$opt}) or next;
8593         my $mac = $net->{macaddr} or next;
8594
8595         my $bridge = $net->{bridge};
8596         if ($have_sdn) {
8597             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8598         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8599             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8600         }
8601     }
8602 }
8603
8604 1;