PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196 };
 197
 198 my $ivshmem_fmt = {
 199     size => {
 200         type => 'integer',
 201         minimum => 1,
 202         description => "The size of the file in MB.",
 203     },
 204     name => {
 205         type => 'string',
 206         pattern => '[a-zA-Z0-9\-]+',
 207         optional => 1,
 208         format_description => 'string',
 209         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 210     },
 211 };
 212
 213 my $audio_fmt = {
 214     device => {
 215         type => 'string',
 216         enum => [qw(ich9-intel-hda intel-hda AC97)],
 217         description =>  "Configure an audio device."
 218     },
 219     driver =>  {
 220         type => 'string',
 221         enum => ['spice', 'none'],
 222         default => 'spice',
 223         optional => 1,
 224         description => "Driver backend for the audio device."
 225     },
 226 };
 227
 228 my $spice_enhancements_fmt = {
 229     foldersharing => {
 230         type => 'boolean',
 231         optional => 1,
 232         default => '0',
 233         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 234     },
 235     videostreaming =>  {
 236         type => 'string',
 237         enum => ['off', 'all', 'filter'],
 238         default => 'off',
 239         optional => 1,
 240         description => "Enable video streaming. Uses compression for detected video streams."
 241     },
 242 };
 243
 244 my $rng_fmt = {
 245     source => {
 246         type => 'string',
 247         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 248         default_key => 1,
 249         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 250             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 251             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 252             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 253             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 254             ." a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 259             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 260             ." `0` to disable limiting (potentially dangerous!).",
 261         optional => 1,
 262
 263         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 264         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 265         # reading from /dev/urandom
 266         default => 1024,
 267     },
 268     period => {
 269         type => 'integer',
 270         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 271             ." the guest to retrieve another 'max_bytes' of entropy.",
 272         optional => 1,
 273         default => 1000,
 274     },
 275 };
 276
 277 my $meta_info_fmt = {
 278     'ctime' => {
 279         type => 'integer',
 280         description => "The guest creation timestamp as UNIX epoch time",
 281         minimum => 0,
 282         optional => 1,
 283     },
 284     'creation-qemu' => {
 285         type => 'string',
 286         description => "The QEMU (machine) version from the time this VM was created.",
 287         pattern => '\d+(\.\d+)+',
 288         optional => 1,
 289     },
 290 };
 291
 292 my $confdesc = {
 293     onboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Specifies whether a VM will be started during system bootup.",
 297         default => 0,
 298     },
 299     autostart => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Automatic restart after crash (currently ignored).",
 303         default => 0,
 304     },
 305     hotplug => {
 306         optional => 1,
 307         type => 'string', format => 'pve-hotplug-features',
 308         description => "Selectively enable hotplug features. This is a comma separated list of"
 309             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 310             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 311             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 312             ." windows > 7.",
 313         default => 'network,disk,usb',
 314     },
 315     reboot => {
 316         optional => 1,
 317         type => 'boolean',
 318         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 319         default => 1,
 320     },
 321     lock => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Lock/unlock the VM.",
 325         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 326     },
 327     cpulimit => {
 328         optional => 1,
 329         type => 'number',
 330         description => "Limit of CPU usage.",
 331         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 332             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 333         minimum => 0,
 334         maximum => 128,
 335         default => 0,
 336     },
 337     cpuunits => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 341         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 342             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 343             ." weights of all the other running VMs.",
 344         minimum => 1,
 345         maximum => 262144,
 346         default => 'cgroup v1: 1024, cgroup v2: 100',
 347     },
 348     memory => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 352             ." you use the balloon device.",
 353         minimum => 16,
 354         default => 512,
 355     },
 356     balloon => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 360         minimum => 0,
 361     },
 362     shares => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 366             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 367             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 368         minimum => 0,
 369         maximum => 50000,
 370         default => 1000,
 371     },
 372     keyboard => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Keyboard layout for VNC server. This option is generally not required and"
 376         ." is often better handled from within the guest OS.",
 377         enum => PVE::Tools::kvmkeymaplist(),
 378         default => undef,
 379     },
 380     name => {
 381         optional => 1,
 382         type => 'string', format => 'dns-name',
 383         description => "Set a name for the VM. Only used on the configuration web interface.",
 384     },
 385     scsihw => {
 386         optional => 1,
 387         type => 'string',
 388         description => "SCSI controller model",
 389         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 390         default => 'lsi',
 391     },
 392     description => {
 393         optional => 1,
 394         type => 'string',
 395         description => "Description for the VM. Shown in the web-interface VM's summary."
 396             ." This is saved as comment inside the configuration file.",
 397         maxLength => 1024 * 8,
 398     },
 399     ostype => {
 400         optional => 1,
 401         type => 'string',
 402         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 403         description => "Specify guest operating system.",
 404         verbose_description => <<EODESC,
 405 Specify guest operating system. This is used to enable special
 406 optimization/features for specific operating systems:
 407
 408 [horizontal]
 409 other;; unspecified OS
 410 wxp;; Microsoft Windows XP
 411 w2k;; Microsoft Windows 2000
 412 w2k3;; Microsoft Windows 2003
 413 w2k8;; Microsoft Windows 2008
 414 wvista;; Microsoft Windows Vista
 415 win7;; Microsoft Windows 7
 416 win8;; Microsoft Windows 8/2012/2012r2
 417 win10;; Microsoft Windows 10/2016/2019
 418 win11;; Microsoft Windows 11/2022
 419 l24;; Linux 2.4 Kernel
 420 l26;; Linux 2.6 - 6.X Kernel
 421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 422 EODESC
 423     },
 424     boot => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-boot',
 427         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 428             ." key or 'legacy=' is deprecated.",
 429     },
 430     bootdisk => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-bootdisk',
 433         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 434         pattern => '(ide|sata|scsi|virtio)\d+',
 435     },
 436     smp => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPUs. Please use option -sockets instead.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     sockets => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of CPU sockets.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     cores => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of cores per socket.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     numa => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable NUMA.",
 461         default => 0,
 462     },
 463     hugepages => {
 464         optional => 1,
 465         type => 'string',
 466         description => "Enable/disable hugepages memory.",
 467         enum => [qw(any 2 1024)],
 468     },
 469     keephugepages => {
 470         optional => 1,
 471         type => 'boolean',
 472         default => 0,
 473         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 474             ." after VM shutdown and can be used for subsequent starts.",
 475     },
 476     vcpus => {
 477         optional => 1,
 478         type => 'integer',
 479         description => "Number of hotplugged vcpus.",
 480         minimum => 1,
 481         default => 0,
 482     },
 483     acpi => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Enable/disable ACPI.",
 487         default => 1,
 488     },
 489     agent => {
 490         optional => 1,
 491         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 492         type => 'string',
 493         format => $agent_fmt,
 494     },
 495     kvm => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable KVM hardware virtualization.",
 499         default => 1,
 500     },
 501     tdf => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable time drift fix.",
 505         default => 0,
 506     },
 507     localtime => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 511             ." the `ostype` indicates a Microsoft Windows OS.",
 512     },
 513     freeze => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 517     },
 518     vga => {
 519         optional => 1,
 520         type => 'string', format => $vga_fmt,
 521         description => "Configure the VGA hardware.",
 522         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 523             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 524             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 525             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 526             ." display server. For win* OS you can select how many independent displays you want,"
 527             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 528             ." using a serial device as terminal.",
 529     },
 530     watchdog => {
 531         optional => 1,
 532         type => 'string', format => 'pve-qm-watchdog',
 533         description => "Create a virtual hardware watchdog device.",
 534         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 535             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 536             ." else the watchdog will reset the guest (or execute the respective action specified)",
 537     },
 538     startdate => {
 539         optional => 1,
 540         type => 'string',
 541         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 542         description => "Set the initial date of the real time clock. Valid format for date are:"
 543             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 544         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 545         default => 'now',
 546     },
 547     startup =>  get_standard_option('pve-startup-order'),
 548     template => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Enable/disable Template.",
 552         default => 0,
 553     },
 554     args => {
 555         optional => 1,
 556         type => 'string',
 557         description => "Arbitrary arguments passed to kvm.",
 558         verbose_description => <<EODESCR,
 559 Arbitrary arguments passed to kvm, for example:
 560
 561 args: -no-reboot -smbios 'type=0,vendor=FOO'
 562
 563 NOTE: this option is for experts only.
 564 EODESCR
 565     },
 566     tablet => {
 567         optional => 1,
 568         type => 'boolean',
 569         default => 1,
 570         description => "Enable/disable the USB tablet device.",
 571         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 572             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 573             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 574             ." may consider disabling this to save some context switches. This is turned off by"
 575             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 576     },
 577     migrate_speed => {
 578         optional => 1,
 579         type => 'integer',
 580         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 581         minimum => 0,
 582         default => 0,
 583     },
 584     migrate_downtime => {
 585         optional => 1,
 586         type => 'number',
 587         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 588         minimum => 0,
 589         default => 0.1,
 590     },
 591     cdrom => {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-ide',
 594         typetext => '<volume>',
 595         description => "This is an alias for option -ide2",
 596     },
 597     cpu => {
 598         optional => 1,
 599         description => "Emulated CPU type.",
 600         type => 'string',
 601         format => 'pve-vm-cpu-conf',
 602     },
 603     parent => get_standard_option('pve-snapshot-name', {
 604         optional => 1,
 605         description => "Parent snapshot name. This is used internally, and should not be modified.",
 606     }),
 607     snaptime => {
 608         optional => 1,
 609         description => "Timestamp for snapshots.",
 610         type => 'integer',
 611         minimum => 0,
 612     },
 613     vmstate => {
 614         optional => 1,
 615         type => 'string', format => 'pve-volume-id',
 616         description => "Reference to a volume which stores the VM state. This is used internally"
 617             ." for snapshots.",
 618     },
 619     vmstatestorage => get_standard_option('pve-storage-id', {
 620         description => "Default storage for VM state volumes/files.",
 621         optional => 1,
 622     }),
 623     runningmachine => get_standard_option('pve-qemu-machine', {
 624         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 625             ." for snapshots.",
 626     }),
 627     runningcpu => {
 628         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 629             ." internally for snapshots.",
 630         optional => 1,
 631         type => 'string',
 632         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 633         format_description => 'QEMU -cpu parameter'
 634     },
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 652             ." remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 667             ." to disable explicitly.",
 668         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 669             ." value identifier to the guest OS. This allows to notify the guest operating system"
 670             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 671             ." execution or creation from a template). The guest operating system notices the"
 672             ." change, and is then able to react as appropriate by marking its copies of"
 673             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 674             ."Note that auto-creation only works when done through API/CLI create or update methods"
 675             .", but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 689             ." the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704     tags => {
 705         type => 'string', format => 'pve-tag-list',
 706         description => 'Tags of the VM. This is only meta information.',
 707         optional => 1,
 708     },
 709     rng0 => {
 710         type => 'string',
 711         format => $rng_fmt,
 712         description => "Configure a VirtIO-based Random Number Generator.",
 713         optional => 1,
 714     },
 715     meta => {
 716         type => 'string',
 717         format => $meta_info_fmt,
 718         description => "Some (read-only) meta-information about this guest.",
 719         optional => 1,
 720     },
 721     affinity => {
 722         type => 'string', format => 'pve-cpuset',
 723         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 724         optional => 1,
 725     },
 726 };
 727
 728 my $cicustom_fmt = {
 729     meta => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all meta data passed to the VM via"
 733             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     network => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     user => {
 745         type => 'string',
 746         optional => 1,
 747         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 748         format => 'pve-volume-id',
 749         format_description => 'volume',
 750     },
 751     vendor => {
 752         type => 'string',
 753         optional => 1,
 754         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 760
 761 my $confdesc_cloudinit = {
 762     citype => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'Specifies the cloud-init configuration format. The default depends on the'
 766             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 767             .' and `configdrive2` for windows.',
 768         enum => ['configdrive2', 'nocloud', 'opennebula'],
 769     },
 770     ciuser => {
 771         optional => 1,
 772         type => 'string',
 773         description => "cloud-init: User name to change ssh keys and password for instead of the"
 774             ." image's configured default user.",
 775     },
 776     cipassword => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Password to assign the user. Using this is generally not'
 780             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 781             .' support hashed passwords.',
 782     },
 783     ciupgrade => {
 784         optional => 1,
 785         type => 'boolean',
 786         description => 'cloud-init: do an automatic package upgrade after the first boot.'
 787     },
 788     cicustom => {
 789         optional => 1,
 790         type => 'string',
 791         description => 'cloud-init: Specify custom files to replace the automatically generated'
 792             .' ones at start.',
 793         format => 'pve-qm-cicustom',
 794     },
 795     searchdomain => {
 796         optional => 1,
 797         type => 'string',
 798         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 799             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 800             .' are set.',
 801     },
 802     nameserver => {
 803         optional => 1,
 804         type => 'string', format => 'address-list',
 805         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 806             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 807             .' are set.',
 808     },
 809     sshkeys => {
 810         optional => 1,
 811         type => 'string',
 812         format => 'urlencoded',
 813         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 814     },
 815 };
 816
 817 # what about other qemu settings ?
 818 #cpu => 'string',
 819 #machine => 'string',
 820 #fda => 'file',
 821 #fdb => 'file',
 822 #mtdblock => 'file',
 823 #sd => 'file',
 824 #pflash => 'file',
 825 #snapshot => 'bool',
 826 #bootp => 'file',
 827 ##tftp => 'dir',
 828 ##smb => 'dir',
 829 #kernel => 'file',
 830 #append => 'string',
 831 #initrd => 'file',
 832 ##soundhw => 'string',
 833
 834 while (my ($k, $v) = each %$confdesc) {
 835     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 836 }
 837
 838 my $MAX_USB_DEVICES = 14;
 839 my $MAX_NETS = 32;
 840 my $MAX_SERIAL_PORTS = 4;
 841 my $MAX_PARALLEL_PORTS = 3;
 842 my $MAX_NUMA = 8;
 843
 844 my $numa_fmt = {
 845     cpus => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "CPUs accessing this NUMA node.",
 849         format_description => "id[-id];...",
 850     },
 851     memory => {
 852         type => "number",
 853         description => "Amount of memory this NUMA node provides.",
 854         optional => 1,
 855     },
 856     hostnodes => {
 857         type => "string",
 858         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 859         description => "Host NUMA nodes to use.",
 860         format_description => "id[-id];...",
 861         optional => 1,
 862     },
 863     policy => {
 864         type => 'string',
 865         enum => [qw(preferred bind interleave)],
 866         description => "NUMA allocation policy.",
 867         optional => 1,
 868     },
 869 };
 870 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 871 my $numadesc = {
 872     optional => 1,
 873     type => 'string', format => $numa_fmt,
 874     description => "NUMA topology.",
 875 };
 876 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 877
 878 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 879     $confdesc->{"numa$i"} = $numadesc;
 880 }
 881
 882 my $nic_model_list = [
 883     'e1000',
 884     'e1000-82540em',
 885     'e1000-82544gc',
 886     'e1000-82545em',
 887     'e1000e',
 888     'i82551',
 889     'i82557b',
 890     'i82559er',
 891     'ne2k_isa',
 892     'ne2k_pci',
 893     'pcnet',
 894     'rtl8139',
 895     'virtio',
 896     'vmxnet3',
 897 ];
 898 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 899
 900 my $net_fmt_bridge_descr = <<__EOD__;
 901 Bridge to attach the network device to. The Proxmox VE standard bridge
 902 is called 'vmbr0'.
 903
 904 If you do not specify a bridge, we create a kvm user (NATed) network
 905 device, which provides DHCP and DNS services. The following addresses
 906 are used:
 907
 908  10.0.2.2   Gateway
 909  10.0.2.3   DNS Server
 910  10.0.2.4   SMB Server
 911
 912 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 913 __EOD__
 914
 915 my $net_fmt = {
 916     macaddr  => get_standard_option('mac-addr', {
 917         description => "MAC address. That address must be unique withing your network. This is"
 918             ." automatically generated if not specified.",
 919     }),
 920     model => {
 921         type => 'string',
 922         description => "Network Card Model. The 'virtio' model provides the best performance with"
 923             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 924             ." best to use 'e1000'.",
 925         enum => $nic_model_list,
 926         default_key => 1,
 927     },
 928     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 929     bridge => get_standard_option('pve-bridge-id', {
 930         description => $net_fmt_bridge_descr,
 931         optional => 1,
 932     }),
 933     queues => {
 934         type => 'integer',
 935         minimum => 0, maximum => 64,
 936         description => 'Number of packet queues to be used on the device.',
 937         optional => 1,
 938     },
 939     rate => {
 940         type => 'number',
 941         minimum => 0,
 942         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 943         optional => 1,
 944     },
 945     tag => {
 946         type => 'integer',
 947         minimum => 1, maximum => 4094,
 948         description => 'VLAN tag to apply to packets on this interface.',
 949         optional => 1,
 950     },
 951     trunks => {
 952         type => 'string',
 953         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 954         description => 'VLAN trunks to pass through this interface.',
 955         format_description => 'vlanid[;vlanid...]',
 956         optional => 1,
 957     },
 958     firewall => {
 959         type => 'boolean',
 960         description => 'Whether this interface should be protected by the firewall.',
 961         optional => 1,
 962     },
 963     link_down => {
 964         type => 'boolean',
 965         description => 'Whether this interface should be disconnected (like pulling the plug).',
 966         optional => 1,
 967     },
 968     mtu => {
 969         type => 'integer',
 970         minimum => 1, maximum => 65520,
 971         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 972         optional => 1,
 973     },
 974 };
 975
 976 my $netdesc = {
 977     optional => 1,
 978     type => 'string', format => $net_fmt,
 979     description => "Specify network devices.",
 980 };
 981
 982 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 983
 984 my $ipconfig_fmt = {
 985     ip => {
 986         type => 'string',
 987         format => 'pve-ipv4-config',
 988         format_description => 'IPv4Format/CIDR',
 989         description => 'IPv4 address in CIDR format.',
 990         optional => 1,
 991         default => 'dhcp',
 992     },
 993     gw => {
 994         type => 'string',
 995         format => 'ipv4',
 996         format_description => 'GatewayIPv4',
 997         description => 'Default gateway for IPv4 traffic.',
 998         optional => 1,
 999         requires => 'ip',
1000     },
1001     ip6 => {
1002         type => 'string',
1003         format => 'pve-ipv6-config',
1004         format_description => 'IPv6Format/CIDR',
1005         description => 'IPv6 address in CIDR format.',
1006         optional => 1,
1007         default => 'dhcp',
1008     },
1009     gw6 => {
1010         type => 'string',
1011         format => 'ipv6',
1012         format_description => 'GatewayIPv6',
1013         description => 'Default gateway for IPv6 traffic.',
1014         optional => 1,
1015         requires => 'ip6',
1016     },
1017 };
1018 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1019 my $ipconfigdesc = {
1020     optional => 1,
1021     type => 'string', format => 'pve-qm-ipconfig',
1022     description => <<'EODESCR',
1023 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1024
1025 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1026
1027 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1028 gateway should be provided.
1029 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1030 cloud-init 19.4 or newer.
1031
1032 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1033 dhcp on IPv4.
1034 EODESCR
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1037
1038 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1039     $confdesc->{"net$i"} = $netdesc;
1040     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1041 }
1042
1043 foreach my $key (keys %$confdesc_cloudinit) {
1044     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1045 }
1046
1047 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1048 sub pve_verify_cpuset {
1049     my ($set_text, $noerr) = @_;
1050
1051     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1052
1053     if ($@) {
1054         return if $noerr;
1055         die "unable to parse cpuset option\n";
1056     }
1057
1058     return PVE::CpuSet->new($members)->short_string();
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1062 sub verify_volume_id_or_qm_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1066
1067     return verify_volume_id_or_absolute_path($volid, $noerr);
1068 }
1069
1070 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1071 sub verify_volume_id_or_absolute_path {
1072     my ($volid, $noerr) = @_;
1073
1074     return $volid if $volid =~ m|^/|;
1075
1076     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1077     if ($@) {
1078         return if $noerr;
1079         die $@;
1080     }
1081     return $volid;
1082 }
1083
1084 my $usb_fmt = {
1085     host => {
1086         default_key => 1,
1087         type => 'string', format => 'pve-qm-usb-device',
1088         format_description => 'HOSTUSBDEVICE|spice',
1089         description => <<EODESCR,
1090 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1091
1092  'bus-port(.port)*' (decimal numbers) or
1093  'vendor_id:product_id' (hexadeciaml numbers) or
1094  'spice'
1095
1096 You can use the 'lsusb -t' command to list existing usb devices.
1097
1098 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1099 machines - use with special care.
1100
1101 The value 'spice' can be used to add a usb redirection devices for spice.
1102 EODESCR
1103     },
1104     usb3 => {
1105         optional => 1,
1106         type => 'boolean',
1107         description => "Specifies whether if given host option is a USB3 device or port."
1108             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1109             ." is irrelevant (all devices are plugged into a xhci controller).",
1110         default => 0,
1111     },
1112 };
1113
1114 my $usbdesc = {
1115     optional => 1,
1116     type => 'string', format => $usb_fmt,
1117     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1118         ." l26 or windows > 7, n can be up to 14).",
1119 };
1120 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1121
1122 my $serialdesc = {
1123         optional => 1,
1124         type => 'string',
1125         pattern => '(/dev/.+|socket)',
1126         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1127         verbose_description =>  <<EODESCR,
1128 Create a serial device inside the VM (n is 0 to 3), and pass through a
1129 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1130 host side (use 'qm terminal' to open a terminal connection).
1131
1132 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1133 use with special care.
1134
1135 CAUTION: Experimental! User reported problems with this option.
1136 EODESCR
1137 };
1138
1139 my $paralleldesc= {
1140         optional => 1,
1141         type => 'string',
1142         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1143         description =>  "Map host parallel devices (n is 0 to 2).",
1144         verbose_description =>  <<EODESCR,
1145 Map host parallel devices (n is 0 to 2).
1146
1147 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1148 machines - use with special care.
1149
1150 CAUTION: Experimental! User reported problems with this option.
1151 EODESCR
1152 };
1153
1154 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1155     $confdesc->{"parallel$i"} = $paralleldesc;
1156 }
1157
1158 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1159     $confdesc->{"serial$i"} = $serialdesc;
1160 }
1161
1162 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1163     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1164 }
1165
1166 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1167     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1168 }
1169
1170 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1171     $confdesc->{"usb$i"} = $usbdesc;
1172 }
1173
1174 my $boot_fmt = {
1175     legacy => {
1176         optional => 1,
1177         default_key => 1,
1178         type => 'string',
1179         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1180                      . " Deprecated, use 'order=' instead.",
1181         pattern => '[acdn]{1,4}',
1182         format_description => "[acdn]{1,4}",
1183
1184         # note: this is also the fallback if boot: is not given at all
1185         default => 'cdn',
1186     },
1187     order => {
1188         optional => 1,
1189         type => 'string',
1190         format => 'pve-qm-bootdev-list',
1191         format_description => "device[;device...]",
1192         description => <<EODESC,
1193 The guest will attempt to boot from devices in the order they appear here.
1194
1195 Disks, optical drives and passed-through storage USB devices will be directly
1196 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1197 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1198
1199 Note that only devices in this list will be marked as bootable and thus loaded
1200 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1201 (e.g. software-raid), you need to specify all of them here.
1202
1203 Overrides the deprecated 'legacy=[acdn]*' value when given.
1204 EODESC
1205     },
1206 };
1207 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1208
1209 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1210 sub verify_bootdev {
1211     my ($dev, $noerr) = @_;
1212
1213     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1214     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1215
1216     my $check = sub {
1217         my ($base) = @_;
1218         return 0 if $dev !~ m/^$base\d+$/;
1219         return 0 if !$confdesc->{$dev};
1220         return 1;
1221     };
1222
1223     return $dev if $check->("net");
1224     return $dev if $check->("usb");
1225     return $dev if $check->("hostpci");
1226
1227     return if $noerr;
1228     die "invalid boot device '$dev'\n";
1229 }
1230
1231 sub print_bootorder {
1232     my ($devs) = @_;
1233     return "" if !@$devs;
1234     my $data = { order => join(';', @$devs) };
1235     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1236 }
1237
1238 my $kvm_api_version = 0;
1239
1240 sub kvm_version {
1241     return $kvm_api_version if $kvm_api_version;
1242
1243     open my $fh, '<', '/dev/kvm' or return;
1244
1245     # 0xae00 => KVM_GET_API_VERSION
1246     $kvm_api_version = ioctl($fh, 0xae00, 0);
1247     close($fh);
1248
1249     return $kvm_api_version;
1250 }
1251
1252 my $kvm_user_version = {};
1253 my $kvm_mtime = {};
1254
1255 sub kvm_user_version {
1256     my ($binary) = @_;
1257
1258     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1259     my $st = stat($binary);
1260
1261     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1262     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1263         $cachedmtime == $st->mtime;
1264
1265     $kvm_user_version->{$binary} = 'unknown';
1266     $kvm_mtime->{$binary} = $st->mtime;
1267
1268     my $code = sub {
1269         my $line = shift;
1270         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1271             $kvm_user_version->{$binary} = $2;
1272         }
1273     };
1274
1275     eval { run_command([$binary, '--version'], outfunc => $code); };
1276     warn $@ if $@;
1277
1278     return $kvm_user_version->{$binary};
1279
1280 }
1281 my sub extract_version {
1282     my ($machine_type, $version) = @_;
1283     $version = kvm_user_version() if !defined($version);
1284     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1285 }
1286
1287 sub kernel_has_vhost_net {
1288     return -c '/dev/vhost-net';
1289 }
1290
1291 sub option_exists {
1292     my $key = shift;
1293     return defined($confdesc->{$key});
1294 }
1295
1296 my $cdrom_path;
1297 sub get_cdrom_path {
1298
1299     return $cdrom_path if defined($cdrom_path);
1300
1301     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1302
1303     if (!defined($cdrom_path)) {
1304         log_warn("no physical CD-ROM available, ignoring");
1305         $cdrom_path = '';
1306     }
1307
1308     return $cdrom_path;
1309 }
1310
1311 sub get_iso_path {
1312     my ($storecfg, $vmid, $cdrom) = @_;
1313
1314     if ($cdrom eq 'cdrom') {
1315         return get_cdrom_path();
1316     } elsif ($cdrom eq 'none') {
1317         return '';
1318     } elsif ($cdrom =~ m|^/|) {
1319         return $cdrom;
1320     } else {
1321         return PVE::Storage::path($storecfg, $cdrom);
1322     }
1323 }
1324
1325 # try to convert old style file names to volume IDs
1326 sub filename_to_volume_id {
1327     my ($vmid, $file, $media) = @_;
1328
1329      if (!($file eq 'none' || $file eq 'cdrom' ||
1330           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1331
1332         return if $file =~ m|/|;
1333
1334         if ($media && $media eq 'cdrom') {
1335             $file = "local:iso/$file";
1336         } else {
1337             $file = "local:$vmid/$file";
1338         }
1339     }
1340
1341     return $file;
1342 }
1343
1344 sub verify_media_type {
1345     my ($opt, $vtype, $media) = @_;
1346
1347     return if !$media;
1348
1349     my $etype;
1350     if ($media eq 'disk') {
1351         $etype = 'images';
1352     } elsif ($media eq 'cdrom') {
1353         $etype = 'iso';
1354     } else {
1355         die "internal error";
1356     }
1357
1358     return if ($vtype eq $etype);
1359
1360     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1361 }
1362
1363 sub cleanup_drive_path {
1364     my ($opt, $storecfg, $drive) = @_;
1365
1366     # try to convert filesystem paths to volume IDs
1367
1368     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1369         ($drive->{file} !~ m|^/dev/.+|) &&
1370         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1371         ($drive->{file} !~ m/^\d+$/)) {
1372         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1373         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1374             if !$vtype;
1375         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1376         verify_media_type($opt, $vtype, $drive->{media});
1377         $drive->{file} = $volid;
1378     }
1379
1380     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1381 }
1382
1383 sub parse_hotplug_features {
1384     my ($data) = @_;
1385
1386     my $res = {};
1387
1388     return $res if $data eq '0';
1389
1390     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1391
1392     foreach my $feature (PVE::Tools::split_list($data)) {
1393         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1394             $res->{$1} = 1;
1395         } else {
1396             die "invalid hotplug feature '$feature'\n";
1397         }
1398     }
1399     return $res;
1400 }
1401
1402 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1403 sub pve_verify_hotplug_features {
1404     my ($value, $noerr) = @_;
1405
1406     return $value if parse_hotplug_features($value);
1407
1408     return if $noerr;
1409
1410     die "unable to parse hotplug option\n";
1411 }
1412
1413 sub scsi_inquiry {
1414     my($fh, $noerr) = @_;
1415
1416     my $SG_IO = 0x2285;
1417     my $SG_GET_VERSION_NUM = 0x2282;
1418
1419     my $versionbuf = "\x00" x 8;
1420     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1421     if (!$ret) {
1422         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1423         return;
1424     }
1425     my $version = unpack("I", $versionbuf);
1426     if ($version < 30000) {
1427         die "scsi generic interface too old\n"  if !$noerr;
1428         return;
1429     }
1430
1431     my $buf = "\x00" x 36;
1432     my $sensebuf = "\x00" x 8;
1433     my $cmd = pack("C x3 C x1", 0x12, 36);
1434
1435     # see /usr/include/scsi/sg.h
1436     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1437
1438     my $packet = pack(
1439         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1440     );
1441
1442     $ret = ioctl($fh, $SG_IO, $packet);
1443     if (!$ret) {
1444         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1445         return;
1446     }
1447
1448     my @res = unpack($sg_io_hdr_t, $packet);
1449     if ($res[17] || $res[18]) {
1450         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1451         return;
1452     }
1453
1454     my $res = {};
1455     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1456
1457     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1458     $res->{type} &= 0x1F;
1459
1460     return $res;
1461 }
1462
1463 sub path_is_scsi {
1464     my ($path) = @_;
1465
1466     my $fh = IO::File->new("+<$path") || return;
1467     my $res = scsi_inquiry($fh, 1);
1468     close($fh);
1469
1470     return $res;
1471 }
1472
1473 sub print_tabletdevice_full {
1474     my ($conf, $arch) = @_;
1475
1476     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1477
1478     # we use uhci for old VMs because tablet driver was buggy in older qemu
1479     my $usbbus;
1480     if ($q35 || $arch eq 'aarch64') {
1481         $usbbus = 'ehci';
1482     } else {
1483         $usbbus = 'uhci';
1484     }
1485
1486     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1487 }
1488
1489 sub print_keyboarddevice_full {
1490     my ($conf, $arch) = @_;
1491
1492     return if $arch ne 'aarch64';
1493
1494     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1495 }
1496
1497 my sub get_drive_id {
1498     my ($drive) = @_;
1499     return "$drive->{interface}$drive->{index}";
1500 }
1501
1502 sub print_drivedevice_full {
1503     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1504
1505     my $device = '';
1506     my $maxdev = 0;
1507
1508     my $drive_id = get_drive_id($drive);
1509     if ($drive->{interface} eq 'virtio') {
1510         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1511         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1512         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1513     } elsif ($drive->{interface} eq 'scsi') {
1514
1515         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1516         my $unit = $drive->{index} % $maxdev;
1517         my $devicetype = 'hd';
1518         my $path = '';
1519         if (drive_is_cdrom($drive)) {
1520             $devicetype = 'cd';
1521         } else {
1522             if ($drive->{file} =~ m|^/|) {
1523                 $path = $drive->{file};
1524                 if (my $info = path_is_scsi($path)) {
1525                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1526                         $devicetype = 'block';
1527                     } elsif ($info->{type} == 1) { # tape
1528                         $devicetype = 'generic';
1529                     }
1530                 }
1531             } else {
1532                  $path = PVE::Storage::path($storecfg, $drive->{file});
1533             }
1534
1535             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1536             my $version = extract_version($machine_type, kvm_user_version());
1537             if ($path =~ m/^iscsi\:\/\// &&
1538                !min_version($version, 4, 1)) {
1539                 $devicetype = 'generic';
1540             }
1541         }
1542
1543         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1544             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1545         } else {
1546             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1547                 .",lun=$drive->{index}";
1548         }
1549         $device .= ",drive=drive-$drive_id,id=$drive_id";
1550
1551         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1552             $device .= ",rotation_rate=1";
1553         }
1554         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1555
1556     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1557         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1558         my $controller = int($drive->{index} / $maxdev);
1559         my $unit = $drive->{index} % $maxdev;
1560         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1561
1562         $device = "ide-$devicetype";
1563         if ($drive->{interface} eq 'ide') {
1564             $device .= ",bus=ide.$controller,unit=$unit";
1565         } else {
1566             $device .= ",bus=ahci$controller.$unit";
1567         }
1568         $device .= ",drive=drive-$drive_id,id=$drive_id";
1569
1570         if ($devicetype eq 'hd') {
1571             if (my $model = $drive->{model}) {
1572                 $model = URI::Escape::uri_unescape($model);
1573                 $device .= ",model=$model";
1574             }
1575             if ($drive->{ssd}) {
1576                 $device .= ",rotation_rate=1";
1577             }
1578         }
1579         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1580     } elsif ($drive->{interface} eq 'usb') {
1581         die "implement me";
1582         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1583     } else {
1584         die "unsupported interface type";
1585     }
1586
1587     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1588
1589     if (my $serial = $drive->{serial}) {
1590         $serial = URI::Escape::uri_unescape($serial);
1591         $device .= ",serial=$serial";
1592     }
1593
1594
1595     return $device;
1596 }
1597
1598 sub get_initiator_name {
1599     my $initiator;
1600
1601     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1602     while (defined(my $line = <$fh>)) {
1603         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1604         $initiator = $1;
1605         last;
1606     }
1607     $fh->close();
1608
1609     return $initiator;
1610 }
1611
1612 my sub storage_allows_io_uring_default {
1613     my ($scfg, $cache_direct) = @_;
1614
1615     # io_uring with cache mode writeback or writethrough on krbd will hang...
1616     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1617
1618     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1619     # sometimes, just plain disable...
1620     return if $scfg && $scfg->{type} eq 'lvm';
1621
1622     # io_uring causes problems when used with CIFS since kernel 5.15
1623     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1624     return if $scfg && $scfg->{type} eq 'cifs';
1625
1626     return 1;
1627 }
1628
1629 my sub drive_uses_cache_direct {
1630     my ($drive, $scfg) = @_;
1631
1632     my $cache_direct = 0;
1633
1634     if (my $cache = $drive->{cache}) {
1635         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1636     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1637         $cache_direct = 1;
1638     }
1639
1640     return $cache_direct;
1641 }
1642
1643 sub print_drive_commandline_full {
1644     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1645
1646     my $path;
1647     my $volid = $drive->{file};
1648     my $format = $drive->{format};
1649     my $drive_id = get_drive_id($drive);
1650
1651     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1652     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1653
1654     if (drive_is_cdrom($drive)) {
1655         $path = get_iso_path($storecfg, $vmid, $volid);
1656         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1657     } else {
1658         if ($storeid) {
1659             $path = PVE::Storage::path($storecfg, $volid);
1660             $format //= qemu_img_format($scfg, $volname);
1661         } else {
1662             $path = $volid;
1663             $format //= "raw";
1664         }
1665    }
1666
1667    my $is_rbd = $path =~ m/^rbd:/;
1668
1669     my $opts = '';
1670     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1671     foreach my $o (@qemu_drive_options) {
1672         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1673     }
1674
1675     # snapshot only accepts on|off
1676     if (defined($drive->{snapshot})) {
1677         my $v = $drive->{snapshot} ? 'on' : 'off';
1678         $opts .= ",snapshot=$v";
1679     }
1680
1681     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1682         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1683     }
1684
1685     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1686         my ($dir, $qmpname) = @$type;
1687         if (my $v = $drive->{"mbps$dir"}) {
1688             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1689         }
1690         if (my $v = $drive->{"mbps${dir}_max"}) {
1691             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1692         }
1693         if (my $v = $drive->{"bps${dir}_max_length"}) {
1694             $opts .= ",throttling.bps$qmpname-max-length=$v";
1695         }
1696         if (my $v = $drive->{"iops${dir}"}) {
1697             $opts .= ",throttling.iops$qmpname=$v";
1698         }
1699         if (my $v = $drive->{"iops${dir}_max"}) {
1700             $opts .= ",throttling.iops$qmpname-max=$v";
1701         }
1702         if (my $v = $drive->{"iops${dir}_max_length"}) {
1703             $opts .= ",throttling.iops$qmpname-max-length=$v";
1704         }
1705     }
1706
1707     if ($pbs_name) {
1708         $format = "rbd" if $is_rbd;
1709         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1710             if !$format;
1711         $opts .= ",format=alloc-track,file.driver=$format";
1712     } elsif ($format) {
1713         $opts .= ",format=$format";
1714     }
1715
1716     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1717
1718     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1719
1720     if (!$drive->{aio}) {
1721         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1722             # io_uring supports all cache modes
1723             $opts .= ",aio=io_uring";
1724         } else {
1725             # aio native works only with O_DIRECT
1726             if($cache_direct) {
1727                 $opts .= ",aio=native";
1728             } else {
1729                 $opts .= ",aio=threads";
1730             }
1731         }
1732     }
1733
1734     if (!drive_is_cdrom($drive)) {
1735         my $detectzeroes;
1736         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1737             $detectzeroes = 'off';
1738         } elsif ($drive->{discard}) {
1739             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1740         } else {
1741             # This used to be our default with discard not being specified:
1742             $detectzeroes = 'on';
1743         }
1744
1745         # note: 'detect-zeroes' works per blockdev and we want it to persist
1746         # after the alloc-track is removed, so put it on 'file' directly
1747         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1748         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1749     }
1750
1751     if ($pbs_name) {
1752         $opts .= ",backing=$pbs_name";
1753         $opts .= ",auto-remove=on";
1754     }
1755
1756     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1757     my $file_param = "file";
1758     if ($pbs_name) {
1759         # non-rbd drivers require the underlying file to be a seperate block
1760         # node, so add a second .file indirection
1761         $file_param .= ".file" if !$is_rbd;
1762         $file_param .= ".filename";
1763     }
1764     my $pathinfo = $path ? "$file_param=$path," : '';
1765
1766     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1767 }
1768
1769 sub print_pbs_blockdev {
1770     my ($pbs_conf, $pbs_name) = @_;
1771     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1772     $blockdev .= ",repository=$pbs_conf->{repository}";
1773     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1774     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1775     $blockdev .= ",archive=$pbs_conf->{archive}";
1776     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1777     return $blockdev;
1778 }
1779
1780 sub print_netdevice_full {
1781     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1782
1783     my $device = $net->{model};
1784     if ($net->{model} eq 'virtio') {
1785          $device = 'virtio-net-pci';
1786      };
1787
1788     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1789     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1790     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1791         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1792         # and out of each queue plus one config interrupt and control vector queue
1793         my $vectors = $net->{queues} * 2 + 2;
1794         $tmpstr .= ",vectors=$vectors,mq=on";
1795         if (min_version($machine_version, 7, 1)) {
1796             $tmpstr .= ",packed=on";
1797         }
1798     }
1799
1800     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1801         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1802     }
1803
1804     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1805
1806     if (my $mtu = $net->{mtu}) {
1807         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1808             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1809             if ($mtu == 1) {
1810                  $mtu = $bridge_mtu;
1811             } elsif ($mtu < 576) {
1812                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1813             } elsif ($mtu > $bridge_mtu) {
1814                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1815             }
1816             $tmpstr .= ",host_mtu=$mtu";
1817         } else {
1818             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1819         }
1820     }
1821
1822     if ($use_old_bios_files) {
1823         my $romfile;
1824         if ($device eq 'virtio-net-pci') {
1825             $romfile = 'pxe-virtio.rom';
1826         } elsif ($device eq 'e1000') {
1827             $romfile = 'pxe-e1000.rom';
1828         } elsif ($device eq 'e1000e') {
1829             $romfile = 'pxe-e1000e.rom';
1830         } elsif ($device eq 'ne2k') {
1831             $romfile = 'pxe-ne2k_pci.rom';
1832         } elsif ($device eq 'pcnet') {
1833             $romfile = 'pxe-pcnet.rom';
1834         } elsif ($device eq 'rtl8139') {
1835             $romfile = 'pxe-rtl8139.rom';
1836         }
1837         $tmpstr .= ",romfile=$romfile" if $romfile;
1838     }
1839
1840     return $tmpstr;
1841 }
1842
1843 sub print_netdev_full {
1844     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1845
1846     my $i = '';
1847     if ($netid =~ m/^net(\d+)$/) {
1848         $i = int($1);
1849     }
1850
1851     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1852
1853     my $ifname = "tap${vmid}i$i";
1854
1855     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1856     die "interface name '$ifname' is too long (max 15 character)\n"
1857         if length($ifname) >= 16;
1858
1859     my $vhostparam = '';
1860     if (is_native($arch)) {
1861         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1862     }
1863
1864     my $vmname = $conf->{name} || "vm$vmid";
1865
1866     my $netdev = "";
1867     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1868
1869     if ($net->{bridge}) {
1870         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1871             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881 my $vga_map = {
1882     'cirrus' => 'cirrus-vga',
1883     'std' => 'VGA',
1884     'vmware' => 'vmware-svga',
1885     'virtio' => 'virtio-vga',
1886     'virtio-gl' => 'virtio-vga-gl',
1887 };
1888
1889 sub print_vga_device {
1890     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1891
1892     my $type = $vga_map->{$vga->{type}};
1893     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1894         $type = 'virtio-gpu';
1895     }
1896     my $vgamem_mb = $vga->{memory};
1897
1898     my $max_outputs = '';
1899     if ($qxlnum) {
1900         $type = $id ? 'qxl' : 'qxl-vga';
1901
1902         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1903             # set max outputs so linux can have up to 4 qxl displays with one device
1904             if (min_version($machine_version, 4, 1)) {
1905                 $max_outputs = ",max_outputs=4";
1906             }
1907         }
1908     }
1909
1910     die "no devicetype for $vga->{type}\n" if !$type;
1911
1912     my $memory = "";
1913     if ($vgamem_mb) {
1914         if ($vga->{type} =~ /^virtio/) {
1915             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1916             $memory = ",max_hostmem=$bytes";
1917         } elsif ($qxlnum) {
1918             # from https://www.spice-space.org/multiple-monitors.html
1919             $memory = ",vgamem_mb=$vga->{memory}";
1920             my $ram = $vgamem_mb * 4;
1921             my $vram = $vgamem_mb * 2;
1922             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1923         } else {
1924             $memory = ",vgamem_mb=$vga->{memory}";
1925         }
1926     } elsif ($qxlnum && $id) {
1927         $memory = ",ram_size=67108864,vram_size=33554432";
1928     }
1929
1930     my $edidoff = "";
1931     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1932         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1933     }
1934
1935     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1936     my $vgaid = "vga" . ($id // '');
1937     my $pciaddr;
1938     if ($q35 && $vgaid eq 'vga') {
1939         # the first display uses pcie.0 bus on q35 machines
1940         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1941     } else {
1942         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1943     }
1944
1945     if ($vga->{type} eq 'virtio-gl') {
1946         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1947         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1948             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1949
1950         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1951             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1952     }
1953
1954     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1955 }
1956
1957 sub parse_number_sets {
1958     my ($set) = @_;
1959     my $res = [];
1960     foreach my $part (split(/;/, $set)) {
1961         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1962             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1963             push @$res, [ $1, $2 ];
1964         } else {
1965             die "invalid range: $part\n";
1966         }
1967     }
1968     return $res;
1969 }
1970
1971 sub parse_numa {
1972     my ($data) = @_;
1973
1974     my $res = parse_property_string($numa_fmt, $data);
1975     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1976     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1977     return $res;
1978 }
1979
1980 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1981 sub parse_net {
1982     my ($data, $disable_mac_autogen) = @_;
1983
1984     my $res = eval { parse_property_string($net_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return;
1988     }
1989     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1990         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1991         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1992     }
1993     return $res;
1994 }
1995
1996 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1997 sub parse_ipconfig {
1998     my ($data) = @_;
1999
2000     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
2001     if ($@) {
2002         warn $@;
2003         return;
2004     }
2005
2006     if ($res->{gw} && !$res->{ip}) {
2007         warn 'gateway specified without specifying an IP address';
2008         return;
2009     }
2010     if ($res->{gw6} && !$res->{ip6}) {
2011         warn 'IPv6 gateway specified without specifying an IPv6 address';
2012         return;
2013     }
2014     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2015         warn 'gateway specified together with DHCP';
2016         return;
2017     }
2018     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2019         # gw6 + auto/dhcp
2020         warn "IPv6 gateway specified together with $res->{ip6} address";
2021         return;
2022     }
2023
2024     if (!$res->{ip} && !$res->{ip6}) {
2025         return { ip => 'dhcp', ip6 => 'dhcp' };
2026     }
2027
2028     return $res;
2029 }
2030
2031 sub print_net {
2032     my $net = shift;
2033
2034     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2035 }
2036
2037 sub add_random_macs {
2038     my ($settings) = @_;
2039
2040     foreach my $opt (keys %$settings) {
2041         next if $opt !~ m/^net(\d+)$/;
2042         my $net = parse_net($settings->{$opt});
2043         next if !$net;
2044         $settings->{$opt} = print_net($net);
2045     }
2046 }
2047
2048 sub vm_is_volid_owner {
2049     my ($storecfg, $vmid, $volid) = @_;
2050
2051     if ($volid !~  m|^/|) {
2052         my ($path, $owner);
2053         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2054         if ($owner && ($owner == $vmid)) {
2055             return 1;
2056         }
2057     }
2058
2059     return;
2060 }
2061
2062 sub vmconfig_register_unused_drive {
2063     my ($storecfg, $vmid, $conf, $drive) = @_;
2064
2065     if (drive_is_cloudinit($drive)) {
2066         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2067         warn $@ if $@;
2068         delete $conf->{cloudinit};
2069     } elsif (!drive_is_cdrom($drive)) {
2070         my $volid = $drive->{file};
2071         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2072             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2073         }
2074     }
2075 }
2076
2077 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2078 my $smbios1_fmt = {
2079     uuid => {
2080         type => 'string',
2081         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2082         format_description => 'UUID',
2083         description => "Set SMBIOS1 UUID.",
2084         optional => 1,
2085     },
2086     version => {
2087         type => 'string',
2088         pattern => '[A-Za-z0-9+\/]+={0,2}',
2089         format_description => 'Base64 encoded string',
2090         description => "Set SMBIOS1 version.",
2091         optional => 1,
2092     },
2093     serial => {
2094         type => 'string',
2095         pattern => '[A-Za-z0-9+\/]+={0,2}',
2096         format_description => 'Base64 encoded string',
2097         description => "Set SMBIOS1 serial number.",
2098         optional => 1,
2099     },
2100     manufacturer => {
2101         type => 'string',
2102         pattern => '[A-Za-z0-9+\/]+={0,2}',
2103         format_description => 'Base64 encoded string',
2104         description => "Set SMBIOS1 manufacturer.",
2105         optional => 1,
2106     },
2107     product => {
2108         type => 'string',
2109         pattern => '[A-Za-z0-9+\/]+={0,2}',
2110         format_description => 'Base64 encoded string',
2111         description => "Set SMBIOS1 product ID.",
2112         optional => 1,
2113     },
2114     sku => {
2115         type => 'string',
2116         pattern => '[A-Za-z0-9+\/]+={0,2}',
2117         format_description => 'Base64 encoded string',
2118         description => "Set SMBIOS1 SKU string.",
2119         optional => 1,
2120     },
2121     family => {
2122         type => 'string',
2123         pattern => '[A-Za-z0-9+\/]+={0,2}',
2124         format_description => 'Base64 encoded string',
2125         description => "Set SMBIOS1 family string.",
2126         optional => 1,
2127     },
2128     base64 => {
2129         type => 'boolean',
2130         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2131         optional => 1,
2132     },
2133 };
2134
2135 sub parse_smbios1 {
2136     my ($data) = @_;
2137
2138     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2139     warn $@ if $@;
2140     return $res;
2141 }
2142
2143 sub print_smbios1 {
2144     my ($smbios1) = @_;
2145     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2146 }
2147
2148 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2149
2150 sub parse_watchdog {
2151     my ($value) = @_;
2152
2153     return if !$value;
2154
2155     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2156     warn $@ if $@;
2157     return $res;
2158 }
2159
2160 sub parse_guest_agent {
2161     my ($conf) = @_;
2162
2163     return {} if !defined($conf->{agent});
2164
2165     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2166     warn $@ if $@;
2167
2168     # if the agent is disabled ignore the other potentially set properties
2169     return {} if !$res->{enabled};
2170     return $res;
2171 }
2172
2173 sub get_qga_key {
2174     my ($conf, $key) = @_;
2175     return undef if !defined($conf->{agent});
2176
2177     my $agent = parse_guest_agent($conf);
2178     return $agent->{$key};
2179 }
2180
2181 sub parse_vga {
2182     my ($value) = @_;
2183
2184     return {} if !$value;
2185     my $res = eval { parse_property_string($vga_fmt, $value) };
2186     warn $@ if $@;
2187     return $res;
2188 }
2189
2190 sub parse_rng {
2191     my ($value) = @_;
2192
2193     return if !$value;
2194
2195     my $res = eval { parse_property_string($rng_fmt, $value) };
2196     warn $@ if $@;
2197     return $res;
2198 }
2199
2200 sub parse_meta_info {
2201     my ($value) = @_;
2202
2203     return if !$value;
2204
2205     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2206     warn $@ if $@;
2207     return $res;
2208 }
2209
2210 sub new_meta_info_string {
2211     my () = @_; # for now do not allow to override any value
2212
2213     return PVE::JSONSchema::print_property_string(
2214         {
2215             'creation-qemu' => kvm_user_version(),
2216             ctime => "". int(time()),
2217         },
2218         $meta_info_fmt
2219     );
2220 }
2221
2222 sub qemu_created_version_fixups {
2223     my ($conf, $forcemachine, $kvmver) = @_;
2224
2225     my $meta = parse_meta_info($conf->{meta}) // {};
2226     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2227
2228     # check if we need to apply some handling for VMs that always use the latest machine version but
2229     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2230     # would be required (we do not want to pin machine version for non-windows OS type)
2231     if (
2232         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2233         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2234         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2235         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2236     ) {
2237         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2238         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2239             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2240             # and thus with the predictable interface naming of systemd
2241             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2242         }
2243     }
2244     return;
2245 }
2246
2247 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2248 sub verify_usb_device {
2249     my ($value, $noerr) = @_;
2250
2251     return $value if parse_usb_device($value);
2252
2253     return if $noerr;
2254
2255     die "unable to parse usb device\n";
2256 }
2257
2258 # add JSON properties for create and set function
2259 sub json_config_properties {
2260     my ($prop, $with_disk_alloc) = @_;
2261
2262     my $skip_json_config_opts = {
2263         parent => 1,
2264         snaptime => 1,
2265         vmstate => 1,
2266         runningmachine => 1,
2267         runningcpu => 1,
2268         meta => 1,
2269     };
2270
2271     foreach my $opt (keys %$confdesc) {
2272         next if $skip_json_config_opts->{$opt};
2273
2274         if ($with_disk_alloc && is_valid_drivename($opt)) {
2275             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2276         } else {
2277             $prop->{$opt} = $confdesc->{$opt};
2278         }
2279     }
2280
2281     return $prop;
2282 }
2283
2284 # Properties that we can read from an OVF file
2285 sub json_ovf_properties {
2286     my $prop = {};
2287
2288     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2289         $prop->{$device} = {
2290             type => 'string',
2291             format => 'pve-volume-id-or-absolute-path',
2292             description => "Disk image that gets imported to $device",
2293             optional => 1,
2294         };
2295     }
2296
2297     $prop->{cores} = {
2298         type => 'integer',
2299         description => "The number of CPU cores.",
2300         optional => 1,
2301     };
2302     $prop->{memory} = {
2303         type => 'integer',
2304         description => "Amount of RAM for the VM in MB.",
2305         optional => 1,
2306     };
2307     $prop->{name} = {
2308         type => 'string',
2309         description => "Name of the VM.",
2310         optional => 1,
2311     };
2312
2313     return $prop;
2314 }
2315
2316 # return copy of $confdesc_cloudinit to generate documentation
2317 sub cloudinit_config_properties {
2318
2319     return dclone($confdesc_cloudinit);
2320 }
2321
2322 sub cloudinit_pending_properties {
2323     my $p = {
2324         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2325         name => 1,
2326     };
2327     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2328     return $p;
2329 }
2330
2331 sub check_type {
2332     my ($key, $value) = @_;
2333
2334     die "unknown setting '$key'\n" if !$confdesc->{$key};
2335
2336     my $type = $confdesc->{$key}->{type};
2337
2338     if (!defined($value)) {
2339         die "got undefined value\n";
2340     }
2341
2342     if ($value =~ m/[\n\r]/) {
2343         die "property contains a line feed\n";
2344     }
2345
2346     if ($type eq 'boolean') {
2347         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2348         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2349         die "type check ('boolean') failed - got '$value'\n";
2350     } elsif ($type eq 'integer') {
2351         return int($1) if $value =~ m/^(\d+)$/;
2352         die "type check ('integer') failed - got '$value'\n";
2353     } elsif ($type eq 'number') {
2354         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2355         die "type check ('number') failed - got '$value'\n";
2356     } elsif ($type eq 'string') {
2357         if (my $fmt = $confdesc->{$key}->{format}) {
2358             PVE::JSONSchema::check_format($fmt, $value);
2359             return $value;
2360         }
2361         $value =~ s/^\"(.*)\"$/$1/;
2362         return $value;
2363     } else {
2364         die "internal error"
2365     }
2366 }
2367
2368 sub destroy_vm {
2369     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2370
2371     my $conf = PVE::QemuConfig->load_config($vmid);
2372
2373     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2374         PVE::QemuConfig->check_lock($conf);
2375     }
2376
2377     if ($conf->{template}) {
2378         # check if any base image is still used by a linked clone
2379         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2380                 my ($ds, $drive) = @_;
2381                 return if drive_is_cdrom($drive);
2382
2383                 my $volid = $drive->{file};
2384                 return if !$volid || $volid =~ m|^/|;
2385
2386                 die "base volume '$volid' is still in use by linked cloned\n"
2387                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2388
2389         });
2390     }
2391
2392     my $volids = {};
2393     my $remove_owned_drive = sub {
2394         my ($ds, $drive) = @_;
2395         return if drive_is_cdrom($drive, 1);
2396
2397         my $volid = $drive->{file};
2398         return if !$volid || $volid =~ m|^/|;
2399         return if $volids->{$volid};
2400
2401         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2402         return if !$path || !$owner || ($owner != $vmid);
2403
2404         $volids->{$volid} = 1;
2405         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2406         warn "Could not remove disk '$volid', check manually: $@" if $@;
2407     };
2408
2409     # only remove disks owned by this VM (referenced in the config)
2410     my $include_opts = {
2411         include_unused => 1,
2412         extra_keys => ['vmstate'],
2413     };
2414     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2415
2416     for my $snap (values %{$conf->{snapshots}}) {
2417         next if !defined($snap->{vmstate});
2418         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2419         next if !defined($drive);
2420         $remove_owned_drive->('vmstate', $drive);
2421     }
2422
2423     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2424
2425     if ($purge_unreferenced) { # also remove unreferenced disk
2426         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2427         PVE::Storage::foreach_volid($vmdisks, sub {
2428             my ($volid, $sid, $volname, $d) = @_;
2429             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2430             warn $@ if $@;
2431         });
2432     }
2433
2434     if (defined $replacement_conf) {
2435         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2436     } else {
2437         PVE::QemuConfig->destroy_config($vmid);
2438     }
2439 }
2440
2441 sub parse_vm_config {
2442     my ($filename, $raw, $strict) = @_;
2443
2444     return if !defined($raw);
2445
2446     my $res = {
2447         digest => Digest::SHA::sha1_hex($raw),
2448         snapshots => {},
2449         pending => {},
2450         cloudinit => {},
2451     };
2452
2453     my $handle_error = sub {
2454         my ($msg) = @_;
2455
2456         if ($strict) {
2457             die $msg;
2458         } else {
2459             warn $msg;
2460         }
2461     };
2462
2463     $filename =~ m|/qemu-server/(\d+)\.conf$|
2464         || die "got strange filename '$filename'";
2465
2466     my $vmid = $1;
2467
2468     my $conf = $res;
2469     my $descr;
2470     my $finish_description = sub {
2471         if (defined($descr)) {
2472             $descr =~ s/\s+$//;
2473             $conf->{description} = $descr;
2474         }
2475         $descr = undef;
2476     };
2477     my $section = '';
2478
2479     my @lines = split(/\n/, $raw);
2480     foreach my $line (@lines) {
2481         next if $line =~ m/^\s*$/;
2482
2483         if ($line =~ m/^\[PENDING\]\s*$/i) {
2484             $section = 'pending';
2485             $finish_description->();
2486             $conf = $res->{$section} = {};
2487             next;
2488         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2489             $section = 'cloudinit';
2490             $finish_description->();
2491             $conf = $res->{$section} = {};
2492             next;
2493
2494         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2495             $section = $1;
2496             $finish_description->();
2497             $conf = $res->{snapshots}->{$section} = {};
2498             next;
2499         }
2500
2501         if ($line =~ m/^\#(.*)$/) {
2502             $descr = '' if !defined($descr);
2503             $descr .= PVE::Tools::decode_text($1) . "\n";
2504             next;
2505         }
2506
2507         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2508             $descr = '' if !defined($descr);
2509             $descr .= PVE::Tools::decode_text($2);
2510         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2511             $conf->{snapstate} = $1;
2512         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2513             my $key = $1;
2514             my $value = $2;
2515             $conf->{$key} = $value;
2516         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2517             my $value = $1;
2518             if ($section eq 'pending') {
2519                 $conf->{delete} = $value; # we parse this later
2520             } else {
2521                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2522             }
2523         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2524             my $key = $1;
2525             my $value = $2;
2526             if ($section eq 'cloudinit') {
2527                 # ignore validation only used for informative purpose
2528                 $conf->{$key} = $value;
2529                 next;
2530             }
2531             eval { $value = check_type($key, $value); };
2532             if ($@) {
2533                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2534             } else {
2535                 $key = 'ide2' if $key eq 'cdrom';
2536                 my $fmt = $confdesc->{$key}->{format};
2537                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2538                     my $v = parse_drive($key, $value);
2539                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2540                         $v->{file} = $volid;
2541                         $value = print_drive($v);
2542                     } else {
2543                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2544                         next;
2545                     }
2546                 }
2547
2548                 $conf->{$key} = $value;
2549             }
2550         } else {
2551             $handle_error->("vm $vmid - unable to parse config: $line\n");
2552         }
2553     }
2554
2555     $finish_description->();
2556     delete $res->{snapstate}; # just to be sure
2557
2558     return $res;
2559 }
2560
2561 sub write_vm_config {
2562     my ($filename, $conf) = @_;
2563
2564     delete $conf->{snapstate}; # just to be sure
2565
2566     if ($conf->{cdrom}) {
2567         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2568         $conf->{ide2} = $conf->{cdrom};
2569         delete $conf->{cdrom};
2570     }
2571
2572     # we do not use 'smp' any longer
2573     if ($conf->{sockets}) {
2574         delete $conf->{smp};
2575     } elsif ($conf->{smp}) {
2576         $conf->{sockets} = $conf->{smp};
2577         delete $conf->{cores};
2578         delete $conf->{smp};
2579     }
2580
2581     my $used_volids = {};
2582
2583     my $cleanup_config = sub {
2584         my ($cref, $pending, $snapname) = @_;
2585
2586         foreach my $key (keys %$cref) {
2587             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2588                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2589             my $value = $cref->{$key};
2590             if ($key eq 'delete') {
2591                 die "propertry 'delete' is only allowed in [PENDING]\n"
2592                     if !$pending;
2593                 # fixme: check syntax?
2594                 next;
2595             }
2596             eval { $value = check_type($key, $value); };
2597             die "unable to parse value of '$key' - $@" if $@;
2598
2599             $cref->{$key} = $value;
2600
2601             if (!$snapname && is_valid_drivename($key)) {
2602                 my $drive = parse_drive($key, $value);
2603                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2604             }
2605         }
2606     };
2607
2608     &$cleanup_config($conf);
2609
2610     &$cleanup_config($conf->{pending}, 1);
2611
2612     foreach my $snapname (keys %{$conf->{snapshots}}) {
2613         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2614         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2615     }
2616
2617     # remove 'unusedX' settings if we re-add a volume
2618     foreach my $key (keys %$conf) {
2619         my $value = $conf->{$key};
2620         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2621             delete $conf->{$key};
2622         }
2623     }
2624
2625     my $generate_raw_config = sub {
2626         my ($conf, $pending) = @_;
2627
2628         my $raw = '';
2629
2630         # add description as comment to top of file
2631         if (defined(my $descr = $conf->{description})) {
2632             if ($descr) {
2633                 foreach my $cl (split(/\n/, $descr)) {
2634                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2635                 }
2636             } else {
2637                 $raw .= "#\n" if $pending;
2638             }
2639         }
2640
2641         foreach my $key (sort keys %$conf) {
2642             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2643             $raw .= "$key: $conf->{$key}\n";
2644         }
2645         return $raw;
2646     };
2647
2648     my $raw = &$generate_raw_config($conf);
2649
2650     if (scalar(keys %{$conf->{pending}})){
2651         $raw .= "\n[PENDING]\n";
2652         $raw .= &$generate_raw_config($conf->{pending}, 1);
2653     }
2654
2655     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2656         $raw .= "\n[special:cloudinit]\n";
2657         $raw .= &$generate_raw_config($conf->{cloudinit});
2658     }
2659
2660     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2661         $raw .= "\n[$snapname]\n";
2662         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2663     }
2664
2665     return $raw;
2666 }
2667
2668 sub load_defaults {
2669
2670     my $res = {};
2671
2672     # we use static defaults from our JSON schema configuration
2673     foreach my $key (keys %$confdesc) {
2674         if (defined(my $default = $confdesc->{$key}->{default})) {
2675             $res->{$key} = $default;
2676         }
2677     }
2678
2679     return $res;
2680 }
2681
2682 sub config_list {
2683     my $vmlist = PVE::Cluster::get_vmlist();
2684     my $res = {};
2685     return $res if !$vmlist || !$vmlist->{ids};
2686     my $ids = $vmlist->{ids};
2687     my $nodename = nodename();
2688
2689     foreach my $vmid (keys %$ids) {
2690         my $d = $ids->{$vmid};
2691         next if !$d->{node} || $d->{node} ne $nodename;
2692         next if !$d->{type} || $d->{type} ne 'qemu';
2693         $res->{$vmid}->{exists} = 1;
2694     }
2695     return $res;
2696 }
2697
2698 # test if VM uses local resources (to prevent migration)
2699 sub check_local_resources {
2700     my ($conf, $noerr) = @_;
2701
2702     my @loc_res = ();
2703
2704     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2705     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2706
2707     push @loc_res, "ivshmem" if $conf->{ivshmem};
2708
2709     foreach my $k (keys %$conf) {
2710         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2711         # sockets are safe: they will recreated be on the target side post-migrate
2712         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2713         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2714     }
2715
2716     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2717
2718     return \@loc_res;
2719 }
2720
2721 # check if used storages are available on all nodes (use by migrate)
2722 sub check_storage_availability {
2723     my ($storecfg, $conf, $node) = @_;
2724
2725     PVE::QemuConfig->foreach_volume($conf, sub {
2726         my ($ds, $drive) = @_;
2727
2728         my $volid = $drive->{file};
2729         return if !$volid;
2730
2731         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2732         return if !$sid;
2733
2734         # check if storage is available on both nodes
2735         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2736         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2737
2738         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2739
2740         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2741             if !$scfg->{content}->{$vtype};
2742    });
2743 }
2744
2745 # list nodes where all VM images are available (used by has_feature API)
2746 sub shared_nodes {
2747     my ($conf, $storecfg) = @_;
2748
2749     my $nodelist = PVE::Cluster::get_nodelist();
2750     my $nodehash = { map { $_ => 1 } @$nodelist };
2751     my $nodename = nodename();
2752
2753     PVE::QemuConfig->foreach_volume($conf, sub {
2754         my ($ds, $drive) = @_;
2755
2756         my $volid = $drive->{file};
2757         return if !$volid;
2758
2759         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2760         if ($storeid) {
2761             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2762             if ($scfg->{disable}) {
2763                 $nodehash = {};
2764             } elsif (my $avail = $scfg->{nodes}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     delete $nodehash->{$node} if !$avail->{$node};
2767                 }
2768             } elsif (!$scfg->{shared}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     delete $nodehash->{$node} if $node ne $nodename
2771                 }
2772             }
2773         }
2774     });
2775
2776     return $nodehash
2777 }
2778
2779 sub check_local_storage_availability {
2780     my ($conf, $storecfg) = @_;
2781
2782     my $nodelist = PVE::Cluster::get_nodelist();
2783     my $nodehash = { map { $_ => {} } @$nodelist };
2784
2785     PVE::QemuConfig->foreach_volume($conf, sub {
2786         my ($ds, $drive) = @_;
2787
2788         my $volid = $drive->{file};
2789         return if !$volid;
2790
2791         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2792         if ($storeid) {
2793             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2794
2795             if ($scfg->{disable}) {
2796                 foreach my $node (keys %$nodehash) {
2797                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2798                 }
2799             } elsif (my $avail = $scfg->{nodes}) {
2800                 foreach my $node (keys %$nodehash) {
2801                     if (!$avail->{$node}) {
2802                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2803                     }
2804                 }
2805             }
2806         }
2807     });
2808
2809     foreach my $node (values %$nodehash) {
2810         if (my $unavail = $node->{unavailable_storages}) {
2811             $node->{unavailable_storages} = [ sort keys %$unavail ];
2812         }
2813     }
2814
2815     return $nodehash
2816 }
2817
2818 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2819 sub check_running {
2820     my ($vmid, $nocheck, $node) = @_;
2821
2822     # $nocheck is set when called during a migration, in which case the config
2823     # file might still or already reside on the *other* node
2824     # - because rename has already happened, and current node is source
2825     # - because rename hasn't happened yet, and current node is target
2826     # - because rename has happened, current node is target, but hasn't yet
2827     # processed it yet
2828     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2829     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2830 }
2831
2832 sub vzlist {
2833
2834     my $vzlist = config_list();
2835
2836     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2837
2838     while (defined(my $de = $fd->read)) {
2839         next if $de !~ m/^(\d+)\.pid$/;
2840         my $vmid = $1;
2841         next if !defined($vzlist->{$vmid});
2842         if (my $pid = check_running($vmid)) {
2843             $vzlist->{$vmid}->{pid} = $pid;
2844         }
2845     }
2846
2847     return $vzlist;
2848 }
2849
2850 our $vmstatus_return_properties = {
2851     vmid => get_standard_option('pve-vmid'),
2852     status => {
2853         description => "QEMU process status.",
2854         type => 'string',
2855         enum => ['stopped', 'running'],
2856     },
2857     maxmem => {
2858         description => "Maximum memory in bytes.",
2859         type => 'integer',
2860         optional => 1,
2861         renderer => 'bytes',
2862     },
2863     maxdisk => {
2864         description => "Root disk size in bytes.",
2865         type => 'integer',
2866         optional => 1,
2867         renderer => 'bytes',
2868     },
2869     name => {
2870         description => "VM name.",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     qmpstatus => {
2875         description => "QEMU QMP agent status.",
2876         type => 'string',
2877         optional => 1,
2878     },
2879     pid => {
2880         description => "PID of running qemu process.",
2881         type => 'integer',
2882         optional => 1,
2883     },
2884     uptime => {
2885         description => "Uptime.",
2886         type => 'integer',
2887         optional => 1,
2888         renderer => 'duration',
2889     },
2890     cpus => {
2891         description => "Maximum usable CPUs.",
2892         type => 'number',
2893         optional => 1,
2894     },
2895     lock => {
2896         description => "The current config lock, if any.",
2897         type => 'string',
2898         optional => 1,
2899     },
2900     tags => {
2901         description  => "The current configured tags, if any",
2902         type => 'string',
2903         optional => 1,
2904     },
2905     'running-machine' => {
2906         description => "The currently running machine type (if running).",
2907         type => 'string',
2908         optional => 1,
2909     },
2910     'running-qemu' => {
2911         description => "The currently running QEMU version (if running).",
2912         type => 'string',
2913         optional => 1,
2914     },
2915 };
2916
2917 my $last_proc_pid_stat;
2918
2919 # get VM status information
2920 # This must be fast and should not block ($full == false)
2921 # We only query KVM using QMP if $full == true (this can be slow)
2922 sub vmstatus {
2923     my ($opt_vmid, $full) = @_;
2924
2925     my $res = {};
2926
2927     my $storecfg = PVE::Storage::config();
2928
2929     my $list = vzlist();
2930     my $defaults = load_defaults();
2931
2932     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2933
2934     my $cpucount = $cpuinfo->{cpus} || 1;
2935
2936     foreach my $vmid (keys %$list) {
2937         next if $opt_vmid && ($vmid ne $opt_vmid);
2938
2939         my $conf = PVE::QemuConfig->load_config($vmid);
2940
2941         my $d = { vmid => int($vmid) };
2942         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2943
2944         # fixme: better status?
2945         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2946
2947         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2948         if (defined($size)) {
2949             $d->{disk} = 0; # no info available
2950             $d->{maxdisk} = $size;
2951         } else {
2952             $d->{disk} = 0;
2953             $d->{maxdisk} = 0;
2954         }
2955
2956         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2957             * ($conf->{cores} || $defaults->{cores});
2958         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2959         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2960
2961         $d->{name} = $conf->{name} || "VM $vmid";
2962         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2963             : $defaults->{memory}*(1024*1024);
2964
2965         if ($conf->{balloon}) {
2966             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2967             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2968                 : $defaults->{shares};
2969         }
2970
2971         $d->{uptime} = 0;
2972         $d->{cpu} = 0;
2973         $d->{mem} = 0;
2974
2975         $d->{netout} = 0;
2976         $d->{netin} = 0;
2977
2978         $d->{diskread} = 0;
2979         $d->{diskwrite} = 0;
2980
2981         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2982
2983         $d->{serial} = 1 if conf_has_serial($conf);
2984         $d->{lock} = $conf->{lock} if $conf->{lock};
2985         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2986
2987         $res->{$vmid} = $d;
2988     }
2989
2990     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2991     foreach my $dev (keys %$netdev) {
2992         next if $dev !~ m/^tap([1-9]\d*)i/;
2993         my $vmid = $1;
2994         my $d = $res->{$vmid};
2995         next if !$d;
2996
2997         $d->{netout} += $netdev->{$dev}->{receive};
2998         $d->{netin} += $netdev->{$dev}->{transmit};
2999
3000         if ($full) {
3001             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
3002             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
3003         }
3004
3005     }
3006
3007     my $ctime = gettimeofday;
3008
3009     foreach my $vmid (keys %$list) {
3010
3011         my $d = $res->{$vmid};
3012         my $pid = $d->{pid};
3013         next if !$pid;
3014
3015         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3016         next if !$pstat; # not running
3017
3018         my $used = $pstat->{utime} + $pstat->{stime};
3019
3020         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3021
3022         if ($pstat->{vsize}) {
3023             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3024         }
3025
3026         my $old = $last_proc_pid_stat->{$pid};
3027         if (!$old) {
3028             $last_proc_pid_stat->{$pid} = {
3029                 time => $ctime,
3030                 used => $used,
3031                 cpu => 0,
3032             };
3033             next;
3034         }
3035
3036         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3037
3038         if ($dtime > 1000) {
3039             my $dutime = $used -  $old->{used};
3040
3041             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3042             $last_proc_pid_stat->{$pid} = {
3043                 time => $ctime,
3044                 used => $used,
3045                 cpu => $d->{cpu},
3046             };
3047         } else {
3048             $d->{cpu} = $old->{cpu};
3049         }
3050     }
3051
3052     return $res if !$full;
3053
3054     my $qmpclient = PVE::QMPClient->new();
3055
3056     my $ballooncb = sub {
3057         my ($vmid, $resp) = @_;
3058
3059         my $info = $resp->{'return'};
3060         return if !$info->{max_mem};
3061
3062         my $d = $res->{$vmid};
3063
3064         # use memory assigned to VM
3065         $d->{maxmem} = $info->{max_mem};
3066         $d->{balloon} = $info->{actual};
3067
3068         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3069             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3070             $d->{freemem} = $info->{free_mem};
3071         }
3072
3073         $d->{ballooninfo} = $info;
3074     };
3075
3076     my $blockstatscb = sub {
3077         my ($vmid, $resp) = @_;
3078         my $data = $resp->{'return'} || [];
3079         my $totalrdbytes = 0;
3080         my $totalwrbytes = 0;
3081
3082         for my $blockstat (@$data) {
3083             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3084             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3085
3086             $blockstat->{device} =~ s/drive-//;
3087             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3088         }
3089         $res->{$vmid}->{diskread} = $totalrdbytes;
3090         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3091     };
3092
3093     my $machinecb = sub {
3094         my ($vmid, $resp) = @_;
3095         my $data = $resp->{'return'} || [];
3096
3097         $res->{$vmid}->{'running-machine'} =
3098             PVE::QemuServer::Machine::current_from_query_machines($data);
3099     };
3100
3101     my $versioncb = sub {
3102         my ($vmid, $resp) = @_;
3103         my $data = $resp->{'return'} // {};
3104         my $version = 'unknown';
3105
3106         if (my $v = $data->{qemu}) {
3107             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3108         }
3109
3110         $res->{$vmid}->{'running-qemu'} = $version;
3111     };
3112
3113     my $statuscb = sub {
3114         my ($vmid, $resp) = @_;
3115
3116         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3117         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3118         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3119         # this fails if ballon driver is not loaded, so this must be
3120         # the last commnand (following command are aborted if this fails).
3121         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3122
3123         my $status = 'unknown';
3124         if (!defined($status = $resp->{'return'}->{status})) {
3125             warn "unable to get VM status\n";
3126             return;
3127         }
3128
3129         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3130     };
3131
3132     foreach my $vmid (keys %$list) {
3133         next if $opt_vmid && ($vmid ne $opt_vmid);
3134         next if !$res->{$vmid}->{pid}; # not running
3135         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3136     }
3137
3138     $qmpclient->queue_execute(undef, 2);
3139
3140     foreach my $vmid (keys %$list) {
3141         next if $opt_vmid && ($vmid ne $opt_vmid);
3142         next if !$res->{$vmid}->{pid}; #not running
3143
3144         # we can't use the $qmpclient since it might have already aborted on
3145         # 'query-balloon', but this might also fail for older versions...
3146         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3147         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3148     }
3149
3150     foreach my $vmid (keys %$list) {
3151         next if $opt_vmid && ($vmid ne $opt_vmid);
3152         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3153     }
3154
3155     return $res;
3156 }
3157
3158 sub conf_has_serial {
3159     my ($conf) = @_;
3160
3161     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3162         if ($conf->{"serial$i"}) {
3163             return 1;
3164         }
3165     }
3166
3167     return 0;
3168 }
3169
3170 sub conf_has_audio {
3171     my ($conf, $id) = @_;
3172
3173     $id //= 0;
3174     my $audio = $conf->{"audio$id"};
3175     return if !defined($audio);
3176
3177     my $audioproperties = parse_property_string($audio_fmt, $audio);
3178     my $audiodriver = $audioproperties->{driver} // 'spice';
3179
3180     return {
3181         dev => $audioproperties->{device},
3182         dev_id => "audiodev$id",
3183         backend => $audiodriver,
3184         backend_id => "$audiodriver-backend${id}",
3185     };
3186 }
3187
3188 sub audio_devs {
3189     my ($audio, $audiopciaddr, $machine_version) = @_;
3190
3191     my $devs = [];
3192
3193     my $id = $audio->{dev_id};
3194     my $audiodev = "";
3195     if (min_version($machine_version, 4, 2)) {
3196         $audiodev = ",audiodev=$audio->{backend_id}";
3197     }
3198
3199     if ($audio->{dev} eq 'AC97') {
3200         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3201     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3202         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3203         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3204         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3205     } else {
3206         die "unkown audio device '$audio->{dev}', implement me!";
3207     }
3208
3209     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3210
3211     return $devs;
3212 }
3213
3214 sub get_tpm_paths {
3215     my ($vmid) = @_;
3216     return {
3217         socket => "/var/run/qemu-server/$vmid.swtpm",
3218         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3219     };
3220 }
3221
3222 sub add_tpm_device {
3223     my ($vmid, $devices, $conf) = @_;
3224
3225     return if !$conf->{tpmstate0};
3226
3227     my $paths = get_tpm_paths($vmid);
3228
3229     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3230     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3231     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3232 }
3233
3234 sub start_swtpm {
3235     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3236
3237     return if !$tpmdrive;
3238
3239     my $state;
3240     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3241     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3242     if ($storeid) {
3243         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3244     } else {
3245         $state = $tpm->{file};
3246     }
3247
3248     my $paths = get_tpm_paths($vmid);
3249
3250     # during migration, we will get state from remote
3251     #
3252     if (!$migration) {
3253         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3254         my $setup_cmd = [
3255             "swtpm_setup",
3256             "--tpmstate",
3257             "file://$state",
3258             "--createek",
3259             "--create-ek-cert",
3260             "--create-platform-cert",
3261             "--lock-nvram",
3262             "--config",
3263             "/etc/swtpm_setup.conf", # do not use XDG configs
3264             "--runas",
3265             "0", # force creation as root, error if not possible
3266             "--not-overwrite", # ignore existing state, do not modify
3267         ];
3268
3269         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3270         # TPM 2.0 supports ECC crypto, use if possible
3271         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3272
3273         run_command($setup_cmd, outfunc => sub {
3274             print "swtpm_setup: $1\n";
3275         });
3276     }
3277
3278     # Used to distinguish different invocations in the log.
3279     my $log_prefix = "[id=" . int(time()) . "] ";
3280
3281     my $emulator_cmd = [
3282         "swtpm",
3283         "socket",
3284         "--tpmstate",
3285         "backend-uri=file://$state,mode=0600",
3286         "--ctrl",
3287         "type=unixio,path=$paths->{socket},mode=0600",
3288         "--pid",
3289         "file=$paths->{pid}",
3290         "--terminate", # terminate on QEMU disconnect
3291         "--daemon",
3292         "--log",
3293         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3294     ];
3295     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3296     run_command($emulator_cmd, outfunc => sub { print $1; });
3297
3298     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3299     while (! -e $paths->{pid}) {
3300         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3301         usleep(50_000);
3302     }
3303
3304     # return untainted PID of swtpm daemon so it can be killed on error
3305     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3306     return $1;
3307 }
3308
3309 sub vga_conf_has_spice {
3310     my ($vga) = @_;
3311
3312     my $vgaconf = parse_vga($vga);
3313     my $vgatype = $vgaconf->{type};
3314     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3315
3316     return $1 || 1;
3317 }
3318
3319 sub is_native($) {
3320     my ($arch) = @_;
3321     return get_host_arch() eq $arch;
3322 }
3323
3324 sub get_vm_arch {
3325     my ($conf) = @_;
3326     return $conf->{arch} // get_host_arch();
3327 }
3328
3329 my $default_machines = {
3330     x86_64 => 'pc',
3331     aarch64 => 'virt',
3332 };
3333
3334 sub get_installed_machine_version {
3335     my ($kvmversion) = @_;
3336     $kvmversion = kvm_user_version() if !defined($kvmversion);
3337     $kvmversion =~ m/^(\d+\.\d+)/;
3338     return $1;
3339 }
3340
3341 sub windows_get_pinned_machine_version {
3342     my ($machine, $base_version, $kvmversion) = @_;
3343
3344     my $pin_version = $base_version;
3345     if (!defined($base_version) ||
3346         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3347     ) {
3348         $pin_version = get_installed_machine_version($kvmversion);
3349     }
3350     if (!$machine || $machine eq 'pc') {
3351         $machine = "pc-i440fx-$pin_version";
3352     } elsif ($machine eq 'q35') {
3353         $machine = "pc-q35-$pin_version";
3354     } elsif ($machine eq 'virt') {
3355         $machine = "virt-$pin_version";
3356     } else {
3357         warn "unknown machine type '$machine', not touching that!\n";
3358     }
3359
3360     return $machine;
3361 }
3362
3363 sub get_vm_machine {
3364     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3365
3366     my $machine = $forcemachine || $conf->{machine};
3367
3368     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3369         $kvmversion //= kvm_user_version();
3370         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3371         # layout which confuses windows quite a bit and may result in various regressions..
3372         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3373         if (windows_version($conf->{ostype})) {
3374             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3375         }
3376         $arch //= 'x86_64';
3377         $machine ||= $default_machines->{$arch};
3378         if ($add_pve_version) {
3379             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3380             $machine .= "+pve$pvever";
3381         }
3382     }
3383
3384     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3385         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3386         $machine = $1 if $is_pxe;
3387
3388         # for version-pinned machines that do not include a pve-version (e.g.
3389         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3390         $machine .= '+pve0';
3391
3392         $machine .= '.pxe' if $is_pxe;
3393     }
3394
3395     return $machine;
3396 }
3397
3398 sub get_ovmf_files($$$) {
3399     my ($arch, $efidisk, $smm) = @_;
3400
3401     my $types = $OVMF->{$arch}
3402         or die "no OVMF images known for architecture '$arch'\n";
3403
3404     my $type = 'default';
3405     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3406         $type = $smm ? "4m" : "4m-no-smm";
3407         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3408     }
3409
3410     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3411     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3412     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3413
3414     return ($ovmf_code, $ovmf_vars);
3415 }
3416
3417 my $Arch2Qemu = {
3418     aarch64 => '/usr/bin/qemu-system-aarch64',
3419     x86_64 => '/usr/bin/qemu-system-x86_64',
3420 };
3421 sub get_command_for_arch($) {
3422     my ($arch) = @_;
3423     return '/usr/bin/kvm' if is_native($arch);
3424
3425     my $cmd = $Arch2Qemu->{$arch}
3426         or die "don't know how to emulate architecture '$arch'\n";
3427     return $cmd;
3428 }
3429
3430 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3431 # to use in a QEMU command line (-cpu element), first array_intersect the result
3432 # of query_supported_ with query_understood_. This is necessary because:
3433 #
3434 # a) query_understood_ returns flags the host cannot use and
3435 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3436 #    flags, but CPU settings - with most of them being flags. Those settings
3437 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3438 #
3439 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3440 # expensive. If you need the value returned from this, you can get it much
3441 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3442 # $accel being 'kvm' or 'tcg'.
3443 #
3444 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3445 # changes, automatically populating pmxcfs.
3446 #
3447 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3448 # since kvm and tcg machines support different flags
3449 #
3450 sub query_supported_cpu_flags {
3451     my ($arch) = @_;
3452
3453     $arch //= get_host_arch();
3454     my $default_machine = $default_machines->{$arch};
3455
3456     my $flags = {};
3457
3458     # FIXME: Once this is merged, the code below should work for ARM as well:
3459     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3460     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3461         $arch eq "aarch64";
3462
3463     my $kvm_supported = defined(kvm_version());
3464     my $qemu_cmd = get_command_for_arch($arch);
3465     my $fakevmid = -1;
3466     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3467
3468     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3469     my $query_supported_run_qemu = sub {
3470         my ($kvm) = @_;
3471
3472         my $flags = {};
3473         my $cmd = [
3474             $qemu_cmd,
3475             '-machine', $default_machine,
3476             '-display', 'none',
3477             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3478             '-mon', 'chardev=qmp,mode=control',
3479             '-pidfile', $pidfile,
3480             '-S', '-daemonize'
3481         ];
3482
3483         if (!$kvm) {
3484             push @$cmd, '-accel', 'tcg';
3485         }
3486
3487         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3488         die "QEMU flag querying VM exited with code " . $rc if $rc;
3489
3490         eval {
3491             my $cmd_result = mon_cmd(
3492                 $fakevmid,
3493                 'query-cpu-model-expansion',
3494                 type => 'full',
3495                 model => { name => 'host' }
3496             );
3497
3498             my $props = $cmd_result->{model}->{props};
3499             foreach my $prop (keys %$props) {
3500                 next if $props->{$prop} ne '1';
3501                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3502                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3503                 # We only keep those with underscores, to match /proc/cpuinfo
3504                 $prop =~ s/\.|-/_/g;
3505                 $flags->{$prop} = 1;
3506             }
3507         };
3508         my $err = $@;
3509
3510         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3511         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3512
3513         die $err if $err;
3514
3515         return [ sort keys %$flags ];
3516     };
3517
3518     # We need to query QEMU twice, since KVM and TCG have different supported flags
3519     PVE::QemuConfig->lock_config($fakevmid, sub {
3520         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3521         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3522
3523         if ($kvm_supported) {
3524             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3525             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3526         }
3527     });
3528
3529     return $flags;
3530 }
3531
3532 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3533 my $understood_cpu_flag_dir = "/usr/share/kvm";
3534 sub query_understood_cpu_flags {
3535     my $arch = get_host_arch();
3536     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3537
3538     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3539         if ! -e $filepath;
3540
3541     my $raw = file_get_contents($filepath);
3542     $raw =~ s/^\s+|\s+$//g;
3543     my @flags = split(/\s+/, $raw);
3544
3545     return \@flags;
3546 }
3547
3548 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3549 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3550 my sub should_disable_smm {
3551     my ($conf, $vga, $machine) = @_;
3552
3553     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3554
3555     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3556         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3557 }
3558
3559 my sub print_ovmf_drive_commandlines {
3560     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3561
3562     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3563
3564     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3565
3566     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3567     if ($d) {
3568         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3569         my ($path, $format) = $d->@{'file', 'format'};
3570         if ($storeid) {
3571             $path = PVE::Storage::path($storecfg, $d->{file});
3572             if (!defined($format)) {
3573                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3574                 $format = qemu_img_format($scfg, $volname);
3575             }
3576         } elsif (!defined($format)) {
3577             die "efidisk format must be specified\n";
3578         }
3579         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3580         if ($path =~ m/^rbd:/) {
3581             $var_drive_str .= ',cache=writeback';
3582             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3583         }
3584         $var_drive_str .= ",format=$format,file=$path";
3585
3586         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3587         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3588     } else {
3589         log_warn("no efidisk configured! Using temporary efivars disk.");
3590         my $path = "/tmp/$vmid-ovmf.fd";
3591         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3592         $var_drive_str .= ",format=raw,file=$path";
3593         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3594     }
3595
3596     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3597 }
3598
3599 sub config_to_command {
3600     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3601         $pbs_backing) = @_;
3602
3603     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3604     my $devices = [];
3605     my $bridges = {};
3606     my $ostype = $conf->{ostype};
3607     my $winversion = windows_version($ostype);
3608     my $kvm = $conf->{kvm};
3609     my $nodename = nodename();
3610
3611     my $arch = get_vm_arch($conf);
3612     my $kvm_binary = get_command_for_arch($arch);
3613     my $kvmver = kvm_user_version($kvm_binary);
3614
3615     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3616         $kvmver //= "undefined";
3617         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3618     }
3619
3620     my $add_pve_version = min_version($kvmver, 4, 1);
3621
3622     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3623     my $machine_version = extract_version($machine_type, $kvmver);
3624     $kvm //= 1 if is_native($arch);
3625
3626     $machine_version =~ m/(\d+)\.(\d+)/;
3627     my ($machine_major, $machine_minor) = ($1, $2);
3628
3629     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3630         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3631     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3632         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3633             ." please upgrade node '$nodename'\n"
3634     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3635         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3636         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3637             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3638             ." node '$nodename'\n";
3639     }
3640
3641     # if a specific +pve version is required for a feature, use $version_guard
3642     # instead of min_version to allow machines to be run with the minimum
3643     # required version
3644     my $required_pve_version = 0;
3645     my $version_guard = sub {
3646         my ($major, $minor, $pve) = @_;
3647         return 0 if !min_version($machine_version, $major, $minor, $pve);
3648         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3649         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3650         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3651         return 1;
3652     };
3653
3654     if ($kvm && !defined kvm_version()) {
3655         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3656             ." or enable in BIOS.\n";
3657     }
3658
3659     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3660     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3661     my $use_old_bios_files = undef;
3662     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3663
3664     my $cmd = [];
3665     if ($conf->{affinity}) {
3666         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3667     }
3668
3669     push @$cmd, $kvm_binary;
3670
3671     push @$cmd, '-id', $vmid;
3672
3673     my $vmname = $conf->{name} || "vm$vmid";
3674
3675     push @$cmd, '-name', "$vmname,debug-threads=on";
3676
3677     push @$cmd, '-no-shutdown';
3678
3679     my $use_virtio = 0;
3680
3681     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3682     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3683     push @$cmd, '-mon', "chardev=qmp,mode=control";
3684
3685     if (min_version($machine_version, 2, 12)) {
3686         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3687         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3688     }
3689
3690     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3691
3692     push @$cmd, '-daemonize';
3693
3694     if ($conf->{smbios1}) {
3695         my $smbios_conf = parse_smbios1($conf->{smbios1});
3696         if ($smbios_conf->{base64}) {
3697             # Do not pass base64 flag to qemu
3698             delete $smbios_conf->{base64};
3699             my $smbios_string = "";
3700             foreach my $key (keys %$smbios_conf) {
3701                 my $value;
3702                 if ($key eq "uuid") {
3703                     $value = $smbios_conf->{uuid}
3704                 } else {
3705                     $value = decode_base64($smbios_conf->{$key});
3706                 }
3707                 # qemu accepts any binary data, only commas need escaping by double comma
3708                 $value =~ s/,/,,/g;
3709                 $smbios_string .= "," . $key . "=" . $value if $value;
3710             }
3711             push @$cmd, '-smbios', "type=1" . $smbios_string;
3712         } else {
3713             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3714         }
3715     }
3716
3717     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3718         my ($code_drive_str, $var_drive_str) =
3719             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3720         push $cmd->@*, '-drive', $code_drive_str;
3721         push $cmd->@*, '-drive', $var_drive_str;
3722     }
3723
3724     if ($q35) { # tell QEMU to load q35 config early
3725         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3726         if (min_version($machine_version, 4, 0)) {
3727             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3728         } else {
3729             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3730         }
3731     }
3732
3733     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3734         push @$cmd, $fixups->@*;
3735     }
3736
3737     if ($conf->{vmgenid}) {
3738         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3739     }
3740
3741     # add usb controllers
3742     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3743         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3744     push @$devices, @usbcontrollers if @usbcontrollers;
3745     my $vga = parse_vga($conf->{vga});
3746
3747     my $qxlnum = vga_conf_has_spice($conf->{vga});
3748     $vga->{type} = 'qxl' if $qxlnum;
3749
3750     if (!$vga->{type}) {
3751         if ($arch eq 'aarch64') {
3752             $vga->{type} = 'virtio';
3753         } elsif (min_version($machine_version, 2, 9)) {
3754             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3755         } else {
3756             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3757         }
3758     }
3759
3760     # enable absolute mouse coordinates (needed by vnc)
3761     my $tablet = $conf->{tablet};
3762     if (!defined($tablet)) {
3763         $tablet = $defaults->{tablet};
3764         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3765         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3766     }
3767
3768     if ($tablet) {
3769         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3770         my $kbd = print_keyboarddevice_full($conf, $arch);
3771         push @$devices, '-device', $kbd if defined($kbd);
3772     }
3773
3774     my $bootorder = device_bootorder($conf);
3775
3776     # host pci device passthrough
3777     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3778         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3779
3780     # usb devices
3781     my $usb_dev_features = {};
3782     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3783
3784     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3785         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3786     push @$devices, @usbdevices if @usbdevices;
3787
3788     # serial devices
3789     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3790         my $path = $conf->{"serial$i"} or next;
3791         if ($path eq 'socket') {
3792             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3793             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3794             # On aarch64, serial0 is the UART device. QEMU only allows
3795             # connecting UART devices via the '-serial' command line, as
3796             # the device has a fixed slot on the hardware...
3797             if ($arch eq 'aarch64' && $i == 0) {
3798                 push @$devices, '-serial', "chardev:serial$i";
3799             } else {
3800                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3801             }
3802         } else {
3803             die "no such serial device\n" if ! -c $path;
3804             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3805             push @$devices, '-device', "isa-serial,chardev=serial$i";
3806         }
3807     }
3808
3809     # parallel devices
3810     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3811         if (my $path = $conf->{"parallel$i"}) {
3812             die "no such parallel device\n" if ! -c $path;
3813             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3814             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3815             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3816         }
3817     }
3818
3819     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3820         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3821         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3822         push @$devices, @$audio_devs;
3823     }
3824
3825     add_tpm_device($vmid, $devices, $conf);
3826
3827     my $sockets = 1;
3828     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3829     $sockets = $conf->{sockets} if  $conf->{sockets};
3830
3831     my $cores = $conf->{cores} || 1;
3832
3833     my $maxcpus = $sockets * $cores;
3834
3835     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3836
3837     my $allowed_vcpus = $cpuinfo->{cpus};
3838
3839     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3840
3841     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3842         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3843         for (my $i = 2; $i <= $vcpus; $i++)  {
3844             my $cpustr = print_cpu_device($conf,$i);
3845             push @$cmd, '-device', $cpustr;
3846         }
3847
3848     } else {
3849
3850         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3851     }
3852     push @$cmd, '-nodefaults';
3853
3854     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3855
3856     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3857
3858     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3859
3860     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3861         push @$devices, '-device', print_vga_device(
3862             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3863
3864         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3865
3866         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3867         push @$cmd,  '-vnc', "unix:$socket,password=on";
3868     } else {
3869         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3870         push @$cmd, '-nographic';
3871     }
3872
3873     # time drift fix
3874     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3875     my $useLocaltime = $conf->{localtime};
3876
3877     if ($winversion >= 5) { # windows
3878         $useLocaltime = 1 if !defined($conf->{localtime});
3879
3880         # use time drift fix when acpi is enabled
3881         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3882             $tdf = 1 if !defined($conf->{tdf});
3883         }
3884     }
3885
3886     if ($winversion >= 6) {
3887         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3888         push @$machineFlags, 'hpet=off';
3889     }
3890
3891     push @$rtcFlags, 'driftfix=slew' if $tdf;
3892
3893     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3894         push @$rtcFlags, "base=$conf->{startdate}";
3895     } elsif ($useLocaltime) {
3896         push @$rtcFlags, 'base=localtime';
3897     }
3898
3899     if ($forcecpu) {
3900         push @$cmd, '-cpu', $forcecpu;
3901     } else {
3902         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3903     }
3904
3905     PVE::QemuServer::Memory::config(
3906         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3907
3908     push @$cmd, '-S' if $conf->{freeze};
3909
3910     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3911
3912     my $guest_agent = parse_guest_agent($conf);
3913
3914     if ($guest_agent->{enabled}) {
3915         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3916         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3917
3918         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3919             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3920             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3921             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3922         } elsif ($guest_agent->{type} eq 'isa') {
3923             push @$devices, '-device', "isa-serial,chardev=qga0";
3924         }
3925     }
3926
3927     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3928     if ($rng && $version_guard->(4, 1, 2)) {
3929         check_rng_source($rng->{source});
3930
3931         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3932         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3933         my $limiter_str = "";
3934         if ($max_bytes) {
3935             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3936         }
3937
3938         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3939         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3940         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3941     }
3942
3943     my $spice_port;
3944
3945     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3946         if ($qxlnum > 1) {
3947             if ($winversion){
3948                 for (my $i = 1; $i < $qxlnum; $i++){
3949                     push @$devices, '-device', print_vga_device(
3950                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3951                 }
3952             } else {
3953                 # assume other OS works like Linux
3954                 my ($ram, $vram) = ("134217728", "67108864");
3955                 if ($vga->{memory}) {
3956                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3957                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3958                 }
3959                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3960                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3961             }
3962         }
3963
3964         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3965
3966         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3967         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3968         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3969
3970         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3971         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3972         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3973
3974         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3975         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3976
3977         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3978         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3979         if ($spice_enhancement->{foldersharing}) {
3980             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3981             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3982         }
3983
3984         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3985         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3986             if $spice_enhancement->{videostreaming};
3987
3988         push @$devices, '-spice', "$spice_opts";
3989     }
3990
3991     # enable balloon by default, unless explicitly disabled
3992     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3993         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3994         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3995         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3996         push @$devices, '-device', $ballooncmd;
3997     }
3998
3999     if ($conf->{watchdog}) {
4000         my $wdopts = parse_watchdog($conf->{watchdog});
4001         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4002         my $watchdog = $wdopts->{model} || 'i6300esb';
4003         push @$devices, '-device', "$watchdog$pciaddr";
4004         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4005     }
4006
4007     my $vollist = [];
4008     my $scsicontroller = {};
4009     my $ahcicontroller = {};
4010     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4011
4012     # Add iscsi initiator name if available
4013     if (my $initiator = get_initiator_name()) {
4014         push @$devices, '-iscsi', "initiator-name=$initiator";
4015     }
4016
4017     PVE::QemuConfig->foreach_volume($conf, sub {
4018         my ($ds, $drive) = @_;
4019
4020         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4021             check_volume_storage_type($storecfg, $drive->{file});
4022             push @$vollist, $drive->{file};
4023         }
4024
4025         # ignore efidisk here, already added in bios/fw handling code above
4026         return if $drive->{interface} eq 'efidisk';
4027         # similar for TPM
4028         return if $drive->{interface} eq 'tpmstate';
4029
4030         $use_virtio = 1 if $ds =~ m/^virtio/;
4031
4032         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4033
4034         if ($drive->{interface} eq 'virtio'){
4035            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4036         }
4037
4038         if ($drive->{interface} eq 'scsi') {
4039
4040             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4041
4042             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4043                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4044
4045             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4046             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4047
4048             my $iothread = '';
4049             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4050                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4051                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4052             } elsif ($drive->{iothread}) {
4053                 log_warn(
4054                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4055                 );
4056             }
4057
4058             my $queues = '';
4059             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4060                 $queues = ",num_queues=$drive->{queues}";
4061             }
4062
4063             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4064                 if !$scsicontroller->{$controller};
4065             $scsicontroller->{$controller}=1;
4066         }
4067
4068         if ($drive->{interface} eq 'sata') {
4069             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4070             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4071             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4072                 if !$ahcicontroller->{$controller};
4073             $ahcicontroller->{$controller}=1;
4074         }
4075
4076         my $pbs_conf = $pbs_backing->{$ds};
4077         my $pbs_name = undef;
4078         if ($pbs_conf) {
4079             $pbs_name = "drive-$ds-pbs";
4080             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4081         }
4082
4083         my $drive_cmd = print_drive_commandline_full(
4084             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4085
4086         # extra protection for templates, but SATA and IDE don't support it..
4087         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4088
4089         push @$devices, '-drive',$drive_cmd;
4090         push @$devices, '-device', print_drivedevice_full(
4091             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4092     });
4093
4094     for (my $i = 0; $i < $MAX_NETS; $i++) {
4095         my $netname = "net$i";
4096
4097         next if !$conf->{$netname};
4098         my $d = parse_net($conf->{$netname});
4099         next if !$d;
4100         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4101
4102         $use_virtio = 1 if $d->{model} eq 'virtio';
4103
4104         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4105
4106         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4107         push @$devices, '-netdev', $netdevfull;
4108
4109         my $netdevicefull = print_netdevice_full(
4110             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4111
4112         push @$devices, '-device', $netdevicefull;
4113     }
4114
4115     if ($conf->{ivshmem}) {
4116         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4117
4118         my $bus;
4119         if ($q35) {
4120             $bus = print_pcie_addr("ivshmem");
4121         } else {
4122             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4123         }
4124
4125         my $ivshmem_name = $ivshmem->{name} // $vmid;
4126         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4127
4128         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4129         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4130             .",size=$ivshmem->{size}M";
4131     }
4132
4133     # pci.4 is nested in pci.1
4134     $bridges->{1} = 1 if $bridges->{4};
4135
4136     if (!$q35) { # add pci bridges
4137         if (min_version($machine_version, 2, 3)) {
4138            $bridges->{1} = 1;
4139            $bridges->{2} = 1;
4140         }
4141         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4142     }
4143
4144     for my $k (sort {$b cmp $a} keys %$bridges) {
4145         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4146
4147         my $k_name = $k;
4148         if ($k == 2 && $legacy_igd) {
4149             $k_name = "$k-igd";
4150         }
4151         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4152         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4153
4154         if ($q35) { # add after -readconfig pve-q35.cfg
4155             splice @$devices, 2, 0, '-device', $devstr;
4156         } else {
4157             unshift @$devices, '-device', $devstr if $k > 0;
4158         }
4159     }
4160
4161     if (!$kvm) {
4162         push @$machineFlags, 'accel=tcg';
4163     }
4164
4165     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4166
4167     my $machine_type_min = $machine_type;
4168     if ($add_pve_version) {
4169         $machine_type_min =~ s/\+pve\d+$//;
4170         $machine_type_min .= "+pve$required_pve_version";
4171     }
4172     push @$machineFlags, "type=${machine_type_min}";
4173
4174     push @$cmd, @$devices;
4175     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4176     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4177     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4178
4179     if (my $vmstate = $conf->{vmstate}) {
4180         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4181         push @$vollist, $vmstate;
4182         push @$cmd, '-loadstate', $statepath;
4183         print "activating and using '$vmstate' as vmstate\n";
4184     }
4185
4186     if (PVE::QemuConfig->is_template($conf)) {
4187         # needed to workaround base volumes being read-only
4188         push @$cmd, '-snapshot';
4189     }
4190
4191     # add custom args
4192     if ($conf->{args}) {
4193         my $aa = PVE::Tools::split_args($conf->{args});
4194         push @$cmd, @$aa;
4195     }
4196
4197     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4198 }
4199
4200 sub check_rng_source {
4201     my ($source) = @_;
4202
4203     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4204     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4205         if ! -e $source;
4206
4207     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4208     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4209         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4210         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4211         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4212             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4213             ." to the host.\n";
4214     }
4215 }
4216
4217 sub spice_port {
4218     my ($vmid) = @_;
4219
4220     my $res = mon_cmd($vmid, 'query-spice');
4221
4222     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4223 }
4224
4225 sub vm_devices_list {
4226     my ($vmid) = @_;
4227
4228     my $res = mon_cmd($vmid, 'query-pci');
4229     my $devices_to_check = [];
4230     my $devices = {};
4231     foreach my $pcibus (@$res) {
4232         push @$devices_to_check, @{$pcibus->{devices}},
4233     }
4234
4235     while (@$devices_to_check) {
4236         my $to_check = [];
4237         for my $d (@$devices_to_check) {
4238             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4239             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4240
4241             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4242             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4243         }
4244         $devices_to_check = $to_check;
4245     }
4246
4247     my $resblock = mon_cmd($vmid, 'query-block');
4248     foreach my $block (@$resblock) {
4249         if($block->{device} =~ m/^drive-(\S+)/){
4250                 $devices->{$1} = 1;
4251         }
4252     }
4253
4254     my $resmice = mon_cmd($vmid, 'query-mice');
4255     foreach my $mice (@$resmice) {
4256         if ($mice->{name} eq 'QEMU HID Tablet') {
4257             $devices->{tablet} = 1;
4258             last;
4259         }
4260     }
4261
4262     # for usb devices there is no query-usb
4263     # but we can iterate over the entries in
4264     # qom-list path=/machine/peripheral
4265     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4266     foreach my $per (@$resperipheral) {
4267         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4268             $devices->{$per->{name}} = 1;
4269         }
4270     }
4271
4272     return $devices;
4273 }
4274
4275 sub vm_deviceplug {
4276     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4277
4278     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4279
4280     my $devices_list = vm_devices_list($vmid);
4281     return 1 if defined($devices_list->{$deviceid});
4282
4283     # add PCI bridge if we need it for the device
4284     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4285
4286     if ($deviceid eq 'tablet') {
4287         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4288     } elsif ($deviceid eq 'keyboard') {
4289         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4290     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4291         my $id = $1;
4292         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4293         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4294     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4295         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4296     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4297         qemu_iothread_add($vmid, $deviceid, $device);
4298
4299         qemu_driveadd($storecfg, $vmid, $device);
4300         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4301
4302         qemu_deviceadd($vmid, $devicefull);
4303         eval { qemu_deviceaddverify($vmid, $deviceid); };
4304         if (my $err = $@) {
4305             eval { qemu_drivedel($vmid, $deviceid); };
4306             warn $@ if $@;
4307             die $err;
4308         }
4309     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4310         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4311         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4312         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4313
4314         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4315
4316         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4317             qemu_iothread_add($vmid, $deviceid, $device);
4318             $devicefull .= ",iothread=iothread-$deviceid";
4319         }
4320
4321         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4322             $devicefull .= ",num_queues=$device->{queues}";
4323         }
4324
4325         qemu_deviceadd($vmid, $devicefull);
4326         qemu_deviceaddverify($vmid, $deviceid);
4327     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4328         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4329         qemu_driveadd($storecfg, $vmid, $device);
4330
4331         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4332         eval { qemu_deviceadd($vmid, $devicefull); };
4333         if (my $err = $@) {
4334             eval { qemu_drivedel($vmid, $deviceid); };
4335             warn $@ if $@;
4336             die $err;
4337         }
4338     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4339         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4340
4341         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4342         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4343         my $use_old_bios_files = undef;
4344         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4345
4346         my $netdevicefull = print_netdevice_full(
4347             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4348         qemu_deviceadd($vmid, $netdevicefull);
4349         eval {
4350             qemu_deviceaddverify($vmid, $deviceid);
4351             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4352         };
4353         if (my $err = $@) {
4354             eval { qemu_netdevdel($vmid, $deviceid); };
4355             warn $@ if $@;
4356             die $err;
4357         }
4358     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4359         my $bridgeid = $2;
4360         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4361         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4362
4363         qemu_deviceadd($vmid, $devicefull);
4364         qemu_deviceaddverify($vmid, $deviceid);
4365     } else {
4366         die "can't hotplug device '$deviceid'\n";
4367     }
4368
4369     return 1;
4370 }
4371
4372 # fixme: this should raise exceptions on error!
4373 sub vm_deviceunplug {
4374     my ($vmid, $conf, $deviceid) = @_;
4375
4376     my $devices_list = vm_devices_list($vmid);
4377     return 1 if !defined($devices_list->{$deviceid});
4378
4379     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4380     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4381
4382     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4383         qemu_devicedel($vmid, $deviceid);
4384     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4385         qemu_devicedel($vmid, $deviceid);
4386         qemu_devicedelverify($vmid, $deviceid);
4387     } elsif ($deviceid =~ m/^usb\d+$/) {
4388         qemu_devicedel($vmid, $deviceid);
4389         qemu_devicedelverify($vmid, $deviceid);
4390     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4391         my $device = parse_drive($deviceid, $conf->{$deviceid});
4392
4393         qemu_devicedel($vmid, $deviceid);
4394         qemu_devicedelverify($vmid, $deviceid);
4395         qemu_drivedel($vmid, $deviceid);
4396         qemu_iothread_del($vmid, $deviceid, $device);
4397     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4398         qemu_devicedel($vmid, $deviceid);
4399         qemu_devicedelverify($vmid, $deviceid);
4400     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4401         my $device = parse_drive($deviceid, $conf->{$deviceid});
4402
4403         qemu_devicedel($vmid, $deviceid);
4404         qemu_devicedelverify($vmid, $deviceid);
4405         qemu_drivedel($vmid, $deviceid);
4406         qemu_deletescsihw($conf, $vmid, $deviceid);
4407
4408         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4409             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4410     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4411         qemu_devicedel($vmid, $deviceid);
4412         qemu_devicedelverify($vmid, $deviceid);
4413         qemu_netdevdel($vmid, $deviceid);
4414     } else {
4415         die "can't unplug device '$deviceid'\n";
4416     }
4417
4418     return 1;
4419 }
4420
4421 sub qemu_spice_usbredir_chardev_add {
4422     my ($vmid, $id) = @_;
4423
4424     mon_cmd($vmid, "chardev-add" , (
4425         id => $id,
4426         backend => {
4427             type => 'spicevmc',
4428             data => {
4429                 type => "usbredir",
4430             },
4431         },
4432     ));
4433 }
4434
4435 sub qemu_deviceadd {
4436     my ($vmid, $devicefull) = @_;
4437
4438     $devicefull = "driver=".$devicefull;
4439     my %options =  split(/[=,]/, $devicefull);
4440
4441     mon_cmd($vmid, "device_add" , %options);
4442 }
4443
4444 sub qemu_devicedel {
4445     my ($vmid, $deviceid) = @_;
4446
4447     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4448 }
4449
4450 sub qemu_iothread_add {
4451     my ($vmid, $deviceid, $device) = @_;
4452
4453     if ($device->{iothread}) {
4454         my $iothreads = vm_iothreads_list($vmid);
4455         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4456     }
4457 }
4458
4459 sub qemu_iothread_del {
4460     my ($vmid, $deviceid, $device) = @_;
4461
4462     if ($device->{iothread}) {
4463         my $iothreads = vm_iothreads_list($vmid);
4464         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4465     }
4466 }
4467
4468 sub qemu_objectadd {
4469     my ($vmid, $objectid, $qomtype) = @_;
4470
4471     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4472
4473     return 1;
4474 }
4475
4476 sub qemu_objectdel {
4477     my ($vmid, $objectid) = @_;
4478
4479     mon_cmd($vmid, "object-del", id => $objectid);
4480
4481     return 1;
4482 }
4483
4484 sub qemu_driveadd {
4485     my ($storecfg, $vmid, $device) = @_;
4486
4487     my $kvmver = get_running_qemu_version($vmid);
4488     my $io_uring = min_version($kvmver, 6, 0);
4489     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4490     $drive =~ s/\\/\\\\/g;
4491     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4492
4493     # If the command succeeds qemu prints: "OK"
4494     return 1 if $ret =~ m/OK/s;
4495
4496     die "adding drive failed: $ret\n";
4497 }
4498
4499 sub qemu_drivedel {
4500     my ($vmid, $deviceid) = @_;
4501
4502     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4503     $ret =~ s/^\s+//;
4504
4505     return 1 if $ret eq "";
4506
4507     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4508     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4509
4510     die "deleting drive $deviceid failed : $ret\n";
4511 }
4512
4513 sub qemu_deviceaddverify {
4514     my ($vmid, $deviceid) = @_;
4515
4516     for (my $i = 0; $i <= 5; $i++) {
4517          my $devices_list = vm_devices_list($vmid);
4518          return 1 if defined($devices_list->{$deviceid});
4519          sleep 1;
4520     }
4521
4522     die "error on hotplug device '$deviceid'\n";
4523 }
4524
4525
4526 sub qemu_devicedelverify {
4527     my ($vmid, $deviceid) = @_;
4528
4529     # need to verify that the device is correctly removed as device_del
4530     # is async and empty return is not reliable
4531
4532     for (my $i = 0; $i <= 5; $i++) {
4533          my $devices_list = vm_devices_list($vmid);
4534          return 1 if !defined($devices_list->{$deviceid});
4535          sleep 1;
4536     }
4537
4538     die "error on hot-unplugging device '$deviceid'\n";
4539 }
4540
4541 sub qemu_findorcreatescsihw {
4542     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4543
4544     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4545
4546     my $scsihwid="$controller_prefix$controller";
4547     my $devices_list = vm_devices_list($vmid);
4548
4549     if (!defined($devices_list->{$scsihwid})) {
4550         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4551     }
4552
4553     return 1;
4554 }
4555
4556 sub qemu_deletescsihw {
4557     my ($conf, $vmid, $opt) = @_;
4558
4559     my $device = parse_drive($opt, $conf->{$opt});
4560
4561     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4562         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4563         return 1;
4564     }
4565
4566     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4567
4568     my $devices_list = vm_devices_list($vmid);
4569     foreach my $opt (keys %{$devices_list}) {
4570         if (is_valid_drivename($opt)) {
4571             my $drive = parse_drive($opt, $conf->{$opt});
4572             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4573                 return 1;
4574             }
4575         }
4576     }
4577
4578     my $scsihwid="scsihw$controller";
4579
4580     vm_deviceunplug($vmid, $conf, $scsihwid);
4581
4582     return 1;
4583 }
4584
4585 sub qemu_add_pci_bridge {
4586     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4587
4588     my $bridges = {};
4589
4590     my $bridgeid;
4591
4592     print_pci_addr($device, $bridges, $arch, $machine_type);
4593
4594     while (my ($k, $v) = each %$bridges) {
4595         $bridgeid = $k;
4596     }
4597     return 1 if !defined($bridgeid) || $bridgeid < 1;
4598
4599     my $bridge = "pci.$bridgeid";
4600     my $devices_list = vm_devices_list($vmid);
4601
4602     if (!defined($devices_list->{$bridge})) {
4603         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4604     }
4605
4606     return 1;
4607 }
4608
4609 sub qemu_set_link_status {
4610     my ($vmid, $device, $up) = @_;
4611
4612     mon_cmd($vmid, "set_link", name => $device,
4613                up => $up ? JSON::true : JSON::false);
4614 }
4615
4616 sub qemu_netdevadd {
4617     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4618
4619     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4620     my %options =  split(/[=,]/, $netdev);
4621
4622     if (defined(my $vhost = $options{vhost})) {
4623         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4624     }
4625
4626     if (defined(my $queues = $options{queues})) {
4627         $options{queues} = $queues + 0;
4628     }
4629
4630     mon_cmd($vmid, "netdev_add",  %options);
4631     return 1;
4632 }
4633
4634 sub qemu_netdevdel {
4635     my ($vmid, $deviceid) = @_;
4636
4637     mon_cmd($vmid, "netdev_del", id => $deviceid);
4638 }
4639
4640 sub qemu_usb_hotplug {
4641     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4642
4643     return if !$device;
4644
4645     # remove the old one first
4646     vm_deviceunplug($vmid, $conf, $deviceid);
4647
4648     # check if xhci controller is necessary and available
4649     my $devicelist = vm_devices_list($vmid);
4650
4651     if (!$devicelist->{xhci}) {
4652         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4653         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4654     }
4655
4656     # print_usbdevice_full expects the parsed device
4657     my $d = parse_usb_device($device->{host});
4658     $d->{usb3} = $device->{usb3};
4659
4660     # add the new one
4661     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4662 }
4663
4664 sub qemu_cpu_hotplug {
4665     my ($vmid, $conf, $vcpus) = @_;
4666
4667     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4668
4669     my $sockets = 1;
4670     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671     $sockets = $conf->{sockets} if  $conf->{sockets};
4672     my $cores = $conf->{cores} || 1;
4673     my $maxcpus = $sockets * $cores;
4674
4675     $vcpus = $maxcpus if !$vcpus;
4676
4677     die "you can't add more vcpus than maxcpus\n"
4678         if $vcpus > $maxcpus;
4679
4680     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681
4682     if ($vcpus < $currentvcpus) {
4683
4684         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4685
4686             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687                 qemu_devicedel($vmid, "cpu$i");
4688                 my $retry = 0;
4689                 my $currentrunningvcpus = undef;
4690                 while (1) {
4691                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4692                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4693                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4694                     $retry++;
4695                     sleep 1;
4696                 }
4697                 #update conf after each succesfull cpu unplug
4698                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699                 PVE::QemuConfig->write_config($vmid, $conf);
4700             }
4701         } else {
4702             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4703         }
4704
4705         return;
4706     }
4707
4708     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4709     die "vcpus in running vm does not match its configuration\n"
4710         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711
4712     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4713
4714         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715             my $cpustr = print_cpu_device($conf, $i);
4716             qemu_deviceadd($vmid, $cpustr);
4717
4718             my $retry = 0;
4719             my $currentrunningvcpus = undef;
4720             while (1) {
4721                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4722                 last if scalar(@{$currentrunningvcpus}) == $i;
4723                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4724                 sleep 1;
4725                 $retry++;
4726             }
4727             #update conf after each succesfull cpu hotplug
4728             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729             PVE::QemuConfig->write_config($vmid, $conf);
4730         }
4731     } else {
4732
4733         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734             mon_cmd($vmid, "cpu-add", id => int($i));
4735         }
4736     }
4737 }
4738
4739 sub qemu_block_set_io_throttle {
4740     my ($vmid, $deviceid,
4741         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4742         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745
4746     return if !check_running($vmid) ;
4747
4748     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749         bps => int($bps),
4750         bps_rd => int($bps_rd),
4751         bps_wr => int($bps_wr),
4752         iops => int($iops),
4753         iops_rd => int($iops_rd),
4754         iops_wr => int($iops_wr),
4755         bps_max => int($bps_max),
4756         bps_rd_max => int($bps_rd_max),
4757         bps_wr_max => int($bps_wr_max),
4758         iops_max => int($iops_max),
4759         iops_rd_max => int($iops_rd_max),
4760         iops_wr_max => int($iops_wr_max),
4761         bps_max_length => int($bps_max_length),
4762         bps_rd_max_length => int($bps_rd_max_length),
4763         bps_wr_max_length => int($bps_wr_max_length),
4764         iops_max_length => int($iops_max_length),
4765         iops_rd_max_length => int($iops_rd_max_length),
4766         iops_wr_max_length => int($iops_wr_max_length),
4767     );
4768
4769 }
4770
4771 sub qemu_block_resize {
4772     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4773
4774     my $running = check_running($vmid);
4775
4776     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4777
4778     return if !$running;
4779
4780     my $padding = (1024 - $size % 1024) % 1024;
4781     $size = $size + $padding;
4782
4783     mon_cmd(
4784         $vmid,
4785         "block_resize",
4786         device => $deviceid,
4787         size => int($size),
4788         timeout => 60,
4789     );
4790 }
4791
4792 sub qemu_volume_snapshot {
4793     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4794
4795     my $running = check_running($vmid);
4796
4797     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4798         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4799     } else {
4800         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4801     }
4802 }
4803
4804 sub qemu_volume_snapshot_delete {
4805     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4806
4807     my $running = check_running($vmid);
4808
4809     if($running) {
4810
4811         $running = undef;
4812         my $conf = PVE::QemuConfig->load_config($vmid);
4813         PVE::QemuConfig->foreach_volume($conf, sub {
4814             my ($ds, $drive) = @_;
4815             $running = 1 if $drive->{file} eq $volid;
4816         });
4817     }
4818
4819     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4820         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4821     } else {
4822         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4823     }
4824 }
4825
4826 sub set_migration_caps {
4827     my ($vmid, $savevm) = @_;
4828
4829     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4830
4831     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4832     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4833
4834     my $cap_ref = [];
4835
4836     my $enabled_cap = {
4837         "auto-converge" => 1,
4838         "xbzrle" => 1,
4839         "x-rdma-pin-all" => 0,
4840         "zero-blocks" => 0,
4841         "compress" => 0,
4842         "dirty-bitmaps" => $dirty_bitmaps,
4843     };
4844
4845     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4846
4847     for my $supported_capability (@$supported_capabilities) {
4848         push @$cap_ref, {
4849             capability => $supported_capability->{capability},
4850             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4851         };
4852     }
4853
4854     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4855 }
4856
4857 sub foreach_volid {
4858     my ($conf, $func, @param) = @_;
4859
4860     my $volhash = {};
4861
4862     my $test_volid = sub {
4863         my ($key, $drive, $snapname) = @_;
4864
4865         my $volid = $drive->{file};
4866         return if !$volid;
4867
4868         $volhash->{$volid}->{cdrom} //= 1;
4869         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4870
4871         my $replicate = $drive->{replicate} // 1;
4872         $volhash->{$volid}->{replicate} //= 0;
4873         $volhash->{$volid}->{replicate} = 1 if $replicate;
4874
4875         $volhash->{$volid}->{shared} //= 0;
4876         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4877
4878         $volhash->{$volid}->{referenced_in_config} //= 0;
4879         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4880
4881         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4882             if defined($snapname);
4883
4884         my $size = $drive->{size};
4885         $volhash->{$volid}->{size} //= $size if $size;
4886
4887         $volhash->{$volid}->{is_vmstate} //= 0;
4888         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4889
4890         $volhash->{$volid}->{is_tpmstate} //= 0;
4891         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4892
4893         $volhash->{$volid}->{is_unused} //= 0;
4894         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4895
4896         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4897     };
4898
4899     my $include_opts = {
4900         extra_keys => ['vmstate'],
4901         include_unused => 1,
4902     };
4903
4904     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4905     foreach my $snapname (keys %{$conf->{snapshots}}) {
4906         my $snap = $conf->{snapshots}->{$snapname};
4907         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4908     }
4909
4910     foreach my $volid (keys %$volhash) {
4911         &$func($volid, $volhash->{$volid}, @param);
4912     }
4913 }
4914
4915 my $fast_plug_option = {
4916     'description' => 1,
4917     'hookscript' => 1,
4918     'lock' => 1,
4919     'migrate_downtime' => 1,
4920     'migrate_speed' => 1,
4921     'name' => 1,
4922     'onboot' => 1,
4923     'protection' => 1,
4924     'shares' => 1,
4925     'startup' => 1,
4926     'tags' => 1,
4927     'vmstatestorage' => 1,
4928 };
4929
4930 for my $opt (keys %$confdesc_cloudinit) {
4931     $fast_plug_option->{$opt} = 1;
4932 };
4933
4934 # hotplug changes in [PENDING]
4935 # $selection hash can be used to only apply specified options, for
4936 # example: { cores => 1 } (only apply changed 'cores')
4937 # $errors ref is used to return error messages
4938 sub vmconfig_hotplug_pending {
4939     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4940
4941     my $defaults = load_defaults();
4942     my $arch = get_vm_arch($conf);
4943     my $machine_type = get_vm_machine($conf, undef, $arch);
4944
4945     # commit values which do not have any impact on running VM first
4946     # Note: those option cannot raise errors, we we do not care about
4947     # $selection and always apply them.
4948
4949     my $add_error = sub {
4950         my ($opt, $msg) = @_;
4951         $errors->{$opt} = "hotplug problem - $msg";
4952     };
4953
4954     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4955
4956     my $cloudinit_record_changed = sub {
4957         my ($conf, $opt, $old, $new) = @_;
4958         return if !$cloudinit_pending_properties->{$opt};
4959
4960         my $ci = ($conf->{cloudinit} //= {});
4961
4962         my $recorded = $ci->{$opt};
4963         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4964
4965         if (defined($new)) {
4966             if (defined($old)) {
4967                 # an existing value is being modified
4968                 if (defined($recorded)) {
4969                     # the value was already not in sync
4970                     if ($new eq $recorded) {
4971                         # a value is being reverted to the cloud-init state:
4972                         delete $ci->{$opt};
4973                         delete $added{$opt};
4974                     } else {
4975                         # the value was changed multiple times, do nothing
4976                     }
4977                 } elsif ($added{$opt}) {
4978                     # the value had been marked as added and is being changed, do nothing
4979                 } else {
4980                     # the value is new, record it:
4981                     $ci->{$opt} = $old;
4982                 }
4983             } else {
4984                 # a new value is being added
4985                 if (defined($recorded)) {
4986                     # it was already not in sync
4987                     if ($new eq $recorded) {
4988                         # a value is being reverted to the cloud-init state:
4989                         delete $ci->{$opt};
4990                         delete $added{$opt};
4991                     } else {
4992                         # the value had temporarily been removed, do nothing
4993                     }
4994                 } elsif ($added{$opt}) {
4995                     # the value had been marked as added already, do nothing
4996                 } else {
4997                     # the value is new, add it
4998                     $added{$opt} = 1;
4999                 }
5000             }
5001         } elsif (!defined($old)) {
5002             # a non-existent value is being removed? ignore...
5003         } else {
5004             # a value is being deleted
5005             if (defined($recorded)) {
5006                 # a value was already recorded, just keep it
5007             } elsif ($added{$opt}) {
5008                 # the value was marked as added, remove it
5009                 delete $added{$opt};
5010             } else {
5011                 # a previously unrecorded value is being removed, record the old value:
5012                 $ci->{$opt} = $old;
5013             }
5014         }
5015
5016         my $added = join(',', sort keys %added);
5017         $ci->{added} = $added if length($added);
5018     };
5019
5020     my $changes = 0;
5021     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5022         if ($fast_plug_option->{$opt}) {
5023             my $new = delete $conf->{pending}->{$opt};
5024             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5025             $conf->{$opt} = $new;
5026             $changes = 1;
5027         }
5028     }
5029
5030     if ($changes) {
5031         PVE::QemuConfig->write_config($vmid, $conf);
5032     }
5033
5034     my $ostype = $conf->{ostype};
5035     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5036     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5037     my $usb_hotplug = $hotplug_features->{usb}
5038         && min_version($version, 7, 1)
5039         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5040
5041     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5042     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5043
5044     foreach my $opt (sort keys %$pending_delete_hash) {
5045         next if $selection && !$selection->{$opt};
5046         my $force = $pending_delete_hash->{$opt}->{force};
5047         eval {
5048             if ($opt eq 'hotplug') {
5049                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5050             } elsif ($opt eq 'tablet') {
5051                 die "skip\n" if !$hotplug_features->{usb};
5052                 if ($defaults->{tablet}) {
5053                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5054                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5055                         if $arch eq 'aarch64';
5056                 } else {
5057                     vm_deviceunplug($vmid, $conf, 'tablet');
5058                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5059                 }
5060             } elsif ($opt =~ m/^usb(\d+)$/) {
5061                 my $index = $1;
5062                 die "skip\n" if !$usb_hotplug;
5063                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5064                 vm_deviceunplug($vmid, $conf, $opt);
5065             } elsif ($opt eq 'vcpus') {
5066                 die "skip\n" if !$hotplug_features->{cpu};
5067                 qemu_cpu_hotplug($vmid, $conf, undef);
5068             } elsif ($opt eq 'balloon') {
5069                 # enable balloon device is not hotpluggable
5070                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5071                 # here we reset the ballooning value to memory
5072                 my $balloon = $conf->{memory} || $defaults->{memory};
5073                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5074             } elsif ($fast_plug_option->{$opt}) {
5075                 # do nothing
5076             } elsif ($opt =~ m/^net(\d+)$/) {
5077                 die "skip\n" if !$hotplug_features->{network};
5078                 vm_deviceunplug($vmid, $conf, $opt);
5079             } elsif (is_valid_drivename($opt)) {
5080                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5081                 vm_deviceunplug($vmid, $conf, $opt);
5082                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5083             } elsif ($opt =~ m/^memory$/) {
5084                 die "skip\n" if !$hotplug_features->{memory};
5085                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5086             } elsif ($opt eq 'cpuunits') {
5087                 $cgroup->change_cpu_shares(undef);
5088             } elsif ($opt eq 'cpulimit') {
5089                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5090             } else {
5091                 die "skip\n";
5092             }
5093         };
5094         if (my $err = $@) {
5095             &$add_error($opt, $err) if $err ne "skip\n";
5096         } else {
5097             my $old = delete $conf->{$opt};
5098             $cloudinit_record_changed->($conf, $opt, $old, undef);
5099             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5100         }
5101     }
5102
5103     my $cloudinit_opt;
5104     foreach my $opt (keys %{$conf->{pending}}) {
5105         next if $selection && !$selection->{$opt};
5106         my $value = $conf->{pending}->{$opt};
5107         eval {
5108             if ($opt eq 'hotplug') {
5109                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5110             } elsif ($opt eq 'tablet') {
5111                 die "skip\n" if !$hotplug_features->{usb};
5112                 if ($value == 1) {
5113                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5114                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5115                         if $arch eq 'aarch64';
5116                 } elsif ($value == 0) {
5117                     vm_deviceunplug($vmid, $conf, 'tablet');
5118                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5119                 }
5120             } elsif ($opt =~ m/^usb(\d+)$/) {
5121                 my $index = $1;
5122                 die "skip\n" if !$usb_hotplug;
5123                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5124                 my $id = $opt;
5125                 if ($d->{host} eq 'spice')  {
5126                     $id = "usbredirdev$index";
5127                 }
5128                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5129             } elsif ($opt eq 'vcpus') {
5130                 die "skip\n" if !$hotplug_features->{cpu};
5131                 qemu_cpu_hotplug($vmid, $conf, $value);
5132             } elsif ($opt eq 'balloon') {
5133                 # enable/disable balloning device is not hotpluggable
5134                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5135                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5136                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5137
5138                 # allow manual ballooning if shares is set to zero
5139                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5140                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5141                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5142                 }
5143             } elsif ($opt =~ m/^net(\d+)$/) {
5144                 # some changes can be done without hotplug
5145                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5146                                     $vmid, $opt, $value, $arch, $machine_type);
5147             } elsif (is_valid_drivename($opt)) {
5148                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5149                 # some changes can be done without hotplug
5150                 my $drive = parse_drive($opt, $value);
5151                 if (drive_is_cloudinit($drive)) {
5152                     $cloudinit_opt = [$opt, $drive];
5153                     # apply all the other changes first, then generate the cloudinit disk
5154                     die "skip\n";
5155                 }
5156                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5157                                      $vmid, $opt, $value, $arch, $machine_type);
5158             } elsif ($opt =~ m/^memory$/) { #dimms
5159                 die "skip\n" if !$hotplug_features->{memory};
5160                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5161             } elsif ($opt eq 'cpuunits') {
5162                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5163                 $cgroup->change_cpu_shares($new_cpuunits);
5164             } elsif ($opt eq 'cpulimit') {
5165                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5166                 $cgroup->change_cpu_quota($cpulimit, 100000);
5167             } elsif ($opt eq 'agent') {
5168                 vmconfig_update_agent($conf, $opt, $value);
5169             } else {
5170                 die "skip\n";  # skip non-hot-pluggable options
5171             }
5172         };
5173         if (my $err = $@) {
5174             &$add_error($opt, $err) if $err ne "skip\n";
5175         } else {
5176             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5177             $conf->{$opt} = $value;
5178             delete $conf->{pending}->{$opt};
5179         }
5180     }
5181
5182     if (defined($cloudinit_opt)) {
5183         my ($opt, $drive) = @$cloudinit_opt;
5184         my $value = $conf->{pending}->{$opt};
5185         eval {
5186             my $temp = {%$conf, $opt => $value};
5187             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5188             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5189                                  $vmid, $opt, $value, $arch, $machine_type);
5190         };
5191         if (my $err = $@) {
5192             &$add_error($opt, $err) if $err ne "skip\n";
5193         } else {
5194             $conf->{$opt} = $value;
5195             delete $conf->{pending}->{$opt};
5196         }
5197     }
5198
5199     # unplug xhci controller if no usb device is left
5200     if ($usb_hotplug) {
5201         my $has_usb = 0;
5202         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5203             next if !defined($conf->{"usb$i"});
5204             $has_usb = 1;
5205             last;
5206         }
5207         if (!$has_usb) {
5208             vm_deviceunplug($vmid, $conf, 'xhci');
5209         }
5210     }
5211
5212     PVE::QemuConfig->write_config($vmid, $conf);
5213
5214     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5215         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5216     }
5217 }
5218
5219 sub try_deallocate_drive {
5220     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5221
5222     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5223         my $volid = $drive->{file};
5224         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5225             my $sid = PVE::Storage::parse_volume_id($volid);
5226             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5227
5228             # check if the disk is really unused
5229             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5230                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5231             PVE::Storage::vdisk_free($storecfg, $volid);
5232             return 1;
5233         } else {
5234             # If vm is not owner of this disk remove from config
5235             return 1;
5236         }
5237     }
5238
5239     return;
5240 }
5241
5242 sub vmconfig_delete_or_detach_drive {
5243     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5244
5245     my $drive = parse_drive($opt, $conf->{$opt});
5246
5247     my $rpcenv = PVE::RPCEnvironment::get();
5248     my $authuser = $rpcenv->get_user();
5249
5250     if ($force) {
5251         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5252         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5253     } else {
5254         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5255     }
5256 }
5257
5258
5259
5260 sub vmconfig_apply_pending {
5261     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5262
5263     return if !scalar(keys %{$conf->{pending}});
5264
5265     my $add_apply_error = sub {
5266         my ($opt, $msg) = @_;
5267         my $err_msg = "unable to apply pending change $opt : $msg";
5268         $errors->{$opt} = $err_msg;
5269         warn $err_msg;
5270     };
5271
5272     # cold plug
5273
5274     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5275     foreach my $opt (sort keys %$pending_delete_hash) {
5276         my $force = $pending_delete_hash->{$opt}->{force};
5277         eval {
5278             if ($opt =~ m/^unused/) {
5279                 die "internal error";
5280             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5281                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5282             }
5283         };
5284         if (my $err = $@) {
5285             $add_apply_error->($opt, $err);
5286         } else {
5287             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5288             delete $conf->{$opt};
5289         }
5290     }
5291
5292     PVE::QemuConfig->cleanup_pending($conf);
5293
5294     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5295
5296     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5297         next if $opt eq 'delete'; # just to be sure
5298         eval {
5299             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5300                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5301             }
5302         };
5303         if (my $err = $@) {
5304             $add_apply_error->($opt, $err);
5305         } else {
5306
5307             if (is_valid_drivename($opt)) {
5308                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5309                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5310             }
5311
5312             $conf->{$opt} = delete $conf->{pending}->{$opt};
5313         }
5314     }
5315
5316     # write all changes at once to avoid unnecessary i/o
5317     PVE::QemuConfig->write_config($vmid, $conf);
5318     if ($generate_cloudinit) {
5319         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5320             # After successful generation and if there were changes to be applied, update the
5321             # config to drop the {cloudinit} entry.
5322             PVE::QemuConfig->write_config($vmid, $conf);
5323         }
5324     }
5325 }
5326
5327 sub vmconfig_update_net {
5328     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5329
5330     my $newnet = parse_net($value);
5331
5332     if ($conf->{$opt}) {
5333         my $oldnet = parse_net($conf->{$opt});
5334
5335         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5336             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5337             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5338             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5339             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5340
5341             # for non online change, we try to hot-unplug
5342             die "skip\n" if !$hotplug;
5343             vm_deviceunplug($vmid, $conf, $opt);
5344         } else {
5345
5346             die "internal error" if $opt !~ m/net(\d+)/;
5347             my $iface = "tap${vmid}i$1";
5348
5349             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5350                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5351                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5352                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5353                 PVE::Network::tap_unplug($iface);
5354
5355                 if ($have_sdn) {
5356                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5357                 } else {
5358                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5359                 }
5360             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5361                 # Rate can be applied on its own but any change above needs to
5362                 # include the rate in tap_plug since OVS resets everything.
5363                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5364             }
5365
5366             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5367                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5368             }
5369
5370             return 1;
5371         }
5372     }
5373
5374     if ($hotplug) {
5375         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5376     } else {
5377         die "skip\n";
5378     }
5379 }
5380
5381 sub vmconfig_update_agent {
5382     my ($conf, $opt, $value) = @_;
5383
5384     die "skip\n" if !$conf->{$opt};
5385
5386     my $hotplug_options = { fstrim_cloned_disks => 1 };
5387
5388     my $old_agent = parse_guest_agent($conf);
5389     my $agent = parse_guest_agent({$opt => $value});
5390
5391     for my $option (keys %$agent) { # added/changed options
5392         next if defined($hotplug_options->{$option});
5393         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5394     }
5395
5396     for my $option (keys %$old_agent) { # removed options
5397         next if defined($hotplug_options->{$option});
5398         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5399     }
5400
5401     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5402 }
5403
5404 sub vmconfig_update_disk {
5405     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5406
5407     my $drive = parse_drive($opt, $value);
5408
5409     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5410         my $media = $drive->{media} || 'disk';
5411         my $oldmedia = $old_drive->{media} || 'disk';
5412         die "unable to change media type\n" if $media ne $oldmedia;
5413
5414         if (!drive_is_cdrom($old_drive)) {
5415
5416             if ($drive->{file} ne $old_drive->{file}) {
5417
5418                 die "skip\n" if !$hotplug;
5419
5420                 # unplug and register as unused
5421                 vm_deviceunplug($vmid, $conf, $opt);
5422                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5423
5424             } else {
5425                 # update existing disk
5426
5427                 # skip non hotpluggable value
5428                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5429                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5430                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5431                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5432                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5433                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5434                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5435                     die "skip\n";
5436                 }
5437
5438                 # apply throttle
5439                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5440                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5441                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5442                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5443                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5444                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5445                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5446                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5447                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5448                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5449                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5450                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5451                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5452                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5453                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5454                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5455                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5456                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5457
5458                     qemu_block_set_io_throttle(
5459                         $vmid,"drive-$opt",
5460                         ($drive->{mbps} || 0)*1024*1024,
5461                         ($drive->{mbps_rd} || 0)*1024*1024,
5462                         ($drive->{mbps_wr} || 0)*1024*1024,
5463                          $drive->{iops} || 0,
5464                          $drive->{iops_rd} || 0,
5465                          $drive->{iops_wr} || 0,
5466                         ($drive->{mbps_max} || 0)*1024*1024,
5467                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5468                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5469                          $drive->{iops_max} || 0,
5470                          $drive->{iops_rd_max} || 0,
5471                          $drive->{iops_wr_max} || 0,
5472                          $drive->{bps_max_length} || 1,
5473                          $drive->{bps_rd_max_length} || 1,
5474                          $drive->{bps_wr_max_length} || 1,
5475                          $drive->{iops_max_length} || 1,
5476                          $drive->{iops_rd_max_length} || 1,
5477                          $drive->{iops_wr_max_length} || 1,
5478                     );
5479
5480                 }
5481
5482                 return 1;
5483             }
5484
5485         } else { # cdrom
5486
5487             if ($drive->{file} eq 'none') {
5488                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5489                 if (drive_is_cloudinit($old_drive)) {
5490                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5491                 }
5492             } else {
5493                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5494
5495                 # force eject if locked
5496                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5497
5498                 if ($path) {
5499                     mon_cmd($vmid, "blockdev-change-medium",
5500                         id => "$opt", filename => "$path");
5501                 }
5502             }
5503
5504             return 1;
5505         }
5506     }
5507
5508     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5509     # hotplug new disks
5510     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5511     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5512 }
5513
5514 sub vmconfig_update_cloudinit_drive {
5515     my ($storecfg, $conf, $vmid) = @_;
5516
5517     my $cloudinit_ds = undef;
5518     my $cloudinit_drive = undef;
5519
5520     PVE::QemuConfig->foreach_volume($conf, sub {
5521         my ($ds, $drive) = @_;
5522         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5523             $cloudinit_ds = $ds;
5524             $cloudinit_drive = $drive;
5525         }
5526     });
5527
5528     return if !$cloudinit_drive;
5529
5530     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5531         PVE::QemuConfig->write_config($vmid, $conf);
5532     }
5533
5534     my $running = PVE::QemuServer::check_running($vmid);
5535
5536     if ($running) {
5537         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5538         if ($path) {
5539             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5540             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5541         }
5542     }
5543 }
5544
5545 # called in locked context by incoming migration
5546 sub vm_migrate_get_nbd_disks {
5547     my ($storecfg, $conf, $replicated_volumes) = @_;
5548
5549     my $local_volumes = {};
5550     PVE::QemuConfig->foreach_volume($conf, sub {
5551         my ($ds, $drive) = @_;
5552
5553         return if drive_is_cdrom($drive);
5554         return if $ds eq 'tpmstate0';
5555
5556         my $volid = $drive->{file};
5557
5558         return if !$volid;
5559
5560         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5561
5562         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5563         return if $scfg->{shared};
5564
5565         # replicated disks re-use existing state via bitmap
5566         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5567         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5568     });
5569     return $local_volumes;
5570 }
5571
5572 # called in locked context by incoming migration
5573 sub vm_migrate_alloc_nbd_disks {
5574     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5575
5576     my $nbd = {};
5577     foreach my $opt (sort keys %$source_volumes) {
5578         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5579
5580         if ($use_existing) {
5581             $nbd->{$opt}->{drivestr} = print_drive($drive);
5582             $nbd->{$opt}->{volid} = $volid;
5583             $nbd->{$opt}->{replicated} = 1;
5584             next;
5585         }
5586
5587         # storage mapping + volname = regular migration
5588         # storage mapping + format = remote migration
5589         # order of precedence, filtered by whether storage supports it:
5590         # 1. explicit requested format
5591         # 2. format of current volume
5592         # 3. default format of storage
5593         if (!$storagemap->{identity}) {
5594             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5595             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5596             if (!$format || !grep { $format eq $_ } @$validFormats) {
5597                 if ($volname) {
5598                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5599                     my $fileFormat = qemu_img_format($scfg, $volname);
5600                     $format = $fileFormat
5601                         if grep { $fileFormat eq $_ } @$validFormats;
5602                 }
5603                 $format //= $defFormat;
5604             }
5605         } else {
5606             # can't happen for remote migration, so $volname is always defined
5607             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5608             $format = qemu_img_format($scfg, $volname);
5609         }
5610
5611         my $size = $drive->{size} / 1024;
5612         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5613         my $newdrive = $drive;
5614         $newdrive->{format} = $format;
5615         $newdrive->{file} = $newvolid;
5616         my $drivestr = print_drive($newdrive);
5617         $nbd->{$opt}->{drivestr} = $drivestr;
5618         $nbd->{$opt}->{volid} = $newvolid;
5619     }
5620
5621     return $nbd;
5622 }
5623
5624 # see vm_start_nolock for parameters, additionally:
5625 # migrate_opts:
5626 #   storagemap = parsed storage map for allocating NBD disks
5627 sub vm_start {
5628     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5629
5630     return PVE::QemuConfig->lock_config($vmid, sub {
5631         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5632
5633         die "you can't start a vm if it's a template\n"
5634             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5635
5636         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5637         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5638
5639         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5640
5641         if ($has_backup_lock && $running) {
5642             # a backup is currently running, attempt to start the guest in the
5643             # existing QEMU instance
5644             return vm_resume($vmid);
5645         }
5646
5647         PVE::QemuConfig->check_lock($conf)
5648             if !($params->{skiplock} || $has_suspended_lock);
5649
5650         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5651
5652         die "VM $vmid already running\n" if $running;
5653
5654         if (my $storagemap = $migrate_opts->{storagemap}) {
5655             my $replicated = $migrate_opts->{replicated_volumes};
5656             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5657             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5658
5659             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5660                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5661             }
5662         }
5663
5664         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5665     });
5666 }
5667
5668
5669 # params:
5670 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5671 #   skiplock => 0/1, skip checking for config lock
5672 #   skiptemplate => 0/1, skip checking whether VM is template
5673 #   forcemachine => to force QEMU machine (rollback/migration)
5674 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5675 #   timeout => in seconds
5676 #   paused => start VM in paused state (backup)
5677 #   resume => resume from hibernation
5678 #   pbs-backing => {
5679 #      sata0 => {
5680 #         repository
5681 #         snapshot
5682 #         keyfile
5683 #         archive
5684 #      },
5685 #      virtio2 => ...
5686 #   }
5687 # migrate_opts:
5688 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5689 #   migratedfrom => source node
5690 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5691 #   network => CIDR of migration network
5692 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5693 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5694 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5695 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5696 #       contained in config
5697 sub vm_start_nolock {
5698     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5699
5700     my $statefile = $params->{statefile};
5701     my $resume = $params->{resume};
5702
5703     my $migratedfrom = $migrate_opts->{migratedfrom};
5704     my $migration_type = $migrate_opts->{type};
5705
5706     my $res = {};
5707
5708     # clean up leftover reboot request files
5709     eval { clear_reboot_request($vmid); };
5710     warn $@ if $@;
5711
5712     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5713         vmconfig_apply_pending($vmid, $conf, $storecfg);
5714         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5715     }
5716
5717     # don't regenerate the ISO if the VM is started as part of a live migration
5718     # this way we can reuse the old ISO with the correct config
5719     if (!$migratedfrom) {
5720         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5721             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5722             # $conf->{cloudinit}, so we could just not do this?
5723             # But we do it above, so for now let's be consistent.
5724             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5725         }
5726     }
5727
5728     # override offline migrated volumes, conf is out of date still
5729     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5730         for my $key (sort keys $offline_volumes->%*) {
5731             my $parsed = parse_drive($key, $conf->{$key});
5732             $parsed->{file} = $offline_volumes->{$key};
5733             $conf->{$key} = print_drive($parsed);
5734         }
5735     }
5736
5737     my $defaults = load_defaults();
5738
5739     # set environment variable useful inside network script
5740     # for remote migration the config is available on the target node!
5741     if (!$migrate_opts->{remote_node}) {
5742         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5743     }
5744
5745     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5746
5747     my $forcemachine = $params->{forcemachine};
5748     my $forcecpu = $params->{forcecpu};
5749     if ($resume) {
5750         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5751         $forcemachine = $conf->{runningmachine};
5752         $forcecpu = $conf->{runningcpu};
5753         print "Resuming suspended VM\n";
5754     }
5755
5756     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5757         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5758
5759     my $migration_ip;
5760     my $get_migration_ip = sub {
5761         my ($nodename) = @_;
5762
5763         return $migration_ip if defined($migration_ip);
5764
5765         my $cidr = $migrate_opts->{network};
5766
5767         if (!defined($cidr)) {
5768             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5769             $cidr = $dc_conf->{migration}->{network};
5770         }
5771
5772         if (defined($cidr)) {
5773             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5774
5775             die "could not get IP: no address configured on local " .
5776                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5777
5778             die "could not get IP: multiple addresses configured on local " .
5779                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5780
5781             $migration_ip = @$ips[0];
5782         }
5783
5784         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5785             if !defined($migration_ip);
5786
5787         return $migration_ip;
5788     };
5789
5790     if ($statefile) {
5791         if ($statefile eq 'tcp') {
5792             my $migrate = $res->{migrate} = { proto => 'tcp' };
5793             $migrate->{addr} = "localhost";
5794             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5795             my $nodename = nodename();
5796
5797             if (!defined($migration_type)) {
5798                 if (defined($datacenterconf->{migration}->{type})) {
5799                     $migration_type = $datacenterconf->{migration}->{type};
5800                 } else {
5801                     $migration_type = 'secure';
5802                 }
5803             }
5804
5805             if ($migration_type eq 'insecure') {
5806                 $migrate->{addr} = $get_migration_ip->($nodename);
5807                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5808             }
5809
5810             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5811             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5812             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5813             push @$cmd, '-incoming', $migrate->{uri};
5814             push @$cmd, '-S';
5815
5816         } elsif ($statefile eq 'unix') {
5817             # should be default for secure migrations as a ssh TCP forward
5818             # tunnel is not deterministic reliable ready and fails regurarly
5819             # to set up in time, so use UNIX socket forwards
5820             my $migrate = $res->{migrate} = { proto => 'unix' };
5821             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5822             unlink $migrate->{addr};
5823
5824             $migrate->{uri} = "unix:$migrate->{addr}";
5825             push @$cmd, '-incoming', $migrate->{uri};
5826             push @$cmd, '-S';
5827
5828         } elsif (-e $statefile) {
5829             push @$cmd, '-loadstate', $statefile;
5830         } else {
5831             my $statepath = PVE::Storage::path($storecfg, $statefile);
5832             push @$vollist, $statefile;
5833             push @$cmd, '-loadstate', $statepath;
5834         }
5835     } elsif ($params->{paused}) {
5836         push @$cmd, '-S';
5837     }
5838
5839     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5840
5841     my $pci_devices = {}; # host pci devices
5842     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5843         my $dev = $conf->{"hostpci$i"} or next;
5844         $pci_devices->{$i} = parse_hostpci($dev);
5845     }
5846
5847     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5848     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5849
5850     # map to a flat list of pci ids
5851     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5852
5853     # reserve all PCI IDs before actually doing anything with them
5854     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5855
5856     eval {
5857         my $uuid;
5858         for my $id (sort keys %$pci_devices) {
5859             my $d = $pci_devices->{$id};
5860             for my $dev ($d->{pciid}->@*) {
5861                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5862
5863                 # nvidia grid needs the qemu parameter '-uuid' set
5864                 # use smbios uuid or mdev uuid as fallback for that
5865                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5866                     if (defined($conf->{smbios1})) {
5867                         my $smbios_conf = parse_smbios1($conf->{smbios1});
5868                         $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5869                     }
5870                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id) if !defined($uuid);
5871                 }
5872             }
5873         }
5874         push @$cmd, '-uuid', $uuid if defined($uuid);
5875     };
5876     if (my $err = $@) {
5877         eval { cleanup_pci_devices($vmid, $conf) };
5878         warn $@ if $@;
5879         die $err;
5880     }
5881
5882     PVE::Storage::activate_volumes($storecfg, $vollist);
5883
5884     eval {
5885         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5886     };
5887     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5888     # timeout should be more than enough here...
5889     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5890
5891     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5892
5893     my %run_params = (
5894         timeout => $statefile ? undef : $start_timeout,
5895         umask => 0077,
5896         noerr => 1,
5897     );
5898
5899     # when migrating, prefix QEMU output so other side can pick up any
5900     # errors that might occur and show the user
5901     if ($migratedfrom) {
5902         $run_params{quiet} = 1;
5903         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5904     }
5905
5906     my %systemd_properties = (
5907         Slice => 'qemu.slice',
5908         KillMode => 'process',
5909         SendSIGKILL => 0,
5910         TimeoutStopUSec => ULONG_MAX, # infinity
5911     );
5912
5913     if (PVE::CGroup::cgroup_mode() == 2) {
5914         $systemd_properties{CPUWeight} = $cpuunits;
5915     } else {
5916         $systemd_properties{CPUShares} = $cpuunits;
5917     }
5918
5919     if (my $cpulimit = $conf->{cpulimit}) {
5920         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5921     }
5922     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5923
5924     my $run_qemu = sub {
5925         PVE::Tools::run_fork sub {
5926             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5927
5928             my $tpmpid;
5929             if (my $tpm = $conf->{tpmstate0}) {
5930                 # start the TPM emulator so QEMU can connect on start
5931                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5932             }
5933
5934             my $exitcode = run_command($cmd, %run_params);
5935             if ($exitcode) {
5936                 if ($tpmpid) {
5937                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5938                     kill 'TERM', $tpmpid;
5939                 }
5940                 die "QEMU exited with code $exitcode\n";
5941             }
5942         };
5943     };
5944
5945     if ($conf->{hugepages}) {
5946
5947         my $code = sub {
5948             my $hotplug_features =
5949                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5950             my $hugepages_topology =
5951                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5952
5953             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5954
5955             PVE::QemuServer::Memory::hugepages_mount();
5956             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5957
5958             eval { $run_qemu->() };
5959             if (my $err = $@) {
5960                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5961                     if !$conf->{keephugepages};
5962                 die $err;
5963             }
5964
5965             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5966                 if !$conf->{keephugepages};
5967         };
5968         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5969
5970     } else {
5971         eval { $run_qemu->() };
5972     }
5973
5974     if (my $err = $@) {
5975         # deactivate volumes if start fails
5976         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5977         warn $@ if $@;
5978         eval { cleanup_pci_devices($vmid, $conf) };
5979         warn $@ if $@;
5980
5981         die "start failed: $err";
5982     }
5983
5984     # re-reserve all PCI IDs now that we can know the actual VM PID
5985     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5986     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5987     warn $@ if $@;
5988
5989     if (defined($res->{migrate})) {
5990         print "migration listens on $res->{migrate}->{uri}\n";
5991     } elsif ($statefile) {
5992         eval { mon_cmd($vmid, "cont"); };
5993         warn $@ if $@;
5994     }
5995
5996     #start nbd server for storage migration
5997     if (my $nbd = $migrate_opts->{nbd}) {
5998         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5999
6000         my $migrate_storage_uri;
6001         # nbd_protocol_version > 0 for unix socket support
6002         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6003             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6004             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
6005             $migrate_storage_uri = "nbd:unix:$socket_path";
6006             $res->{migrate}->{unix_sockets} = [$socket_path];
6007         } else {
6008             my $nodename = nodename();
6009             my $localip = $get_migration_ip->($nodename);
6010             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6011             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6012
6013             mon_cmd($vmid, "nbd-server-start", addr => {
6014                 type => 'inet',
6015                 data => {
6016                     host => "${localip}",
6017                     port => "${storage_migrate_port}",
6018                 },
6019             });
6020             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6021             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6022         }
6023
6024         my $block_info = mon_cmd($vmid, "query-block");
6025         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6026
6027         foreach my $opt (sort keys %$nbd) {
6028             my $drivestr = $nbd->{$opt}->{drivestr};
6029             my $volid = $nbd->{$opt}->{volid};
6030
6031             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6032
6033             mon_cmd(
6034                 $vmid,
6035                 "block-export-add",
6036                 id => "drive-$opt",
6037                 'node-name' => $block_node,
6038                 writable => JSON::true,
6039                 type => "nbd",
6040                 name => "drive-$opt", # NBD export name
6041             );
6042
6043             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6044             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6045             print "re-using replicated volume: $opt - $volid\n"
6046                 if $nbd->{$opt}->{replicated};
6047
6048             $res->{drives}->{$opt} = $nbd->{$opt};
6049             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6050         }
6051     }
6052
6053     if ($migratedfrom) {
6054         eval {
6055             set_migration_caps($vmid);
6056         };
6057         warn $@ if $@;
6058
6059         if ($spice_port) {
6060             print "spice listens on port $spice_port\n";
6061             $res->{spice_port} = $spice_port;
6062             if ($migrate_opts->{spice_ticket}) {
6063                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6064                     $migrate_opts->{spice_ticket});
6065                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6066             }
6067         }
6068
6069     } else {
6070         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6071             if !$statefile && $conf->{balloon};
6072
6073         foreach my $opt (keys %$conf) {
6074             next if $opt !~  m/^net\d+$/;
6075             my $nicconf = parse_net($conf->{$opt});
6076             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6077         }
6078         add_nets_bridge_fdb($conf, $vmid);
6079     }
6080
6081    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6082         eval {
6083             mon_cmd(
6084                 $vmid,
6085                 'qom-set',
6086                 path => "machine/peripheral/balloon0",
6087                 property => "guest-stats-polling-interval",
6088                 value => 2
6089             );
6090         };
6091         log_warn("could not set polling interval for ballooning - $@") if $@;
6092     }
6093
6094     if ($resume) {
6095         print "Resumed VM, removing state\n";
6096         if (my $vmstate = $conf->{vmstate}) {
6097             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6098             PVE::Storage::vdisk_free($storecfg, $vmstate);
6099         }
6100         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6101         PVE::QemuConfig->write_config($vmid, $conf);
6102     }
6103
6104     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6105
6106     return $res;
6107 }
6108
6109 sub vm_commandline {
6110     my ($storecfg, $vmid, $snapname) = @_;
6111
6112     my $conf = PVE::QemuConfig->load_config($vmid);
6113
6114     my ($forcemachine, $forcecpu);
6115     if ($snapname) {
6116         my $snapshot = $conf->{snapshots}->{$snapname};
6117         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6118
6119         # check for machine or CPU overrides in snapshot
6120         $forcemachine = $snapshot->{runningmachine};
6121         $forcecpu = $snapshot->{runningcpu};
6122
6123         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6124
6125         $conf = $snapshot;
6126     }
6127
6128     my $defaults = load_defaults();
6129
6130     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6131
6132     return PVE::Tools::cmd2string($cmd);
6133 }
6134
6135 sub vm_reset {
6136     my ($vmid, $skiplock) = @_;
6137
6138     PVE::QemuConfig->lock_config($vmid, sub {
6139
6140         my $conf = PVE::QemuConfig->load_config($vmid);
6141
6142         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6143
6144         mon_cmd($vmid, "system_reset");
6145     });
6146 }
6147
6148 sub get_vm_volumes {
6149     my ($conf) = @_;
6150
6151     my $vollist = [];
6152     foreach_volid($conf, sub {
6153         my ($volid, $attr) = @_;
6154
6155         return if $volid =~ m|^/|;
6156
6157         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6158         return if !$sid;
6159
6160         push @$vollist, $volid;
6161     });
6162
6163     return $vollist;
6164 }
6165
6166 sub cleanup_pci_devices {
6167     my ($vmid, $conf) = @_;
6168
6169     foreach my $key (keys %$conf) {
6170         next if $key !~ m/^hostpci(\d+)$/;
6171         my $hostpciindex = $1;
6172         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6173         my $d = parse_hostpci($conf->{$key});
6174         if ($d->{mdev}) {
6175             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6176             # don't want to break ABI just for this two liner
6177             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6178
6179             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6180             # out when we do it first. so wait for 10 seconds and then try it
6181             my $pciid = $d->{pciid}->[0]->{id};
6182             my $info = PVE::SysFSTools::pci_device_info("$pciid");
6183             if ($info->{vendor} eq '10de') {
6184                 sleep 10;
6185             }
6186
6187             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6188         }
6189     }
6190     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6191 }
6192
6193 sub vm_stop_cleanup {
6194     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6195
6196     eval {
6197
6198         if (!$keepActive) {
6199             my $vollist = get_vm_volumes($conf);
6200             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6201
6202             if (my $tpmdrive = $conf->{tpmstate0}) {
6203                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6204                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6205                 if ($storeid) {
6206                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6207                 }
6208             }
6209         }
6210
6211         foreach my $ext (qw(mon qmp pid vnc qga)) {
6212             unlink "/var/run/qemu-server/${vmid}.$ext";
6213         }
6214
6215         if ($conf->{ivshmem}) {
6216             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6217             # just delete it for now, VMs which have this already open do not
6218             # are affected, but new VMs will get a separated one. If this
6219             # becomes an issue we either add some sort of ref-counting or just
6220             # add a "don't delete on stop" flag to the ivshmem format.
6221             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6222         }
6223
6224         cleanup_pci_devices($vmid, $conf);
6225
6226         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6227     };
6228     warn $@ if $@; # avoid errors - just warn
6229 }
6230
6231 # call only in locked context
6232 sub _do_vm_stop {
6233     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6234
6235     my $pid = check_running($vmid, $nocheck);
6236     return if !$pid;
6237
6238     my $conf;
6239     if (!$nocheck) {
6240         $conf = PVE::QemuConfig->load_config($vmid);
6241         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6242         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6243             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6244             $timeout = $opts->{down} if $opts->{down};
6245         }
6246         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6247     }
6248
6249     eval {
6250         if ($shutdown) {
6251             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6252                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6253             } else {
6254                 mon_cmd($vmid, "system_powerdown");
6255             }
6256         } else {
6257             mon_cmd($vmid, "quit");
6258         }
6259     };
6260     my $err = $@;
6261
6262     if (!$err) {
6263         $timeout = 60 if !defined($timeout);
6264
6265         my $count = 0;
6266         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6267             $count++;
6268             sleep 1;
6269         }
6270
6271         if ($count >= $timeout) {
6272             if ($force) {
6273                 warn "VM still running - terminating now with SIGTERM\n";
6274                 kill 15, $pid;
6275             } else {
6276                 die "VM quit/powerdown failed - got timeout\n";
6277             }
6278         } else {
6279             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6280             return;
6281         }
6282     } else {
6283        if (!check_running($vmid, $nocheck)) {
6284             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6285             return;
6286        }
6287        if ($force) {
6288             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6289             kill 15, $pid;
6290         } else {
6291             die "VM quit/powerdown failed\n";
6292         }
6293     }
6294
6295     # wait again
6296     $timeout = 10;
6297
6298     my $count = 0;
6299     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6300         $count++;
6301         sleep 1;
6302     }
6303
6304     if ($count >= $timeout) {
6305         warn "VM still running - terminating now with SIGKILL\n";
6306         kill 9, $pid;
6307         sleep 1;
6308     }
6309
6310     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6311 }
6312
6313 # Note: use $nocheck to skip tests if VM configuration file exists.
6314 # We need that when migration VMs to other nodes (files already moved)
6315 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6316 sub vm_stop {
6317     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6318
6319     $force = 1 if !defined($force) && !$shutdown;
6320
6321     if ($migratedfrom){
6322         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6323         kill 15, $pid if $pid;
6324         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6325         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6326         return;
6327     }
6328
6329     PVE::QemuConfig->lock_config($vmid, sub {
6330         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6331    });
6332 }
6333
6334 sub vm_reboot {
6335     my ($vmid, $timeout) = @_;
6336
6337     PVE::QemuConfig->lock_config($vmid, sub {
6338         eval {
6339
6340             # only reboot if running, as qmeventd starts it again on a stop event
6341             return if !check_running($vmid);
6342
6343             create_reboot_request($vmid);
6344
6345             my $storecfg = PVE::Storage::config();
6346             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6347
6348         };
6349         if (my $err = $@) {
6350             # avoid that the next normal shutdown will be confused for a reboot
6351             clear_reboot_request($vmid);
6352             die $err;
6353         }
6354    });
6355 }
6356
6357 # note: if using the statestorage parameter, the caller has to check privileges
6358 sub vm_suspend {
6359     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6360
6361     my $conf;
6362     my $path;
6363     my $storecfg;
6364     my $vmstate;
6365
6366     PVE::QemuConfig->lock_config($vmid, sub {
6367
6368         $conf = PVE::QemuConfig->load_config($vmid);
6369
6370         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6371         PVE::QemuConfig->check_lock($conf)
6372             if !($skiplock || $is_backing_up);
6373
6374         die "cannot suspend to disk during backup\n"
6375             if $is_backing_up && $includestate;
6376
6377         if ($includestate) {
6378             $conf->{lock} = 'suspending';
6379             my $date = strftime("%Y-%m-%d", localtime(time()));
6380             $storecfg = PVE::Storage::config();
6381             if (!$statestorage) {
6382                 $statestorage = find_vmstate_storage($conf, $storecfg);
6383                 # check permissions for the storage
6384                 my $rpcenv = PVE::RPCEnvironment::get();
6385                 if ($rpcenv->{type} ne 'cli') {
6386                     my $authuser = $rpcenv->get_user();
6387                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6388                 }
6389             }
6390
6391
6392             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6393                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6394             $path = PVE::Storage::path($storecfg, $vmstate);
6395             PVE::QemuConfig->write_config($vmid, $conf);
6396         } else {
6397             mon_cmd($vmid, "stop");
6398         }
6399     });
6400
6401     if ($includestate) {
6402         # save vm state
6403         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6404
6405         eval {
6406             set_migration_caps($vmid, 1);
6407             mon_cmd($vmid, "savevm-start", statefile => $path);
6408             for(;;) {
6409                 my $state = mon_cmd($vmid, "query-savevm");
6410                 if (!$state->{status}) {
6411                     die "savevm not active\n";
6412                 } elsif ($state->{status} eq 'active') {
6413                     sleep(1);
6414                     next;
6415                 } elsif ($state->{status} eq 'completed') {
6416                     print "State saved, quitting\n";
6417                     last;
6418                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6419                     die "query-savevm failed with error '$state->{error}'\n"
6420                 } else {
6421                     die "query-savevm returned status '$state->{status}'\n";
6422                 }
6423             }
6424         };
6425         my $err = $@;
6426
6427         PVE::QemuConfig->lock_config($vmid, sub {
6428             $conf = PVE::QemuConfig->load_config($vmid);
6429             if ($err) {
6430                 # cleanup, but leave suspending lock, to indicate something went wrong
6431                 eval {
6432                     mon_cmd($vmid, "savevm-end");
6433                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6434                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6435                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6436                     PVE::QemuConfig->write_config($vmid, $conf);
6437                 };
6438                 warn $@ if $@;
6439                 die $err;
6440             }
6441
6442             die "lock changed unexpectedly\n"
6443                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6444
6445             mon_cmd($vmid, "quit");
6446             $conf->{lock} = 'suspended';
6447             PVE::QemuConfig->write_config($vmid, $conf);
6448         });
6449     }
6450 }
6451
6452 # $nocheck is set when called as part of a migration - in this context the
6453 # location of the config file (source or target node) is not deterministic,
6454 # since migration cannot wait for pmxcfs to process the rename
6455 sub vm_resume {
6456     my ($vmid, $skiplock, $nocheck) = @_;
6457
6458     PVE::QemuConfig->lock_config($vmid, sub {
6459         my $res = mon_cmd($vmid, 'query-status');
6460         my $resume_cmd = 'cont';
6461         my $reset = 0;
6462         my $conf;
6463         if ($nocheck) {
6464             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6465             if ($@) {
6466                 my $vmlist = PVE::Cluster::get_vmlist();
6467                 if (exists($vmlist->{ids}->{$vmid})) {
6468                     my $node = $vmlist->{ids}->{$vmid}->{node};
6469                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6470                 }
6471                 if (!$conf) {
6472                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6473                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6474                 }
6475             }
6476         } else {
6477             $conf = PVE::QemuConfig->load_config($vmid);
6478         }
6479
6480         if ($res->{status}) {
6481             return if $res->{status} eq 'running'; # job done, go home
6482             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6483             $reset = 1 if $res->{status} eq 'shutdown';
6484         }
6485
6486         if (!$nocheck) {
6487             PVE::QemuConfig->check_lock($conf)
6488                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6489         }
6490
6491         if ($reset) {
6492             # required if a VM shuts down during a backup and we get a resume
6493             # request before the backup finishes for example
6494             mon_cmd($vmid, "system_reset");
6495         }
6496
6497         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6498
6499         mon_cmd($vmid, $resume_cmd);
6500     });
6501 }
6502
6503 sub vm_sendkey {
6504     my ($vmid, $skiplock, $key) = @_;
6505
6506     PVE::QemuConfig->lock_config($vmid, sub {
6507
6508         my $conf = PVE::QemuConfig->load_config($vmid);
6509
6510         # there is no qmp command, so we use the human monitor command
6511         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6512         die $res if $res ne '';
6513     });
6514 }
6515
6516 sub check_bridge_access {
6517     my ($rpcenv, $authuser, $conf) = @_;
6518
6519     return 1 if $authuser eq 'root@pam';
6520
6521     for my $opt (sort keys $conf->%*) {
6522         next if $opt !~ m/^net\d+$/;
6523         my $net = parse_net($conf->{$opt});
6524         my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6525         PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6526     }
6527     return 1;
6528 };
6529
6530 # vzdump restore implementaion
6531
6532 sub tar_archive_read_firstfile {
6533     my $archive = shift;
6534
6535     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6536
6537     # try to detect archive type first
6538     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6539         die "unable to open file '$archive'\n";
6540     my $firstfile = <$fh>;
6541     kill 15, $pid;
6542     close $fh;
6543
6544     die "ERROR: archive contaions no data\n" if !$firstfile;
6545     chomp $firstfile;
6546
6547     return $firstfile;
6548 }
6549
6550 sub tar_restore_cleanup {
6551     my ($storecfg, $statfile) = @_;
6552
6553     print STDERR "starting cleanup\n";
6554
6555     if (my $fd = IO::File->new($statfile, "r")) {
6556         while (defined(my $line = <$fd>)) {
6557             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6558                 my $volid = $2;
6559                 eval {
6560                     if ($volid =~ m|^/|) {
6561                         unlink $volid || die 'unlink failed\n';
6562                     } else {
6563                         PVE::Storage::vdisk_free($storecfg, $volid);
6564                     }
6565                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6566                 };
6567                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6568             } else {
6569                 print STDERR "unable to parse line in statfile - $line";
6570             }
6571         }
6572         $fd->close();
6573     }
6574 }
6575
6576 sub restore_file_archive {
6577     my ($archive, $vmid, $user, $opts) = @_;
6578
6579     return restore_vma_archive($archive, $vmid, $user, $opts)
6580         if $archive eq '-';
6581
6582     my $info = PVE::Storage::archive_info($archive);
6583     my $format = $opts->{format} // $info->{format};
6584     my $comp = $info->{compression};
6585
6586     # try to detect archive format
6587     if ($format eq 'tar') {
6588         return restore_tar_archive($archive, $vmid, $user, $opts);
6589     } else {
6590         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6591     }
6592 }
6593
6594 # hepler to remove disks that will not be used after restore
6595 my $restore_cleanup_oldconf = sub {
6596     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6597
6598     my $kept_disks = {};
6599
6600     PVE::QemuConfig->foreach_volume($oldconf, sub {
6601         my ($ds, $drive) = @_;
6602
6603         return if drive_is_cdrom($drive, 1);
6604
6605         my $volid = $drive->{file};
6606         return if !$volid || $volid =~ m|^/|;
6607
6608         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6609         return if !$path || !$owner || ($owner != $vmid);
6610
6611         # Note: only delete disk we want to restore
6612         # other volumes will become unused
6613         if ($virtdev_hash->{$ds}) {
6614             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6615             if (my $err = $@) {
6616                 warn $err;
6617             }
6618         } else {
6619             $kept_disks->{$volid} = 1;
6620         }
6621     });
6622
6623     # after the restore we have no snapshots anymore
6624     for my $snapname (keys $oldconf->{snapshots}->%*) {
6625         my $snap = $oldconf->{snapshots}->{$snapname};
6626         if ($snap->{vmstate}) {
6627             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6628             if (my $err = $@) {
6629                 warn $err;
6630             }
6631         }
6632
6633         for my $volid (keys $kept_disks->%*) {
6634             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6635             warn $@ if $@;
6636         }
6637     }
6638 };
6639
6640 # Helper to parse vzdump backup device hints
6641 #
6642 # $rpcenv: Environment, used to ckeck storage permissions
6643 # $user: User ID, to check storage permissions
6644 # $storecfg: Storage configuration
6645 # $fh: the file handle for reading the configuration
6646 # $devinfo: should contain device sizes for all backu-up'ed devices
6647 # $options: backup options (pool, default storage)
6648 #
6649 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6650 my $parse_backup_hints = sub {
6651     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6652
6653     my $check_storage = sub { # assert if an image can be allocate
6654         my ($storeid, $scfg) = @_;
6655         die "Content type 'images' is not available on storage '$storeid'\n"
6656             if !$scfg->{content}->{images};
6657         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6658             if $user ne 'root@pam';
6659     };
6660
6661     my $virtdev_hash = {};
6662     while (defined(my $line = <$fh>)) {
6663         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6664             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6665             die "archive does not contain data for drive '$virtdev'\n"
6666                 if !$devinfo->{$devname};
6667
6668             if (defined($options->{storage})) {
6669                 $storeid = $options->{storage} || 'local';
6670             } elsif (!$storeid) {
6671                 $storeid = 'local';
6672             }
6673             $format = 'raw' if !$format;
6674             $devinfo->{$devname}->{devname} = $devname;
6675             $devinfo->{$devname}->{virtdev} = $virtdev;
6676             $devinfo->{$devname}->{format} = $format;
6677             $devinfo->{$devname}->{storeid} = $storeid;
6678
6679             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6680             $check_storage->($storeid, $scfg); # permission and content type check
6681
6682             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6683         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6684             my $virtdev = $1;
6685             my $drive = parse_drive($virtdev, $2);
6686
6687             if (drive_is_cloudinit($drive)) {
6688                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6689                 $storeid = $options->{storage} if defined ($options->{storage});
6690                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6691                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6692
6693                 $check_storage->($storeid, $scfg); # permission and content type check
6694
6695                 $virtdev_hash->{$virtdev} = {
6696                     format => $format,
6697                     storeid => $storeid,
6698                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6699                     is_cloudinit => 1,
6700                 };
6701             }
6702         }
6703     }
6704
6705     return $virtdev_hash;
6706 };
6707
6708 # Helper to allocate and activate all volumes required for a restore
6709 #
6710 # $storecfg: Storage configuration
6711 # $virtdev_hash: as returned by parse_backup_hints()
6712 #
6713 # Returns: { $virtdev => $volid }
6714 my $restore_allocate_devices = sub {
6715     my ($storecfg, $virtdev_hash, $vmid) = @_;
6716
6717     my $map = {};
6718     foreach my $virtdev (sort keys %$virtdev_hash) {
6719         my $d = $virtdev_hash->{$virtdev};
6720         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6721         my $storeid = $d->{storeid};
6722         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6723
6724         # test if requested format is supported
6725         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6726         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6727         $d->{format} = $defFormat if !$supported;
6728
6729         my $name;
6730         if ($d->{is_cloudinit}) {
6731             $name = "vm-$vmid-cloudinit";
6732             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6733             if ($scfg->{path}) {
6734                 $name .= ".$d->{format}";
6735             }
6736         }
6737
6738         my $volid = PVE::Storage::vdisk_alloc(
6739             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6740
6741         print STDERR "new volume ID is '$volid'\n";
6742         $d->{volid} = $volid;
6743
6744         PVE::Storage::activate_volumes($storecfg, [$volid]);
6745
6746         $map->{$virtdev} = $volid;
6747     }
6748
6749     return $map;
6750 };
6751
6752 sub restore_update_config_line {
6753     my ($cookie, $map, $line, $unique) = @_;
6754
6755     return '' if $line =~ m/^\#qmdump\#/;
6756     return '' if $line =~ m/^\#vzdump\#/;
6757     return '' if $line =~ m/^lock:/;
6758     return '' if $line =~ m/^unused\d+:/;
6759     return '' if $line =~ m/^parent:/;
6760
6761     my $res = '';
6762
6763     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6764     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6765         # try to convert old 1.X settings
6766         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6767         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6768             my ($model, $macaddr) = split(/\=/, $devconfig);
6769             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6770             my $net = {
6771                 model => $model,
6772                 bridge => "vmbr$ind",
6773                 macaddr => $macaddr,
6774             };
6775             my $netstr = print_net($net);
6776
6777             $res .= "net$cookie->{netcount}: $netstr\n";
6778             $cookie->{netcount}++;
6779         }
6780     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6781         my ($id, $netstr) = ($1, $2);
6782         my $net = parse_net($netstr);
6783         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6784         $netstr = print_net($net);
6785         $res .= "$id: $netstr\n";
6786     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6787         my $virtdev = $1;
6788         my $value = $3;
6789         my $di = parse_drive($virtdev, $value);
6790         if (defined($di->{backup}) && !$di->{backup}) {
6791             $res .= "#$line";
6792         } elsif ($map->{$virtdev}) {
6793             delete $di->{format}; # format can change on restore
6794             $di->{file} = $map->{$virtdev};
6795             $value = print_drive($di);
6796             $res .= "$virtdev: $value\n";
6797         } else {
6798             $res .= $line;
6799         }
6800     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6801         my $vmgenid = $1;
6802         if ($vmgenid ne '0') {
6803             # always generate a new vmgenid if there was a valid one setup
6804             $vmgenid = generate_uuid();
6805         }
6806         $res .= "vmgenid: $vmgenid\n";
6807     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6808         my ($uuid, $uuid_str);
6809         UUID::generate($uuid);
6810         UUID::unparse($uuid, $uuid_str);
6811         my $smbios1 = parse_smbios1($2);
6812         $smbios1->{uuid} = $uuid_str;
6813         $res .= $1.print_smbios1($smbios1)."\n";
6814     } else {
6815         $res .= $line;
6816     }
6817
6818     return $res;
6819 }
6820
6821 my $restore_deactivate_volumes = sub {
6822     my ($storecfg, $virtdev_hash) = @_;
6823
6824     my $vollist = [];
6825     for my $dev (values $virtdev_hash->%*) {
6826         push $vollist->@*, $dev->{volid} if $dev->{volid};
6827     }
6828
6829     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6830     print STDERR $@ if $@;
6831 };
6832
6833 my $restore_destroy_volumes = sub {
6834     my ($storecfg, $virtdev_hash) = @_;
6835
6836     for my $dev (values $virtdev_hash->%*) {
6837         my $volid = $dev->{volid} or next;
6838         eval {
6839             PVE::Storage::vdisk_free($storecfg, $volid);
6840             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6841         };
6842         print STDERR "unable to cleanup '$volid' - $@" if $@;
6843     }
6844 };
6845
6846 my $restore_merge_config = sub {
6847     my ($filename, $backup_conf_raw, $override_conf) = @_;
6848
6849     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6850     for my $key (keys $override_conf->%*) {
6851         $backup_conf->{$key} = $override_conf->{$key};
6852     }
6853
6854     return $backup_conf;
6855 };
6856
6857 sub scan_volids {
6858     my ($cfg, $vmid) = @_;
6859
6860     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6861
6862     my $volid_hash = {};
6863     foreach my $storeid (keys %$info) {
6864         foreach my $item (@{$info->{$storeid}}) {
6865             next if !($item->{volid} && $item->{size});
6866             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6867             $volid_hash->{$item->{volid}} = $item;
6868         }
6869     }
6870
6871     return $volid_hash;
6872 }
6873
6874 sub update_disk_config {
6875     my ($vmid, $conf, $volid_hash) = @_;
6876
6877     my $changes;
6878     my $prefix = "VM $vmid";
6879
6880     # used and unused disks
6881     my $referenced = {};
6882
6883     # Note: it is allowed to define multiple storages with same path (alias), so
6884     # we need to check both 'volid' and real 'path' (two different volid can point
6885     # to the same path).
6886
6887     my $referencedpath = {};
6888
6889     # update size info
6890     PVE::QemuConfig->foreach_volume($conf, sub {
6891         my ($opt, $drive) = @_;
6892
6893         my $volid = $drive->{file};
6894         return if !$volid;
6895         my $volume = $volid_hash->{$volid};
6896
6897         # mark volid as "in-use" for next step
6898         $referenced->{$volid} = 1;
6899         if ($volume && (my $path = $volume->{path})) {
6900             $referencedpath->{$path} = 1;
6901         }
6902
6903         return if drive_is_cdrom($drive);
6904         return if !$volume;
6905
6906         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6907         if (defined($updated)) {
6908             $changes = 1;
6909             $conf->{$opt} = print_drive($updated);
6910             print "$prefix ($opt): $msg\n";
6911         }
6912     });
6913
6914     # remove 'unusedX' entry if volume is used
6915     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6916         my ($opt, $drive) = @_;
6917
6918         my $volid = $drive->{file};
6919         return if !$volid;
6920
6921         my $path;
6922         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6923         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6924             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6925             $changes = 1;
6926             delete $conf->{$opt};
6927         }
6928
6929         $referenced->{$volid} = 1;
6930         $referencedpath->{$path} = 1 if $path;
6931     });
6932
6933     foreach my $volid (sort keys %$volid_hash) {
6934         next if $volid =~ m/vm-$vmid-state-/;
6935         next if $referenced->{$volid};
6936         my $path = $volid_hash->{$volid}->{path};
6937         next if !$path; # just to be sure
6938         next if $referencedpath->{$path};
6939         $changes = 1;
6940         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6941         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6942         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6943     }
6944
6945     return $changes;
6946 }
6947
6948 sub rescan {
6949     my ($vmid, $nolock, $dryrun) = @_;
6950
6951     my $cfg = PVE::Storage::config();
6952
6953     print "rescan volumes...\n";
6954     my $volid_hash = scan_volids($cfg, $vmid);
6955
6956     my $updatefn =  sub {
6957         my ($vmid) = @_;
6958
6959         my $conf = PVE::QemuConfig->load_config($vmid);
6960
6961         PVE::QemuConfig->check_lock($conf);
6962
6963         my $vm_volids = {};
6964         foreach my $volid (keys %$volid_hash) {
6965             my $info = $volid_hash->{$volid};
6966             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6967         }
6968
6969         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6970
6971         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6972     };
6973
6974     if (defined($vmid)) {
6975         if ($nolock) {
6976             &$updatefn($vmid);
6977         } else {
6978             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6979         }
6980     } else {
6981         my $vmlist = config_list();
6982         foreach my $vmid (keys %$vmlist) {
6983             if ($nolock) {
6984                 &$updatefn($vmid);
6985             } else {
6986                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6987             }
6988         }
6989     }
6990 }
6991
6992 sub restore_proxmox_backup_archive {
6993     my ($archive, $vmid, $user, $options) = @_;
6994
6995     my $storecfg = PVE::Storage::config();
6996
6997     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6998     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6999
7000     my $fingerprint = $scfg->{fingerprint};
7001     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7002
7003     my $repo = PVE::PBSClient::get_repository($scfg);
7004     my $namespace = $scfg->{namespace};
7005
7006     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7007     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7008     local $ENV{PBS_PASSWORD} = $password;
7009     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7010
7011     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7012         PVE::Storage::parse_volname($storecfg, $archive);
7013
7014     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7015
7016     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7017
7018     my $tmpdir = "/var/tmp/vzdumptmp$$";
7019     rmtree $tmpdir;
7020     mkpath $tmpdir;
7021
7022     my $conffile = PVE::QemuConfig->config_file($vmid);
7023      # disable interrupts (always do cleanups)
7024     local $SIG{INT} =
7025         local $SIG{TERM} =
7026         local $SIG{QUIT} =
7027         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7028
7029     # Note: $oldconf is undef if VM does not exists
7030     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7031     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7032     my $new_conf_raw = '';
7033
7034     my $rpcenv = PVE::RPCEnvironment::get();
7035     my $devinfo = {}; # info about drives included in backup
7036     my $virtdev_hash = {}; # info about allocated drives
7037
7038     eval {
7039         # enable interrupts
7040         local $SIG{INT} =
7041             local $SIG{TERM} =
7042             local $SIG{QUIT} =
7043             local $SIG{HUP} =
7044             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7045
7046         my $cfgfn = "$tmpdir/qemu-server.conf";
7047         my $firewall_config_fn = "$tmpdir/fw.conf";
7048         my $index_fn = "$tmpdir/index.json";
7049
7050         my $cmd = "restore";
7051
7052         my $param = [$pbs_backup_name, "index.json", $index_fn];
7053         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7054         my $index = PVE::Tools::file_get_contents($index_fn);
7055         $index = decode_json($index);
7056
7057         foreach my $info (@{$index->{files}}) {
7058             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7059                 my $devname = $1;
7060                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7061                     $devinfo->{$devname}->{size} = $1;
7062                 } else {
7063                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7064                 }
7065             }
7066         }
7067
7068         my $is_qemu_server_backup = scalar(
7069             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7070         );
7071         if (!$is_qemu_server_backup) {
7072             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7073         }
7074         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7075
7076         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7077         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7078
7079         if ($has_firewall_config) {
7080             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7081             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7082
7083             my $pve_firewall_dir = '/etc/pve/firewall';
7084             mkdir $pve_firewall_dir; # make sure the dir exists
7085             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7086         }
7087
7088         my $fh = IO::File->new($cfgfn, "r") ||
7089             die "unable to read qemu-server.conf - $!\n";
7090
7091         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7092
7093         # fixme: rate limit?
7094
7095         # create empty/temp config
7096         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7097
7098         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7099
7100         # allocate volumes
7101         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7102
7103         foreach my $virtdev (sort keys %$virtdev_hash) {
7104             my $d = $virtdev_hash->{$virtdev};
7105             next if $d->{is_cloudinit}; # no need to restore cloudinit
7106
7107             # this fails if storage is unavailable
7108             my $volid = $d->{volid};
7109             my $path = PVE::Storage::path($storecfg, $volid);
7110
7111             # for live-restore we only want to preload the efidisk and TPM state
7112             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7113
7114             my @ns_arg;
7115             if (defined(my $ns = $scfg->{namespace})) {
7116                 @ns_arg = ('--ns', $ns);
7117             }
7118
7119             my $pbs_restore_cmd = [
7120                 '/usr/bin/pbs-restore',
7121                 '--repository', $repo,
7122                 @ns_arg,
7123                 $pbs_backup_name,
7124                 "$d->{devname}.img.fidx",
7125                 $path,
7126                 '--verbose',
7127                 ];
7128
7129             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7130             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7131
7132             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7133                 push @$pbs_restore_cmd, '--skip-zero';
7134             }
7135
7136             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7137             print "restore proxmox backup image: $dbg_cmdstring\n";
7138             run_command($pbs_restore_cmd);
7139         }
7140
7141         $fh->seek(0, 0) || die "seek failed - $!\n";
7142
7143         my $cookie = { netcount => 0 };
7144         while (defined(my $line = <$fh>)) {
7145             $new_conf_raw .= restore_update_config_line(
7146                 $cookie,
7147                 $map,
7148                 $line,
7149                 $options->{unique},
7150             );
7151         }
7152
7153         $fh->close();
7154     };
7155     my $err = $@;
7156
7157     if ($err || !$options->{live}) {
7158         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7159     }
7160
7161     rmtree $tmpdir;
7162
7163     if ($err) {
7164         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7165         die $err;
7166     }
7167
7168     if ($options->{live}) {
7169         # keep lock during live-restore
7170         $new_conf_raw .= "\nlock: create";
7171     }
7172
7173     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7174     check_bridge_access($rpcenv, $user, $new_conf);
7175     PVE::QemuConfig->write_config($vmid, $new_conf);
7176
7177     eval { rescan($vmid, 1); };
7178     warn $@ if $@;
7179
7180     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7181
7182     if ($options->{live}) {
7183         # enable interrupts
7184         local $SIG{INT} =
7185             local $SIG{TERM} =
7186             local $SIG{QUIT} =
7187             local $SIG{HUP} =
7188             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7189
7190         my $conf = PVE::QemuConfig->load_config($vmid);
7191         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7192
7193         # these special drives are already restored before start
7194         delete $devinfo->{'drive-efidisk0'};
7195         delete $devinfo->{'drive-tpmstate0-backup'};
7196
7197         my $pbs_opts = {
7198             repo => $repo,
7199             keyfile => $keyfile,
7200             snapshot => $pbs_backup_name,
7201             namespace => $namespace,
7202         };
7203         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7204
7205         PVE::QemuConfig->remove_lock($vmid, "create");
7206     }
7207 }
7208
7209 sub pbs_live_restore {
7210     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7211
7212     print "starting VM for live-restore\n";
7213     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7214
7215     my $pbs_backing = {};
7216     for my $ds (keys %$restored_disks) {
7217         $ds =~ m/^drive-(.*)$/;
7218         my $confname = $1;
7219         $pbs_backing->{$confname} = {
7220             repository => $opts->{repo},
7221             snapshot => $opts->{snapshot},
7222             archive => "$ds.img.fidx",
7223         };
7224         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7225         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7226
7227         my $drive = parse_drive($confname, $conf->{$confname});
7228         print "restoring '$ds' to '$drive->{file}'\n";
7229     }
7230
7231     my $drives_streamed = 0;
7232     eval {
7233         # make sure HA doesn't interrupt our restore by stopping the VM
7234         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7235             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7236         }
7237
7238         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7239         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7240         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7241
7242         my $qmeventd_fd = register_qmeventd_handle($vmid);
7243
7244         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7245         # this will effectively collapse the backing image chain consisting of
7246         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7247         # removes itself once all backing images vanish with 'auto-remove=on')
7248         my $jobs = {};
7249         for my $ds (sort keys %$restored_disks) {
7250             my $job_id = "restore-$ds";
7251             mon_cmd($vmid, 'block-stream',
7252                 'job-id' => $job_id,
7253                 device => "$ds",
7254             );
7255             $jobs->{$job_id} = {};
7256         }
7257
7258         mon_cmd($vmid, 'cont');
7259         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7260
7261         print "restore-drive jobs finished successfully, removing all tracking block devices"
7262             ." to disconnect from Proxmox Backup Server\n";
7263
7264         for my $ds (sort keys %$restored_disks) {
7265             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7266         }
7267
7268         close($qmeventd_fd);
7269     };
7270
7271     my $err = $@;
7272
7273     if ($err) {
7274         warn "An error occurred during live-restore: $err\n";
7275         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7276         die "live-restore failed\n";
7277     }
7278 }
7279
7280 sub restore_vma_archive {
7281     my ($archive, $vmid, $user, $opts, $comp) = @_;
7282
7283     my $readfrom = $archive;
7284
7285     my $cfg = PVE::Storage::config();
7286     my $commands = [];
7287     my $bwlimit = $opts->{bwlimit};
7288
7289     my $dbg_cmdstring = '';
7290     my $add_pipe = sub {
7291         my ($cmd) = @_;
7292         push @$commands, $cmd;
7293         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7294         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7295         $readfrom = '-';
7296     };
7297
7298     my $input = undef;
7299     if ($archive eq '-') {
7300         $input = '<&STDIN';
7301     } else {
7302         # If we use a backup from a PVE defined storage we also consider that
7303         # storage's rate limit:
7304         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7305         if (defined($volid)) {
7306             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7307             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7308             if ($readlimit) {
7309                 print STDERR "applying read rate limit: $readlimit\n";
7310                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7311                 $add_pipe->($cstream);
7312             }
7313         }
7314     }
7315
7316     if ($comp) {
7317         my $info = PVE::Storage::decompressor_info('vma', $comp);
7318         my $cmd = $info->{decompressor};
7319         push @$cmd, $readfrom;
7320         $add_pipe->($cmd);
7321     }
7322
7323     my $tmpdir = "/var/tmp/vzdumptmp$$";
7324     rmtree $tmpdir;
7325
7326     # disable interrupts (always do cleanups)
7327     local $SIG{INT} =
7328         local $SIG{TERM} =
7329         local $SIG{QUIT} =
7330         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7331
7332     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7333     POSIX::mkfifo($mapfifo, 0600);
7334     my $fifofh;
7335     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7336
7337     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7338
7339     my $oldtimeout;
7340     my $timeout = 5;
7341
7342     my $devinfo = {}; # info about drives included in backup
7343     my $virtdev_hash = {}; # info about allocated drives
7344
7345     my $rpcenv = PVE::RPCEnvironment::get();
7346
7347     my $conffile = PVE::QemuConfig->config_file($vmid);
7348
7349     # Note: $oldconf is undef if VM does not exist
7350     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7351     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7352     my $new_conf_raw = '';
7353
7354     my %storage_limits;
7355
7356     my $print_devmap = sub {
7357         my $cfgfn = "$tmpdir/qemu-server.conf";
7358
7359         # we can read the config - that is already extracted
7360         my $fh = IO::File->new($cfgfn, "r") ||
7361             die "unable to read qemu-server.conf - $!\n";
7362
7363         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7364         if (-f $fwcfgfn) {
7365             my $pve_firewall_dir = '/etc/pve/firewall';
7366             mkdir $pve_firewall_dir; # make sure the dir exists
7367             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7368         }
7369
7370         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7371
7372         foreach my $info (values %{$virtdev_hash}) {
7373             my $storeid = $info->{storeid};
7374             next if defined($storage_limits{$storeid});
7375
7376             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7377             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7378             $storage_limits{$storeid} = $limit * 1024;
7379         }
7380
7381         foreach my $devname (keys %$devinfo) {
7382             die "found no device mapping information for device '$devname'\n"
7383                 if !$devinfo->{$devname}->{virtdev};
7384         }
7385
7386         # create empty/temp config
7387         if ($oldconf) {
7388             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7389             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7390         }
7391
7392         # allocate volumes
7393         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7394
7395         # print restore information to $fifofh
7396         foreach my $virtdev (sort keys %$virtdev_hash) {
7397             my $d = $virtdev_hash->{$virtdev};
7398             next if $d->{is_cloudinit}; # no need to restore cloudinit
7399
7400             my $storeid = $d->{storeid};
7401             my $volid = $d->{volid};
7402
7403             my $map_opts = '';
7404             if (my $limit = $storage_limits{$storeid}) {
7405                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7406             }
7407
7408             my $write_zeros = 1;
7409             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7410                 $write_zeros = 0;
7411             }
7412
7413             my $path = PVE::Storage::path($cfg, $volid);
7414
7415             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7416
7417             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7418         }
7419
7420         $fh->seek(0, 0) || die "seek failed - $!\n";
7421
7422         my $cookie = { netcount => 0 };
7423         while (defined(my $line = <$fh>)) {
7424             $new_conf_raw .= restore_update_config_line(
7425                 $cookie,
7426                 $map,
7427                 $line,
7428                 $opts->{unique},
7429             );
7430         }
7431
7432         $fh->close();
7433     };
7434
7435     eval {
7436         # enable interrupts
7437         local $SIG{INT} =
7438             local $SIG{TERM} =
7439             local $SIG{QUIT} =
7440             local $SIG{HUP} =
7441             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7442         local $SIG{ALRM} = sub { die "got timeout\n"; };
7443
7444         $oldtimeout = alarm($timeout);
7445
7446         my $parser = sub {
7447             my $line = shift;
7448
7449             print "$line\n";
7450
7451             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7452                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7453                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7454             } elsif ($line =~ m/^CTIME: /) {
7455                 # we correctly received the vma config, so we can disable
7456                 # the timeout now for disk allocation (set to 10 minutes, so
7457                 # that we always timeout if something goes wrong)
7458                 alarm(600);
7459                 &$print_devmap();
7460                 print $fifofh "done\n";
7461                 my $tmp = $oldtimeout || 0;
7462                 $oldtimeout = undef;
7463                 alarm($tmp);
7464                 close($fifofh);
7465                 $fifofh = undef;
7466             }
7467         };
7468
7469         print "restore vma archive: $dbg_cmdstring\n";
7470         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7471     };
7472     my $err = $@;
7473
7474     alarm($oldtimeout) if $oldtimeout;
7475
7476     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7477
7478     close($fifofh) if $fifofh;
7479     unlink $mapfifo;
7480     rmtree $tmpdir;
7481
7482     if ($err) {
7483         $restore_destroy_volumes->($cfg, $virtdev_hash);
7484         die $err;
7485     }
7486
7487     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7488     check_bridge_access($rpcenv, $user, $new_conf);
7489     PVE::QemuConfig->write_config($vmid, $new_conf);
7490
7491     eval { rescan($vmid, 1); };
7492     warn $@ if $@;
7493
7494     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7495 }
7496
7497 sub restore_tar_archive {
7498     my ($archive, $vmid, $user, $opts) = @_;
7499
7500     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7501         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7502         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7503     }
7504
7505     if ($archive ne '-') {
7506         my $firstfile = tar_archive_read_firstfile($archive);
7507         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7508             if $firstfile ne 'qemu-server.conf';
7509     }
7510
7511     my $storecfg = PVE::Storage::config();
7512
7513     # avoid zombie disks when restoring over an existing VM -> cleanup first
7514     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7515     # skiplock=1 because qmrestore has set the 'create' lock itself already
7516     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7517     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7518
7519     my $tocmd = "/usr/lib/qemu-server/qmextract";
7520
7521     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7522     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7523     $tocmd .= ' --prealloc' if $opts->{prealloc};
7524     $tocmd .= ' --info' if $opts->{info};
7525
7526     # tar option "xf" does not autodetect compression when read from STDIN,
7527     # so we pipe to zcat
7528     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7529         PVE::Tools::shellquote("--to-command=$tocmd");
7530
7531     my $tmpdir = "/var/tmp/vzdumptmp$$";
7532     mkpath $tmpdir;
7533
7534     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7535     local $ENV{VZDUMP_VMID} = $vmid;
7536     local $ENV{VZDUMP_USER} = $user;
7537
7538     my $conffile = PVE::QemuConfig->config_file($vmid);
7539     my $new_conf_raw = '';
7540
7541     # disable interrupts (always do cleanups)
7542     local $SIG{INT} =
7543         local $SIG{TERM} =
7544         local $SIG{QUIT} =
7545         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7546
7547     eval {
7548         # enable interrupts
7549         local $SIG{INT} =
7550             local $SIG{TERM} =
7551             local $SIG{QUIT} =
7552             local $SIG{HUP} =
7553             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7554
7555         if ($archive eq '-') {
7556             print "extracting archive from STDIN\n";
7557             run_command($cmd, input => "<&STDIN");
7558         } else {
7559             print "extracting archive '$archive'\n";
7560             run_command($cmd);
7561         }
7562
7563         return if $opts->{info};
7564
7565         # read new mapping
7566         my $map = {};
7567         my $statfile = "$tmpdir/qmrestore.stat";
7568         if (my $fd = IO::File->new($statfile, "r")) {
7569             while (defined (my $line = <$fd>)) {
7570                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7571                     $map->{$1} = $2 if $1;
7572                 } else {
7573                     print STDERR "unable to parse line in statfile - $line\n";
7574                 }
7575             }
7576             $fd->close();
7577         }
7578
7579         my $confsrc = "$tmpdir/qemu-server.conf";
7580
7581         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7582
7583         my $cookie = { netcount => 0 };
7584         while (defined (my $line = <$srcfd>)) {
7585             $new_conf_raw .= restore_update_config_line(
7586                 $cookie,
7587                 $map,
7588                 $line,
7589                 $opts->{unique},
7590             );
7591         }
7592
7593         $srcfd->close();
7594     };
7595     if (my $err = $@) {
7596         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7597         die $err;
7598     }
7599
7600     rmtree $tmpdir;
7601
7602     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7603
7604     PVE::Cluster::cfs_update(); # make sure we read new file
7605
7606     eval { rescan($vmid, 1); };
7607     warn $@ if $@;
7608 };
7609
7610 sub foreach_storage_used_by_vm {
7611     my ($conf, $func) = @_;
7612
7613     my $sidhash = {};
7614
7615     PVE::QemuConfig->foreach_volume($conf, sub {
7616         my ($ds, $drive) = @_;
7617         return if drive_is_cdrom($drive);
7618
7619         my $volid = $drive->{file};
7620
7621         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7622         $sidhash->{$sid} = $sid if $sid;
7623     });
7624
7625     foreach my $sid (sort keys %$sidhash) {
7626         &$func($sid);
7627     }
7628 }
7629
7630 my $qemu_snap_storage = {
7631     rbd => 1,
7632 };
7633 sub do_snapshots_with_qemu {
7634     my ($storecfg, $volid, $deviceid) = @_;
7635
7636     return if $deviceid =~ m/tpmstate0/;
7637
7638     my $storage_name = PVE::Storage::parse_volume_id($volid);
7639     my $scfg = $storecfg->{ids}->{$storage_name};
7640     die "could not find storage '$storage_name'\n" if !defined($scfg);
7641
7642     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7643         return 1;
7644     }
7645
7646     if ($volid =~ m/\.(qcow2|qed)$/){
7647         return 1;
7648     }
7649
7650     return;
7651 }
7652
7653 sub qga_check_running {
7654     my ($vmid, $nowarn) = @_;
7655
7656     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7657     if ($@) {
7658         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7659         return 0;
7660     }
7661     return 1;
7662 }
7663
7664 sub template_create {
7665     my ($vmid, $conf, $disk) = @_;
7666
7667     my $storecfg = PVE::Storage::config();
7668
7669     PVE::QemuConfig->foreach_volume($conf, sub {
7670         my ($ds, $drive) = @_;
7671
7672         return if drive_is_cdrom($drive);
7673         return if $disk && $ds ne $disk;
7674
7675         my $volid = $drive->{file};
7676         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7677
7678         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7679         $drive->{file} = $voliddst;
7680         $conf->{$ds} = print_drive($drive);
7681         PVE::QemuConfig->write_config($vmid, $conf);
7682     });
7683 }
7684
7685 sub convert_iscsi_path {
7686     my ($path) = @_;
7687
7688     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7689         my $portal = $1;
7690         my $target = $2;
7691         my $lun = $3;
7692
7693         my $initiator_name = get_initiator_name();
7694
7695         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7696                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7697     }
7698
7699     die "cannot convert iscsi path '$path', unkown format\n";
7700 }
7701
7702 sub qemu_img_convert {
7703     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7704
7705     my $storecfg = PVE::Storage::config();
7706     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7707     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7708
7709     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7710
7711     my $cachemode;
7712     my $src_path;
7713     my $src_is_iscsi = 0;
7714     my $src_format;
7715
7716     if ($src_storeid) {
7717         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7718         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7719         $src_format = qemu_img_format($src_scfg, $src_volname);
7720         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7721         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7722         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7723     } elsif (-f $src_volid || -b $src_volid) {
7724         $src_path = $src_volid;
7725         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7726             $src_format = $1;
7727         }
7728     }
7729
7730     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7731
7732     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7733     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7734     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7735     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7736
7737     my $cmd = [];
7738     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7739     push @$cmd, '-l', "snapshot.name=$snapname"
7740         if $snapname && $src_format && $src_format eq "qcow2";
7741     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7742     push @$cmd, '-T', $cachemode if defined($cachemode);
7743     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7744
7745     if ($src_is_iscsi) {
7746         push @$cmd, '--image-opts';
7747         $src_path = convert_iscsi_path($src_path);
7748     } elsif ($src_format) {
7749         push @$cmd, '-f', $src_format;
7750     }
7751
7752     if ($dst_is_iscsi) {
7753         push @$cmd, '--target-image-opts';
7754         $dst_path = convert_iscsi_path($dst_path);
7755     } else {
7756         push @$cmd, '-O', $dst_format;
7757     }
7758
7759     push @$cmd, $src_path;
7760
7761     if (!$dst_is_iscsi && $is_zero_initialized) {
7762         push @$cmd, "zeroinit:$dst_path";
7763     } else {
7764         push @$cmd, $dst_path;
7765     }
7766
7767     my $parser = sub {
7768         my $line = shift;
7769         if($line =~ m/\((\S+)\/100\%\)/){
7770             my $percent = $1;
7771             my $transferred = int($size * $percent / 100);
7772             my $total_h = render_bytes($size, 1);
7773             my $transferred_h = render_bytes($transferred, 1);
7774
7775             print "transferred $transferred_h of $total_h ($percent%)\n";
7776         }
7777
7778     };
7779
7780     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7781     my $err = $@;
7782     die "copy failed: $err" if $err;
7783 }
7784
7785 sub qemu_img_format {
7786     my ($scfg, $volname) = @_;
7787
7788     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7789         return $1;
7790     } else {
7791         return "raw";
7792     }
7793 }
7794
7795 sub qemu_drive_mirror {
7796     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7797
7798     $jobs = {} if !$jobs;
7799
7800     my $qemu_target;
7801     my $format;
7802     $jobs->{"drive-$drive"} = {};
7803
7804     if ($dst_volid =~ /^nbd:/) {
7805         $qemu_target = $dst_volid;
7806         $format = "nbd";
7807     } else {
7808         my $storecfg = PVE::Storage::config();
7809         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7810
7811         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7812
7813         $format = qemu_img_format($dst_scfg, $dst_volname);
7814
7815         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7816
7817         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7818     }
7819
7820     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7821     $opts->{format} = $format if $format;
7822
7823     if (defined($src_bitmap)) {
7824         $opts->{sync} = 'incremental';
7825         $opts->{bitmap} = $src_bitmap;
7826         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7827     }
7828
7829     if (defined($bwlimit)) {
7830         $opts->{speed} = $bwlimit * 1024;
7831         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7832     } else {
7833         print "drive mirror is starting for drive-$drive\n";
7834     }
7835
7836     # if a job already runs for this device we get an error, catch it for cleanup
7837     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7838     if (my $err = $@) {
7839         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7840         warn "$@\n" if $@;
7841         die "mirroring error: $err\n";
7842     }
7843
7844     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7845 }
7846
7847 # $completion can be either
7848 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7849 # 'cancel': wait until all jobs are ready, block-job-cancel them
7850 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7851 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7852 sub qemu_drive_mirror_monitor {
7853     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7854
7855     $completion //= 'complete';
7856     $op //= "mirror";
7857
7858     eval {
7859         my $err_complete = 0;
7860
7861         my $starttime = time ();
7862         while (1) {
7863             die "block job ('$op') timed out\n" if $err_complete > 300;
7864
7865             my $stats = mon_cmd($vmid, "query-block-jobs");
7866             my $ctime = time();
7867
7868             my $running_jobs = {};
7869             for my $stat (@$stats) {
7870                 next if $stat->{type} ne $op;
7871                 $running_jobs->{$stat->{device}} = $stat;
7872             }
7873
7874             my $readycounter = 0;
7875
7876             for my $job_id (sort keys %$jobs) {
7877                 my $job = $running_jobs->{$job_id};
7878
7879                 my $vanished = !defined($job);
7880                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7881                 if($complete || ($vanished && $completion eq 'auto')) {
7882                     print "$job_id: $op-job finished\n";
7883                     delete $jobs->{$job_id};
7884                     next;
7885                 }
7886
7887                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7888
7889                 my $busy = $job->{busy};
7890                 my $ready = $job->{ready};
7891                 if (my $total = $job->{len}) {
7892                     my $transferred = $job->{offset} || 0;
7893                     my $remaining = $total - $transferred;
7894                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7895
7896                     my $duration = $ctime - $starttime;
7897                     my $total_h = render_bytes($total, 1);
7898                     my $transferred_h = render_bytes($transferred, 1);
7899
7900                     my $status = sprintf(
7901                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7902                         render_duration($duration),
7903                     );
7904
7905                     if ($ready) {
7906                         if ($busy) {
7907                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7908                         } else {
7909                             $status .= ", ready";
7910                         }
7911                     }
7912                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7913                     $jobs->{$job_id}->{ready} = $ready;
7914                 }
7915
7916                 $readycounter++ if $job->{ready};
7917             }
7918
7919             last if scalar(keys %$jobs) == 0;
7920
7921             if ($readycounter == scalar(keys %$jobs)) {
7922                 print "all '$op' jobs are ready\n";
7923
7924                 # do the complete later (or has already been done)
7925                 last if $completion eq 'skip' || $completion eq 'auto';
7926
7927                 if ($vmiddst && $vmiddst != $vmid) {
7928                     my $agent_running = $qga && qga_check_running($vmid);
7929                     if ($agent_running) {
7930                         print "freeze filesystem\n";
7931                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7932                         warn $@ if $@;
7933                     } else {
7934                         print "suspend vm\n";
7935                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7936                         warn $@ if $@;
7937                     }
7938
7939                     # if we clone a disk for a new target vm, we don't switch the disk
7940                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7941
7942                     if ($agent_running) {
7943                         print "unfreeze filesystem\n";
7944                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7945                         warn $@ if $@;
7946                     } else {
7947                         print "resume vm\n";
7948                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7949                         warn $@ if $@;
7950                     }
7951
7952                     last;
7953                 } else {
7954
7955                     for my $job_id (sort keys %$jobs) {
7956                         # try to switch the disk if source and destination are on the same guest
7957                         print "$job_id: Completing block job_id...\n";
7958
7959                         my $op;
7960                         if ($completion eq 'complete') {
7961                             $op = 'block-job-complete';
7962                         } elsif ($completion eq 'cancel') {
7963                             $op = 'block-job-cancel';
7964                         } else {
7965                             die "invalid completion value: $completion\n";
7966                         }
7967                         eval { mon_cmd($vmid, $op, device => $job_id) };
7968                         if ($@ =~ m/cannot be completed/) {
7969                             print "$job_id: block job cannot be completed, trying again.\n";
7970                             $err_complete++;
7971                         }else {
7972                             print "$job_id: Completed successfully.\n";
7973                             $jobs->{$job_id}->{complete} = 1;
7974                         }
7975                     }
7976                 }
7977             }
7978             sleep 1;
7979         }
7980     };
7981     my $err = $@;
7982
7983     if ($err) {
7984         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7985         die "block job ($op) error: $err";
7986     }
7987 }
7988
7989 sub qemu_blockjobs_cancel {
7990     my ($vmid, $jobs) = @_;
7991
7992     foreach my $job (keys %$jobs) {
7993         print "$job: Cancelling block job\n";
7994         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7995         $jobs->{$job}->{cancel} = 1;
7996     }
7997
7998     while (1) {
7999         my $stats = mon_cmd($vmid, "query-block-jobs");
8000
8001         my $running_jobs = {};
8002         foreach my $stat (@$stats) {
8003             $running_jobs->{$stat->{device}} = $stat;
8004         }
8005
8006         foreach my $job (keys %$jobs) {
8007
8008             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8009                 print "$job: Done.\n";
8010                 delete $jobs->{$job};
8011             }
8012         }
8013
8014         last if scalar(keys %$jobs) == 0;
8015
8016         sleep 1;
8017     }
8018 }
8019
8020 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8021 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8022 my sub clone_disk_check_io_uring {
8023     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8024
8025     return if !$use_drive_mirror;
8026
8027     # Don't complain when not changing storage.
8028     # Assume if it works for the source, it'll work for the target too.
8029     return if $src_storeid eq $dst_storeid;
8030
8031     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8032     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8033
8034     my $cache_direct = drive_uses_cache_direct($src_drive);
8035
8036     my $src_uses_io_uring;
8037     if ($src_drive->{aio}) {
8038         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8039     } else {
8040         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8041     }
8042
8043     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8044         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8045 }
8046
8047 sub clone_disk {
8048     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8049
8050     my ($vmid, $running) = $source->@{qw(vmid running)};
8051     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8052
8053     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8054     my ($storage, $format) = $dest->@{qw(storage format)};
8055
8056     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8057
8058     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8059         die "cloning from/to EFI disk requires EFI disk\n"
8060             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8061         die "cloning from/to TPM state requires TPM state\n"
8062             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8063
8064         # This would lead to two device nodes in QEMU pointing to the same backing image!
8065         die "cannot change drive name when cloning disk from/to the same VM\n"
8066             if $use_drive_mirror && $vmid == $newvmid;
8067     }
8068
8069     die "cannot move TPM state while VM is running\n"
8070         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8071
8072     my $newvolid;
8073
8074     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8075     print "$src_drivename " if $src_drivename;
8076     print "($drive->{file})\n";
8077
8078     if (!$full) {
8079         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8080         push @$newvollist, $newvolid;
8081     } else {
8082         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8083         my $storeid = $storage || $src_storeid;
8084
8085         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8086
8087         my $name = undef;
8088         my $size = undef;
8089         if (drive_is_cloudinit($drive)) {
8090             $name = "vm-$newvmid-cloudinit";
8091             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8092             if ($scfg->{path}) {
8093                 $name .= ".$dst_format";
8094             }
8095             $snapname = undef;
8096             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8097         } elsif ($dst_drivename eq 'efidisk0') {
8098             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8099         } elsif ($dst_drivename eq 'tpmstate0') {
8100             $dst_format = 'raw';
8101             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8102         } else {
8103             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8104
8105             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8106         }
8107         $newvolid = PVE::Storage::vdisk_alloc(
8108             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8109         );
8110         push @$newvollist, $newvolid;
8111
8112         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8113
8114         if (drive_is_cloudinit($drive)) {
8115             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8116             # if this is the case, we have to complete any block-jobs still there from
8117             # previous drive-mirrors
8118             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8119                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8120             }
8121             goto no_data_clone;
8122         }
8123
8124         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8125         if ($use_drive_mirror) {
8126             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8127                 $completion, $qga, $bwlimit);
8128         } else {
8129             if ($dst_drivename eq 'efidisk0') {
8130                 # the relevant data on the efidisk may be smaller than the source
8131                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8132                 # that is given by the OVMF_VARS.fd
8133                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8134                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8135
8136                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8137
8138                 # better for Ceph if block size is not too small, see bug #3324
8139                 my $bs = 1024*1024;
8140
8141                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8142
8143                 if ($src_format eq 'qcow2' && $snapname) {
8144                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8145                         if !min_version(kvm_user_version(), 6, 2);
8146                     push $cmd->@*, '-l', $snapname;
8147                 }
8148                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8149                 run_command($cmd);
8150             } else {
8151                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8152             }
8153         }
8154     }
8155
8156 no_data_clone:
8157     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8158
8159     my $disk = dclone($drive);
8160     delete $disk->{format};
8161     $disk->{file} = $newvolid;
8162     $disk->{size} = $size if defined($size);
8163
8164     return $disk;
8165 }
8166
8167 sub get_running_qemu_version {
8168     my ($vmid) = @_;
8169     my $res = mon_cmd($vmid, "query-version");
8170     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8171 }
8172
8173 sub qemu_use_old_bios_files {
8174     my ($machine_type) = @_;
8175
8176     return if !$machine_type;
8177
8178     my $use_old_bios_files = undef;
8179
8180     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8181         $machine_type = $1;
8182         $use_old_bios_files = 1;
8183     } else {
8184         my $version = extract_version($machine_type, kvm_user_version());
8185         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8186         # load new efi bios files on migration. So this hack is required to allow
8187         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8188         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8189         $use_old_bios_files = !min_version($version, 2, 4);
8190     }
8191
8192     return ($use_old_bios_files, $machine_type);
8193 }
8194
8195 sub get_efivars_size {
8196     my ($conf, $efidisk) = @_;
8197
8198     my $arch = get_vm_arch($conf);
8199     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8200     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8201     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8202     return -s $ovmf_vars;
8203 }
8204
8205 sub update_efidisk_size {
8206     my ($conf) = @_;
8207
8208     return if !defined($conf->{efidisk0});
8209
8210     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8211     $disk->{size} = get_efivars_size($conf);
8212     $conf->{efidisk0} = print_drive($disk);
8213
8214     return;
8215 }
8216
8217 sub update_tpmstate_size {
8218     my ($conf) = @_;
8219
8220     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8221     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8222     $conf->{tpmstate0} = print_drive($disk);
8223 }
8224
8225 sub create_efidisk($$$$$$$) {
8226     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8227
8228     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8229
8230     my $vars_size_b = -s $ovmf_vars;
8231     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8232     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8233     PVE::Storage::activate_volumes($storecfg, [$volid]);
8234
8235     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8236     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8237
8238     return ($volid, $size/1024);
8239 }
8240
8241 sub vm_iothreads_list {
8242     my ($vmid) = @_;
8243
8244     my $res = mon_cmd($vmid, 'query-iothreads');
8245
8246     my $iothreads = {};
8247     foreach my $iothread (@$res) {
8248         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8249     }
8250
8251     return $iothreads;
8252 }
8253
8254 sub scsihw_infos {
8255     my ($conf, $drive) = @_;
8256
8257     my $maxdev = 0;
8258
8259     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8260         $maxdev = 7;
8261     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8262         $maxdev = 1;
8263     } else {
8264         $maxdev = 256;
8265     }
8266
8267     my $controller = int($drive->{index} / $maxdev);
8268     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8269         ? "virtioscsi"
8270         : "scsihw";
8271
8272     return ($maxdev, $controller, $controller_prefix);
8273 }
8274
8275 sub resolve_dst_disk_format {
8276         my ($storecfg, $storeid, $src_volname, $format) = @_;
8277         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8278
8279         if (!$format) {
8280             # if no target format is specified, use the source disk format as hint
8281             if ($src_volname) {
8282                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8283                 $format = qemu_img_format($scfg, $src_volname);
8284             } else {
8285                 return $defFormat;
8286             }
8287         }
8288
8289         # test if requested format is supported - else use default
8290         my $supported = grep { $_ eq $format } @$validFormats;
8291         $format = $defFormat if !$supported;
8292         return $format;
8293 }
8294
8295 # NOTE: if this logic changes, please update docs & possibly gui logic
8296 sub find_vmstate_storage {
8297     my ($conf, $storecfg) = @_;
8298
8299     # first, return storage from conf if set
8300     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8301
8302     my ($target, $shared, $local);
8303
8304     foreach_storage_used_by_vm($conf, sub {
8305         my ($sid) = @_;
8306         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8307         my $dst = $scfg->{shared} ? \$shared : \$local;
8308         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8309     });
8310
8311     # second, use shared storage where VM has at least one disk
8312     # third, use local storage where VM has at least one disk
8313     # fall back to local storage
8314     $target = $shared // $local // 'local';
8315
8316     return $target;
8317 }
8318
8319 sub generate_uuid {
8320     my ($uuid, $uuid_str);
8321     UUID::generate($uuid);
8322     UUID::unparse($uuid, $uuid_str);
8323     return $uuid_str;
8324 }
8325
8326 sub generate_smbios1_uuid {
8327     return "uuid=".generate_uuid();
8328 }
8329
8330 sub nbd_stop {
8331     my ($vmid) = @_;
8332
8333     mon_cmd($vmid, 'nbd-server-stop');
8334 }
8335
8336 sub create_reboot_request {
8337     my ($vmid) = @_;
8338     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8339         or die "failed to create reboot trigger file: $!\n";
8340     close($fh);
8341 }
8342
8343 sub clear_reboot_request {
8344     my ($vmid) = @_;
8345     my $path = "/run/qemu-server/$vmid.reboot";
8346     my $res = 0;
8347
8348     $res = unlink($path);
8349     die "could not remove reboot request for $vmid: $!"
8350         if !$res && $! != POSIX::ENOENT;
8351
8352     return $res;
8353 }
8354
8355 sub bootorder_from_legacy {
8356     my ($conf, $bootcfg) = @_;
8357
8358     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8359     my $bootindex_hash = {};
8360     my $i = 1;
8361     foreach my $o (split(//, $boot)) {
8362         $bootindex_hash->{$o} = $i*100;
8363         $i++;
8364     }
8365
8366     my $bootorder = {};
8367
8368     PVE::QemuConfig->foreach_volume($conf, sub {
8369         my ($ds, $drive) = @_;
8370
8371         if (drive_is_cdrom ($drive, 1)) {
8372             if ($bootindex_hash->{d}) {
8373                 $bootorder->{$ds} = $bootindex_hash->{d};
8374                 $bootindex_hash->{d} += 1;
8375             }
8376         } elsif ($bootindex_hash->{c}) {
8377             $bootorder->{$ds} = $bootindex_hash->{c}
8378                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8379             $bootindex_hash->{c} += 1;
8380         }
8381     });
8382
8383     if ($bootindex_hash->{n}) {
8384         for (my $i = 0; $i < $MAX_NETS; $i++) {
8385             my $netname = "net$i";
8386             next if !$conf->{$netname};
8387             $bootorder->{$netname} = $bootindex_hash->{n};
8388             $bootindex_hash->{n} += 1;
8389         }
8390     }
8391
8392     return $bootorder;
8393 }
8394
8395 # Generate default device list for 'boot: order=' property. Matches legacy
8396 # default boot order, but with explicit device names. This is important, since
8397 # the fallback for when neither 'order' nor the old format is specified relies
8398 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8399 sub get_default_bootdevices {
8400     my ($conf) = @_;
8401
8402     my @ret = ();
8403
8404     # harddisk
8405     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8406     push @ret, $first if $first;
8407
8408     # cdrom
8409     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8410     push @ret, $first if $first;
8411
8412     # network
8413     for (my $i = 0; $i < $MAX_NETS; $i++) {
8414         my $netname = "net$i";
8415         next if !$conf->{$netname};
8416         push @ret, $netname;
8417         last;
8418     }
8419
8420     return \@ret;
8421 }
8422
8423 sub device_bootorder {
8424     my ($conf) = @_;
8425
8426     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8427
8428     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8429
8430     my $bootorder = {};
8431     if (!defined($boot) || $boot->{legacy}) {
8432         $bootorder = bootorder_from_legacy($conf, $boot);
8433     } elsif ($boot->{order}) {
8434         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8435         for my $dev (PVE::Tools::split_list($boot->{order})) {
8436             $bootorder->{$dev} = $i++;
8437         }
8438     }
8439
8440     return $bootorder;
8441 }
8442
8443 sub register_qmeventd_handle {
8444     my ($vmid) = @_;
8445
8446     my $fh;
8447     my $peer = "/var/run/qmeventd.sock";
8448     my $count = 0;
8449
8450     for (;;) {
8451         $count++;
8452         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8453         last if $fh;
8454         if ($! != EINTR && $! != EAGAIN) {
8455             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8456         }
8457         if ($count > 4) {
8458             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8459               . "after $count retries\n";
8460         }
8461         usleep(25000);
8462     }
8463
8464     # send handshake to mark VM as backing up
8465     print $fh to_json({vzdump => {vmid => "$vmid"}});
8466
8467     # return handle to be closed later when inhibit is no longer required
8468     return $fh;
8469 }
8470
8471 # bash completion helper
8472
8473 sub complete_backup_archives {
8474     my ($cmdname, $pname, $cvalue) = @_;
8475
8476     my $cfg = PVE::Storage::config();
8477
8478     my $storeid;
8479
8480     if ($cvalue =~ m/^([^:]+):/) {
8481         $storeid = $1;
8482     }
8483
8484     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8485
8486     my $res = [];
8487     foreach my $id (keys %$data) {
8488         foreach my $item (@{$data->{$id}}) {
8489             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8490             push @$res, $item->{volid} if defined($item->{volid});
8491         }
8492     }
8493
8494     return $res;
8495 }
8496
8497 my $complete_vmid_full = sub {
8498     my ($running) = @_;
8499
8500     my $idlist = vmstatus();
8501
8502     my $res = [];
8503
8504     foreach my $id (keys %$idlist) {
8505         my $d = $idlist->{$id};
8506         if (defined($running)) {
8507             next if $d->{template};
8508             next if $running && $d->{status} ne 'running';
8509             next if !$running && $d->{status} eq 'running';
8510         }
8511         push @$res, $id;
8512
8513     }
8514     return $res;
8515 };
8516
8517 sub complete_vmid {
8518     return &$complete_vmid_full();
8519 }
8520
8521 sub complete_vmid_stopped {
8522     return &$complete_vmid_full(0);
8523 }
8524
8525 sub complete_vmid_running {
8526     return &$complete_vmid_full(1);
8527 }
8528
8529 sub complete_storage {
8530
8531     my $cfg = PVE::Storage::config();
8532     my $ids = $cfg->{ids};
8533
8534     my $res = [];
8535     foreach my $sid (keys %$ids) {
8536         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8537         next if !$ids->{$sid}->{content}->{images};
8538         push @$res, $sid;
8539     }
8540
8541     return $res;
8542 }
8543
8544 sub complete_migration_storage {
8545     my ($cmd, $param, $current_value, $all_args) = @_;
8546
8547     my $targetnode = @$all_args[1];
8548
8549     my $cfg = PVE::Storage::config();
8550     my $ids = $cfg->{ids};
8551
8552     my $res = [];
8553     foreach my $sid (keys %$ids) {
8554         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8555         next if !$ids->{$sid}->{content}->{images};
8556         push @$res, $sid;
8557     }
8558
8559     return $res;
8560 }
8561
8562 sub vm_is_paused {
8563     my ($vmid) = @_;
8564     my $qmpstatus = eval {
8565         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8566         mon_cmd($vmid, "query-status");
8567     };
8568     warn "$@\n" if $@;
8569     return $qmpstatus && $qmpstatus->{status} eq "paused";
8570 }
8571
8572 sub check_volume_storage_type {
8573     my ($storecfg, $vol) = @_;
8574
8575     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8576     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8577     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8578
8579     die "storage '$storeid' does not support content-type '$vtype'\n"
8580         if !$scfg->{content}->{$vtype};
8581
8582     return 1;
8583 }
8584
8585 sub add_nets_bridge_fdb {
8586     my ($conf, $vmid) = @_;
8587
8588     for my $opt (keys %$conf) {
8589         next if $opt !~ m/^net(\d+)$/;
8590         my $iface = "tap${vmid}i$1";
8591         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8592         my $net = parse_net($conf->{$opt}, 1) or next;
8593
8594         my $mac = $net->{macaddr};
8595         if (!$mac) {
8596             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8597                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8598             next;
8599         }
8600
8601         my $bridge = $net->{bridge};
8602         if (!$bridge) {
8603             log_warn("Interface '$iface' not attached to any bridge.");
8604             next;
8605         }
8606         if ($have_sdn) {
8607             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8608         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8609             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8610         }
8611     }
8612 }
8613
8614 sub del_nets_bridge_fdb {
8615     my ($conf, $vmid) = @_;
8616
8617     for my $opt (keys %$conf) {
8618         next if $opt !~ m/^net(\d+)$/;
8619         my $iface = "tap${vmid}i$1";
8620
8621         my $net = parse_net($conf->{$opt}) or next;
8622         my $mac = $net->{macaddr} or next;
8623
8624         my $bridge = $net->{bridge};
8625         if ($have_sdn) {
8626             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8627         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8628             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8629         }
8630     }
8631 }
8632
8633 1;