PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196 };
 197
 198 my $ivshmem_fmt = {
 199     size => {
 200         type => 'integer',
 201         minimum => 1,
 202         description => "The size of the file in MB.",
 203     },
 204     name => {
 205         type => 'string',
 206         pattern => '[a-zA-Z0-9\-]+',
 207         optional => 1,
 208         format_description => 'string',
 209         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 210     },
 211 };
 212
 213 my $audio_fmt = {
 214     device => {
 215         type => 'string',
 216         enum => [qw(ich9-intel-hda intel-hda AC97)],
 217         description =>  "Configure an audio device."
 218     },
 219     driver =>  {
 220         type => 'string',
 221         enum => ['spice', 'none'],
 222         default => 'spice',
 223         optional => 1,
 224         description => "Driver backend for the audio device."
 225     },
 226 };
 227
 228 my $spice_enhancements_fmt = {
 229     foldersharing => {
 230         type => 'boolean',
 231         optional => 1,
 232         default => '0',
 233         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 234     },
 235     videostreaming =>  {
 236         type => 'string',
 237         enum => ['off', 'all', 'filter'],
 238         default => 'off',
 239         optional => 1,
 240         description => "Enable video streaming. Uses compression for detected video streams."
 241     },
 242 };
 243
 244 my $rng_fmt = {
 245     source => {
 246         type => 'string',
 247         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 248         default_key => 1,
 249         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 250             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 251             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 252             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 253             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 254             ." a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 259             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 260             ." `0` to disable limiting (potentially dangerous!).",
 261         optional => 1,
 262
 263         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 264         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 265         # reading from /dev/urandom
 266         default => 1024,
 267     },
 268     period => {
 269         type => 'integer',
 270         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 271             ." the guest to retrieve another 'max_bytes' of entropy.",
 272         optional => 1,
 273         default => 1000,
 274     },
 275 };
 276
 277 my $meta_info_fmt = {
 278     'ctime' => {
 279         type => 'integer',
 280         description => "The guest creation timestamp as UNIX epoch time",
 281         minimum => 0,
 282         optional => 1,
 283     },
 284     'creation-qemu' => {
 285         type => 'string',
 286         description => "The QEMU (machine) version from the time this VM was created.",
 287         pattern => '\d+(\.\d+)+',
 288         optional => 1,
 289     },
 290 };
 291
 292 my $confdesc = {
 293     onboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Specifies whether a VM will be started during system bootup.",
 297         default => 0,
 298     },
 299     autostart => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Automatic restart after crash (currently ignored).",
 303         default => 0,
 304     },
 305     hotplug => {
 306         optional => 1,
 307         type => 'string', format => 'pve-hotplug-features',
 308         description => "Selectively enable hotplug features. This is a comma separated list of"
 309             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 310             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 311             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 312             ." windows > 7.",
 313         default => 'network,disk,usb',
 314     },
 315     reboot => {
 316         optional => 1,
 317         type => 'boolean',
 318         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 319         default => 1,
 320     },
 321     lock => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Lock/unlock the VM.",
 325         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 326     },
 327     cpulimit => {
 328         optional => 1,
 329         type => 'number',
 330         description => "Limit of CPU usage.",
 331         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 332             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 333         minimum => 0,
 334         maximum => 128,
 335         default => 0,
 336     },
 337     cpuunits => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 341         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 342             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 343             ." weights of all the other running VMs.",
 344         minimum => 1,
 345         maximum => 262144,
 346         default => 'cgroup v1: 1024, cgroup v2: 100',
 347     },
 348     memory => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 352             ." you use the balloon device.",
 353         minimum => 16,
 354         default => 512,
 355     },
 356     balloon => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 360         minimum => 0,
 361     },
 362     shares => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 366             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 367             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 368         minimum => 0,
 369         maximum => 50000,
 370         default => 1000,
 371     },
 372     keyboard => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Keyboard layout for VNC server. This option is generally not required and"
 376         ." is often better handled from within the guest OS.",
 377         enum => PVE::Tools::kvmkeymaplist(),
 378         default => undef,
 379     },
 380     name => {
 381         optional => 1,
 382         type => 'string', format => 'dns-name',
 383         description => "Set a name for the VM. Only used on the configuration web interface.",
 384     },
 385     scsihw => {
 386         optional => 1,
 387         type => 'string',
 388         description => "SCSI controller model",
 389         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 390         default => 'lsi',
 391     },
 392     description => {
 393         optional => 1,
 394         type => 'string',
 395         description => "Description for the VM. Shown in the web-interface VM's summary."
 396             ." This is saved as comment inside the configuration file.",
 397         maxLength => 1024 * 8,
 398     },
 399     ostype => {
 400         optional => 1,
 401         type => 'string',
 402         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 403         description => "Specify guest operating system.",
 404         verbose_description => <<EODESC,
 405 Specify guest operating system. This is used to enable special
 406 optimization/features for specific operating systems:
 407
 408 [horizontal]
 409 other;; unspecified OS
 410 wxp;; Microsoft Windows XP
 411 w2k;; Microsoft Windows 2000
 412 w2k3;; Microsoft Windows 2003
 413 w2k8;; Microsoft Windows 2008
 414 wvista;; Microsoft Windows Vista
 415 win7;; Microsoft Windows 7
 416 win8;; Microsoft Windows 8/2012/2012r2
 417 win10;; Microsoft Windows 10/2016/2019
 418 win11;; Microsoft Windows 11/2022
 419 l24;; Linux 2.4 Kernel
 420 l26;; Linux 2.6 - 6.X Kernel
 421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 422 EODESC
 423     },
 424     boot => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-boot',
 427         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 428             ." key or 'legacy=' is deprecated.",
 429     },
 430     bootdisk => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-bootdisk',
 433         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 434         pattern => '(ide|sata|scsi|virtio)\d+',
 435     },
 436     smp => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPUs. Please use option -sockets instead.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     sockets => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of CPU sockets.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     cores => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of cores per socket.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     numa => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable NUMA.",
 461         default => 0,
 462     },
 463     hugepages => {
 464         optional => 1,
 465         type => 'string',
 466         description => "Enable/disable hugepages memory.",
 467         enum => [qw(any 2 1024)],
 468     },
 469     keephugepages => {
 470         optional => 1,
 471         type => 'boolean',
 472         default => 0,
 473         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 474             ." after VM shutdown and can be used for subsequent starts.",
 475     },
 476     vcpus => {
 477         optional => 1,
 478         type => 'integer',
 479         description => "Number of hotplugged vcpus.",
 480         minimum => 1,
 481         default => 0,
 482     },
 483     acpi => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Enable/disable ACPI.",
 487         default => 1,
 488     },
 489     agent => {
 490         optional => 1,
 491         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 492         type => 'string',
 493         format => $agent_fmt,
 494     },
 495     kvm => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable KVM hardware virtualization.",
 499         default => 1,
 500     },
 501     tdf => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable time drift fix.",
 505         default => 0,
 506     },
 507     localtime => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 511             ." the `ostype` indicates a Microsoft Windows OS.",
 512     },
 513     freeze => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 517     },
 518     vga => {
 519         optional => 1,
 520         type => 'string', format => $vga_fmt,
 521         description => "Configure the VGA hardware.",
 522         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 523             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 524             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 525             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 526             ." display server. For win* OS you can select how many independent displays you want,"
 527             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 528             ." using a serial device as terminal.",
 529     },
 530     watchdog => {
 531         optional => 1,
 532         type => 'string', format => 'pve-qm-watchdog',
 533         description => "Create a virtual hardware watchdog device.",
 534         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 535             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 536             ." else the watchdog will reset the guest (or execute the respective action specified)",
 537     },
 538     startdate => {
 539         optional => 1,
 540         type => 'string',
 541         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 542         description => "Set the initial date of the real time clock. Valid format for date are:"
 543             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 544         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 545         default => 'now',
 546     },
 547     startup =>  get_standard_option('pve-startup-order'),
 548     template => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Enable/disable Template.",
 552         default => 0,
 553     },
 554     args => {
 555         optional => 1,
 556         type => 'string',
 557         description => "Arbitrary arguments passed to kvm.",
 558         verbose_description => <<EODESCR,
 559 Arbitrary arguments passed to kvm, for example:
 560
 561 args: -no-reboot -smbios 'type=0,vendor=FOO'
 562
 563 NOTE: this option is for experts only.
 564 EODESCR
 565     },
 566     tablet => {
 567         optional => 1,
 568         type => 'boolean',
 569         default => 1,
 570         description => "Enable/disable the USB tablet device.",
 571         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 572             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 573             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 574             ." may consider disabling this to save some context switches. This is turned off by"
 575             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 576     },
 577     migrate_speed => {
 578         optional => 1,
 579         type => 'integer',
 580         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 581         minimum => 0,
 582         default => 0,
 583     },
 584     migrate_downtime => {
 585         optional => 1,
 586         type => 'number',
 587         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 588         minimum => 0,
 589         default => 0.1,
 590     },
 591     cdrom => {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-ide',
 594         typetext => '<volume>',
 595         description => "This is an alias for option -ide2",
 596     },
 597     cpu => {
 598         optional => 1,
 599         description => "Emulated CPU type.",
 600         type => 'string',
 601         format => 'pve-vm-cpu-conf',
 602     },
 603     parent => get_standard_option('pve-snapshot-name', {
 604         optional => 1,
 605         description => "Parent snapshot name. This is used internally, and should not be modified.",
 606     }),
 607     snaptime => {
 608         optional => 1,
 609         description => "Timestamp for snapshots.",
 610         type => 'integer',
 611         minimum => 0,
 612     },
 613     vmstate => {
 614         optional => 1,
 615         type => 'string', format => 'pve-volume-id',
 616         description => "Reference to a volume which stores the VM state. This is used internally"
 617             ." for snapshots.",
 618     },
 619     vmstatestorage => get_standard_option('pve-storage-id', {
 620         description => "Default storage for VM state volumes/files.",
 621         optional => 1,
 622     }),
 623     runningmachine => get_standard_option('pve-qemu-machine', {
 624         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 625             ." for snapshots.",
 626     }),
 627     runningcpu => {
 628         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 629             ." internally for snapshots.",
 630         optional => 1,
 631         type => 'string',
 632         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 633         format_description => 'QEMU -cpu parameter'
 634     },
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 652             ." remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 667             ." to disable explicitly.",
 668         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 669             ." value identifier to the guest OS. This allows to notify the guest operating system"
 670             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 671             ." execution or creation from a template). The guest operating system notices the"
 672             ." change, and is then able to react as appropriate by marking its copies of"
 673             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 674             ."Note that auto-creation only works when done through API/CLI create or update methods"
 675             .", but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 689             ." the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704     tags => {
 705         type => 'string', format => 'pve-tag-list',
 706         description => 'Tags of the VM. This is only meta information.',
 707         optional => 1,
 708     },
 709     rng0 => {
 710         type => 'string',
 711         format => $rng_fmt,
 712         description => "Configure a VirtIO-based Random Number Generator.",
 713         optional => 1,
 714     },
 715     meta => {
 716         type => 'string',
 717         format => $meta_info_fmt,
 718         description => "Some (read-only) meta-information about this guest.",
 719         optional => 1,
 720     },
 721     affinity => {
 722         type => 'string', format => 'pve-cpuset',
 723         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 724         optional => 1,
 725     },
 726 };
 727
 728 my $cicustom_fmt = {
 729     meta => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all meta data passed to the VM via"
 733             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     network => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     user => {
 745         type => 'string',
 746         optional => 1,
 747         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 748         format => 'pve-volume-id',
 749         format_description => 'volume',
 750     },
 751     vendor => {
 752         type => 'string',
 753         optional => 1,
 754         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 760
 761 my $confdesc_cloudinit = {
 762     citype => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'Specifies the cloud-init configuration format. The default depends on the'
 766             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 767             .' and `configdrive2` for windows.',
 768         enum => ['configdrive2', 'nocloud', 'opennebula'],
 769     },
 770     ciuser => {
 771         optional => 1,
 772         type => 'string',
 773         description => "cloud-init: User name to change ssh keys and password for instead of the"
 774             ." image's configured default user.",
 775     },
 776     cipassword => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Password to assign the user. Using this is generally not'
 780             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 781             .' support hashed passwords.',
 782     },
 783     ciupgrade => {
 784         optional => 1,
 785         type => 'boolean',
 786         description => 'cloud-init: do an automatic package upgrade after the first boot.'
 787     },
 788     cicustom => {
 789         optional => 1,
 790         type => 'string',
 791         description => 'cloud-init: Specify custom files to replace the automatically generated'
 792             .' ones at start.',
 793         format => 'pve-qm-cicustom',
 794     },
 795     searchdomain => {
 796         optional => 1,
 797         type => 'string',
 798         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 799             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 800             .' are set.',
 801     },
 802     nameserver => {
 803         optional => 1,
 804         type => 'string', format => 'address-list',
 805         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 806             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 807             .' are set.',
 808     },
 809     sshkeys => {
 810         optional => 1,
 811         type => 'string',
 812         format => 'urlencoded',
 813         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 814     },
 815 };
 816
 817 # what about other qemu settings ?
 818 #cpu => 'string',
 819 #machine => 'string',
 820 #fda => 'file',
 821 #fdb => 'file',
 822 #mtdblock => 'file',
 823 #sd => 'file',
 824 #pflash => 'file',
 825 #snapshot => 'bool',
 826 #bootp => 'file',
 827 ##tftp => 'dir',
 828 ##smb => 'dir',
 829 #kernel => 'file',
 830 #append => 'string',
 831 #initrd => 'file',
 832 ##soundhw => 'string',
 833
 834 while (my ($k, $v) = each %$confdesc) {
 835     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 836 }
 837
 838 my $MAX_USB_DEVICES = 14;
 839 my $MAX_NETS = 32;
 840 my $MAX_SERIAL_PORTS = 4;
 841 my $MAX_PARALLEL_PORTS = 3;
 842 my $MAX_NUMA = 8;
 843
 844 my $numa_fmt = {
 845     cpus => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "CPUs accessing this NUMA node.",
 849         format_description => "id[-id];...",
 850     },
 851     memory => {
 852         type => "number",
 853         description => "Amount of memory this NUMA node provides.",
 854         optional => 1,
 855     },
 856     hostnodes => {
 857         type => "string",
 858         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 859         description => "Host NUMA nodes to use.",
 860         format_description => "id[-id];...",
 861         optional => 1,
 862     },
 863     policy => {
 864         type => 'string',
 865         enum => [qw(preferred bind interleave)],
 866         description => "NUMA allocation policy.",
 867         optional => 1,
 868     },
 869 };
 870 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 871 my $numadesc = {
 872     optional => 1,
 873     type => 'string', format => $numa_fmt,
 874     description => "NUMA topology.",
 875 };
 876 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 877
 878 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 879     $confdesc->{"numa$i"} = $numadesc;
 880 }
 881
 882 my $nic_model_list = [
 883     'e1000',
 884     'e1000-82540em',
 885     'e1000-82544gc',
 886     'e1000-82545em',
 887     'e1000e',
 888     'i82551',
 889     'i82557b',
 890     'i82559er',
 891     'ne2k_isa',
 892     'ne2k_pci',
 893     'pcnet',
 894     'rtl8139',
 895     'virtio',
 896     'vmxnet3',
 897 ];
 898 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 899
 900 my $net_fmt_bridge_descr = <<__EOD__;
 901 Bridge to attach the network device to. The Proxmox VE standard bridge
 902 is called 'vmbr0'.
 903
 904 If you do not specify a bridge, we create a kvm user (NATed) network
 905 device, which provides DHCP and DNS services. The following addresses
 906 are used:
 907
 908  10.0.2.2   Gateway
 909  10.0.2.3   DNS Server
 910  10.0.2.4   SMB Server
 911
 912 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 913 __EOD__
 914
 915 my $net_fmt = {
 916     macaddr  => get_standard_option('mac-addr', {
 917         description => "MAC address. That address must be unique withing your network. This is"
 918             ." automatically generated if not specified.",
 919     }),
 920     model => {
 921         type => 'string',
 922         description => "Network Card Model. The 'virtio' model provides the best performance with"
 923             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 924             ." best to use 'e1000'.",
 925         enum => $nic_model_list,
 926         default_key => 1,
 927     },
 928     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 929     bridge => get_standard_option('pve-bridge-id', {
 930         description => $net_fmt_bridge_descr,
 931         optional => 1,
 932     }),
 933     queues => {
 934         type => 'integer',
 935         minimum => 0, maximum => 64,
 936         description => 'Number of packet queues to be used on the device.',
 937         optional => 1,
 938     },
 939     rate => {
 940         type => 'number',
 941         minimum => 0,
 942         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 943         optional => 1,
 944     },
 945     tag => {
 946         type => 'integer',
 947         minimum => 1, maximum => 4094,
 948         description => 'VLAN tag to apply to packets on this interface.',
 949         optional => 1,
 950     },
 951     trunks => {
 952         type => 'string',
 953         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 954         description => 'VLAN trunks to pass through this interface.',
 955         format_description => 'vlanid[;vlanid...]',
 956         optional => 1,
 957     },
 958     firewall => {
 959         type => 'boolean',
 960         description => 'Whether this interface should be protected by the firewall.',
 961         optional => 1,
 962     },
 963     link_down => {
 964         type => 'boolean',
 965         description => 'Whether this interface should be disconnected (like pulling the plug).',
 966         optional => 1,
 967     },
 968     mtu => {
 969         type => 'integer',
 970         minimum => 1, maximum => 65520,
 971         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 972         optional => 1,
 973     },
 974 };
 975
 976 my $netdesc = {
 977     optional => 1,
 978     type => 'string', format => $net_fmt,
 979     description => "Specify network devices.",
 980 };
 981
 982 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 983
 984 my $ipconfig_fmt = {
 985     ip => {
 986         type => 'string',
 987         format => 'pve-ipv4-config',
 988         format_description => 'IPv4Format/CIDR',
 989         description => 'IPv4 address in CIDR format.',
 990         optional => 1,
 991         default => 'dhcp',
 992     },
 993     gw => {
 994         type => 'string',
 995         format => 'ipv4',
 996         format_description => 'GatewayIPv4',
 997         description => 'Default gateway for IPv4 traffic.',
 998         optional => 1,
 999         requires => 'ip',
1000     },
1001     ip6 => {
1002         type => 'string',
1003         format => 'pve-ipv6-config',
1004         format_description => 'IPv6Format/CIDR',
1005         description => 'IPv6 address in CIDR format.',
1006         optional => 1,
1007         default => 'dhcp',
1008     },
1009     gw6 => {
1010         type => 'string',
1011         format => 'ipv6',
1012         format_description => 'GatewayIPv6',
1013         description => 'Default gateway for IPv6 traffic.',
1014         optional => 1,
1015         requires => 'ip6',
1016     },
1017 };
1018 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1019 my $ipconfigdesc = {
1020     optional => 1,
1021     type => 'string', format => 'pve-qm-ipconfig',
1022     description => <<'EODESCR',
1023 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1024
1025 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1026
1027 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1028 gateway should be provided.
1029 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1030 cloud-init 19.4 or newer.
1031
1032 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1033 dhcp on IPv4.
1034 EODESCR
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1037
1038 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1039     $confdesc->{"net$i"} = $netdesc;
1040     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1041 }
1042
1043 foreach my $key (keys %$confdesc_cloudinit) {
1044     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1045 }
1046
1047 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1048 sub pve_verify_cpuset {
1049     my ($set_text, $noerr) = @_;
1050
1051     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1052
1053     if ($@) {
1054         return if $noerr;
1055         die "unable to parse cpuset option\n";
1056     }
1057
1058     return PVE::CpuSet->new($members)->short_string();
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1062 sub verify_volume_id_or_qm_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1066
1067     return verify_volume_id_or_absolute_path($volid, $noerr);
1068 }
1069
1070 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1071 sub verify_volume_id_or_absolute_path {
1072     my ($volid, $noerr) = @_;
1073
1074     return $volid if $volid =~ m|^/|;
1075
1076     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1077     if ($@) {
1078         return if $noerr;
1079         die $@;
1080     }
1081     return $volid;
1082 }
1083
1084 my $usb_fmt = {
1085     host => {
1086         default_key => 1,
1087         type => 'string', format => 'pve-qm-usb-device',
1088         format_description => 'HOSTUSBDEVICE|spice',
1089         description => <<EODESCR,
1090 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1091
1092  'bus-port(.port)*' (decimal numbers) or
1093  'vendor_id:product_id' (hexadeciaml numbers) or
1094  'spice'
1095
1096 You can use the 'lsusb -t' command to list existing usb devices.
1097
1098 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1099 machines - use with special care.
1100
1101 The value 'spice' can be used to add a usb redirection devices for spice.
1102 EODESCR
1103     },
1104     usb3 => {
1105         optional => 1,
1106         type => 'boolean',
1107         description => "Specifies whether if given host option is a USB3 device or port."
1108             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1109             ." is irrelevant (all devices are plugged into a xhci controller).",
1110         default => 0,
1111     },
1112 };
1113
1114 my $usbdesc = {
1115     optional => 1,
1116     type => 'string', format => $usb_fmt,
1117     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1118         ." l26 or windows > 7, n can be up to 14).",
1119 };
1120 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1121
1122 my $serialdesc = {
1123         optional => 1,
1124         type => 'string',
1125         pattern => '(/dev/.+|socket)',
1126         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1127         verbose_description =>  <<EODESCR,
1128 Create a serial device inside the VM (n is 0 to 3), and pass through a
1129 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1130 host side (use 'qm terminal' to open a terminal connection).
1131
1132 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1133 use with special care.
1134
1135 CAUTION: Experimental! User reported problems with this option.
1136 EODESCR
1137 };
1138
1139 my $paralleldesc= {
1140         optional => 1,
1141         type => 'string',
1142         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1143         description =>  "Map host parallel devices (n is 0 to 2).",
1144         verbose_description =>  <<EODESCR,
1145 Map host parallel devices (n is 0 to 2).
1146
1147 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1148 machines - use with special care.
1149
1150 CAUTION: Experimental! User reported problems with this option.
1151 EODESCR
1152 };
1153
1154 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1155     $confdesc->{"parallel$i"} = $paralleldesc;
1156 }
1157
1158 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1159     $confdesc->{"serial$i"} = $serialdesc;
1160 }
1161
1162 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1163     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1164 }
1165
1166 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1167     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1168 }
1169
1170 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1171     $confdesc->{"usb$i"} = $usbdesc;
1172 }
1173
1174 my $boot_fmt = {
1175     legacy => {
1176         optional => 1,
1177         default_key => 1,
1178         type => 'string',
1179         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1180                      . " Deprecated, use 'order=' instead.",
1181         pattern => '[acdn]{1,4}',
1182         format_description => "[acdn]{1,4}",
1183
1184         # note: this is also the fallback if boot: is not given at all
1185         default => 'cdn',
1186     },
1187     order => {
1188         optional => 1,
1189         type => 'string',
1190         format => 'pve-qm-bootdev-list',
1191         format_description => "device[;device...]",
1192         description => <<EODESC,
1193 The guest will attempt to boot from devices in the order they appear here.
1194
1195 Disks, optical drives and passed-through storage USB devices will be directly
1196 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1197 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1198
1199 Note that only devices in this list will be marked as bootable and thus loaded
1200 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1201 (e.g. software-raid), you need to specify all of them here.
1202
1203 Overrides the deprecated 'legacy=[acdn]*' value when given.
1204 EODESC
1205     },
1206 };
1207 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1208
1209 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1210 sub verify_bootdev {
1211     my ($dev, $noerr) = @_;
1212
1213     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1214     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1215
1216     my $check = sub {
1217         my ($base) = @_;
1218         return 0 if $dev !~ m/^$base\d+$/;
1219         return 0 if !$confdesc->{$dev};
1220         return 1;
1221     };
1222
1223     return $dev if $check->("net");
1224     return $dev if $check->("usb");
1225     return $dev if $check->("hostpci");
1226
1227     return if $noerr;
1228     die "invalid boot device '$dev'\n";
1229 }
1230
1231 sub print_bootorder {
1232     my ($devs) = @_;
1233     return "" if !@$devs;
1234     my $data = { order => join(';', @$devs) };
1235     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1236 }
1237
1238 my $kvm_api_version = 0;
1239
1240 sub kvm_version {
1241     return $kvm_api_version if $kvm_api_version;
1242
1243     open my $fh, '<', '/dev/kvm' or return;
1244
1245     # 0xae00 => KVM_GET_API_VERSION
1246     $kvm_api_version = ioctl($fh, 0xae00, 0);
1247     close($fh);
1248
1249     return $kvm_api_version;
1250 }
1251
1252 my $kvm_user_version = {};
1253 my $kvm_mtime = {};
1254
1255 sub kvm_user_version {
1256     my ($binary) = @_;
1257
1258     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1259     my $st = stat($binary);
1260
1261     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1262     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1263         $cachedmtime == $st->mtime;
1264
1265     $kvm_user_version->{$binary} = 'unknown';
1266     $kvm_mtime->{$binary} = $st->mtime;
1267
1268     my $code = sub {
1269         my $line = shift;
1270         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1271             $kvm_user_version->{$binary} = $2;
1272         }
1273     };
1274
1275     eval { run_command([$binary, '--version'], outfunc => $code); };
1276     warn $@ if $@;
1277
1278     return $kvm_user_version->{$binary};
1279
1280 }
1281 my sub extract_version {
1282     my ($machine_type, $version) = @_;
1283     $version = kvm_user_version() if !defined($version);
1284     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1285 }
1286
1287 sub kernel_has_vhost_net {
1288     return -c '/dev/vhost-net';
1289 }
1290
1291 sub option_exists {
1292     my $key = shift;
1293     return defined($confdesc->{$key});
1294 }
1295
1296 my $cdrom_path;
1297 sub get_cdrom_path {
1298
1299     return $cdrom_path if defined($cdrom_path);
1300
1301     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1302
1303     if (!defined($cdrom_path)) {
1304         log_warn("no physical CD-ROM available, ignoring");
1305         $cdrom_path = '';
1306     }
1307
1308     return $cdrom_path;
1309 }
1310
1311 sub get_iso_path {
1312     my ($storecfg, $vmid, $cdrom) = @_;
1313
1314     if ($cdrom eq 'cdrom') {
1315         return get_cdrom_path();
1316     } elsif ($cdrom eq 'none') {
1317         return '';
1318     } elsif ($cdrom =~ m|^/|) {
1319         return $cdrom;
1320     } else {
1321         return PVE::Storage::path($storecfg, $cdrom);
1322     }
1323 }
1324
1325 # try to convert old style file names to volume IDs
1326 sub filename_to_volume_id {
1327     my ($vmid, $file, $media) = @_;
1328
1329      if (!($file eq 'none' || $file eq 'cdrom' ||
1330           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1331
1332         return if $file =~ m|/|;
1333
1334         if ($media && $media eq 'cdrom') {
1335             $file = "local:iso/$file";
1336         } else {
1337             $file = "local:$vmid/$file";
1338         }
1339     }
1340
1341     return $file;
1342 }
1343
1344 sub verify_media_type {
1345     my ($opt, $vtype, $media) = @_;
1346
1347     return if !$media;
1348
1349     my $etype;
1350     if ($media eq 'disk') {
1351         $etype = 'images';
1352     } elsif ($media eq 'cdrom') {
1353         $etype = 'iso';
1354     } else {
1355         die "internal error";
1356     }
1357
1358     return if ($vtype eq $etype);
1359
1360     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1361 }
1362
1363 sub cleanup_drive_path {
1364     my ($opt, $storecfg, $drive) = @_;
1365
1366     # try to convert filesystem paths to volume IDs
1367
1368     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1369         ($drive->{file} !~ m|^/dev/.+|) &&
1370         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1371         ($drive->{file} !~ m/^\d+$/)) {
1372         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1373         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1374             if !$vtype;
1375         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1376         verify_media_type($opt, $vtype, $drive->{media});
1377         $drive->{file} = $volid;
1378     }
1379
1380     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1381 }
1382
1383 sub parse_hotplug_features {
1384     my ($data) = @_;
1385
1386     my $res = {};
1387
1388     return $res if $data eq '0';
1389
1390     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1391
1392     foreach my $feature (PVE::Tools::split_list($data)) {
1393         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1394             $res->{$1} = 1;
1395         } else {
1396             die "invalid hotplug feature '$feature'\n";
1397         }
1398     }
1399     return $res;
1400 }
1401
1402 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1403 sub pve_verify_hotplug_features {
1404     my ($value, $noerr) = @_;
1405
1406     return $value if parse_hotplug_features($value);
1407
1408     return if $noerr;
1409
1410     die "unable to parse hotplug option\n";
1411 }
1412
1413 sub scsi_inquiry {
1414     my($fh, $noerr) = @_;
1415
1416     my $SG_IO = 0x2285;
1417     my $SG_GET_VERSION_NUM = 0x2282;
1418
1419     my $versionbuf = "\x00" x 8;
1420     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1421     if (!$ret) {
1422         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1423         return;
1424     }
1425     my $version = unpack("I", $versionbuf);
1426     if ($version < 30000) {
1427         die "scsi generic interface too old\n"  if !$noerr;
1428         return;
1429     }
1430
1431     my $buf = "\x00" x 36;
1432     my $sensebuf = "\x00" x 8;
1433     my $cmd = pack("C x3 C x1", 0x12, 36);
1434
1435     # see /usr/include/scsi/sg.h
1436     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1437
1438     my $packet = pack(
1439         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1440     );
1441
1442     $ret = ioctl($fh, $SG_IO, $packet);
1443     if (!$ret) {
1444         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1445         return;
1446     }
1447
1448     my @res = unpack($sg_io_hdr_t, $packet);
1449     if ($res[17] || $res[18]) {
1450         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1451         return;
1452     }
1453
1454     my $res = {};
1455     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1456
1457     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1458     $res->{type} &= 0x1F;
1459
1460     return $res;
1461 }
1462
1463 sub path_is_scsi {
1464     my ($path) = @_;
1465
1466     my $fh = IO::File->new("+<$path") || return;
1467     my $res = scsi_inquiry($fh, 1);
1468     close($fh);
1469
1470     return $res;
1471 }
1472
1473 sub print_tabletdevice_full {
1474     my ($conf, $arch) = @_;
1475
1476     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1477
1478     # we use uhci for old VMs because tablet driver was buggy in older qemu
1479     my $usbbus;
1480     if ($q35 || $arch eq 'aarch64') {
1481         $usbbus = 'ehci';
1482     } else {
1483         $usbbus = 'uhci';
1484     }
1485
1486     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1487 }
1488
1489 sub print_keyboarddevice_full {
1490     my ($conf, $arch) = @_;
1491
1492     return if $arch ne 'aarch64';
1493
1494     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1495 }
1496
1497 my sub get_drive_id {
1498     my ($drive) = @_;
1499     return "$drive->{interface}$drive->{index}";
1500 }
1501
1502 sub print_drivedevice_full {
1503     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1504
1505     my $device = '';
1506     my $maxdev = 0;
1507
1508     my $drive_id = get_drive_id($drive);
1509     if ($drive->{interface} eq 'virtio') {
1510         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1511         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1512         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1513     } elsif ($drive->{interface} eq 'scsi') {
1514
1515         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1516         my $unit = $drive->{index} % $maxdev;
1517         my $devicetype = 'hd';
1518         my $path = '';
1519         if (drive_is_cdrom($drive)) {
1520             $devicetype = 'cd';
1521         } else {
1522             if ($drive->{file} =~ m|^/|) {
1523                 $path = $drive->{file};
1524                 if (my $info = path_is_scsi($path)) {
1525                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1526                         $devicetype = 'block';
1527                     } elsif ($info->{type} == 1) { # tape
1528                         $devicetype = 'generic';
1529                     }
1530                 }
1531             } else {
1532                  $path = PVE::Storage::path($storecfg, $drive->{file});
1533             }
1534
1535             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1536             my $version = extract_version($machine_type, kvm_user_version());
1537             if ($path =~ m/^iscsi\:\/\// &&
1538                !min_version($version, 4, 1)) {
1539                 $devicetype = 'generic';
1540             }
1541         }
1542
1543         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1544             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1545         } else {
1546             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1547                 .",lun=$drive->{index}";
1548         }
1549         $device .= ",drive=drive-$drive_id,id=$drive_id";
1550
1551         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1552             $device .= ",rotation_rate=1";
1553         }
1554         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1555
1556     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1557         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1558         my $controller = int($drive->{index} / $maxdev);
1559         my $unit = $drive->{index} % $maxdev;
1560         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1561
1562         $device = "ide-$devicetype";
1563         if ($drive->{interface} eq 'ide') {
1564             $device .= ",bus=ide.$controller,unit=$unit";
1565         } else {
1566             $device .= ",bus=ahci$controller.$unit";
1567         }
1568         $device .= ",drive=drive-$drive_id,id=$drive_id";
1569
1570         if ($devicetype eq 'hd') {
1571             if (my $model = $drive->{model}) {
1572                 $model = URI::Escape::uri_unescape($model);
1573                 $device .= ",model=$model";
1574             }
1575             if ($drive->{ssd}) {
1576                 $device .= ",rotation_rate=1";
1577             }
1578         }
1579         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1580     } elsif ($drive->{interface} eq 'usb') {
1581         die "implement me";
1582         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1583     } else {
1584         die "unsupported interface type";
1585     }
1586
1587     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1588
1589     if (my $serial = $drive->{serial}) {
1590         $serial = URI::Escape::uri_unescape($serial);
1591         $device .= ",serial=$serial";
1592     }
1593
1594
1595     return $device;
1596 }
1597
1598 sub get_initiator_name {
1599     my $initiator;
1600
1601     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1602     while (defined(my $line = <$fh>)) {
1603         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1604         $initiator = $1;
1605         last;
1606     }
1607     $fh->close();
1608
1609     return $initiator;
1610 }
1611
1612 my sub storage_allows_io_uring_default {
1613     my ($scfg, $cache_direct) = @_;
1614
1615     # io_uring with cache mode writeback or writethrough on krbd will hang...
1616     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1617
1618     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1619     # sometimes, just plain disable...
1620     return if $scfg && $scfg->{type} eq 'lvm';
1621
1622     # io_uring causes problems when used with CIFS since kernel 5.15
1623     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1624     return if $scfg && $scfg->{type} eq 'cifs';
1625
1626     return 1;
1627 }
1628
1629 my sub drive_uses_cache_direct {
1630     my ($drive, $scfg) = @_;
1631
1632     my $cache_direct = 0;
1633
1634     if (my $cache = $drive->{cache}) {
1635         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1636     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1637         $cache_direct = 1;
1638     }
1639
1640     return $cache_direct;
1641 }
1642
1643 sub print_drive_commandline_full {
1644     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1645
1646     my $path;
1647     my $volid = $drive->{file};
1648     my $format = $drive->{format};
1649     my $drive_id = get_drive_id($drive);
1650
1651     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1652     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1653
1654     if (drive_is_cdrom($drive)) {
1655         $path = get_iso_path($storecfg, $vmid, $volid);
1656         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1657     } else {
1658         if ($storeid) {
1659             $path = PVE::Storage::path($storecfg, $volid);
1660             $format //= qemu_img_format($scfg, $volname);
1661         } else {
1662             $path = $volid;
1663             $format //= "raw";
1664         }
1665    }
1666
1667    my $is_rbd = $path =~ m/^rbd:/;
1668
1669     my $opts = '';
1670     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1671     foreach my $o (@qemu_drive_options) {
1672         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1673     }
1674
1675     # snapshot only accepts on|off
1676     if (defined($drive->{snapshot})) {
1677         my $v = $drive->{snapshot} ? 'on' : 'off';
1678         $opts .= ",snapshot=$v";
1679     }
1680
1681     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1682         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1683     }
1684
1685     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1686         my ($dir, $qmpname) = @$type;
1687         if (my $v = $drive->{"mbps$dir"}) {
1688             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1689         }
1690         if (my $v = $drive->{"mbps${dir}_max"}) {
1691             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1692         }
1693         if (my $v = $drive->{"bps${dir}_max_length"}) {
1694             $opts .= ",throttling.bps$qmpname-max-length=$v";
1695         }
1696         if (my $v = $drive->{"iops${dir}"}) {
1697             $opts .= ",throttling.iops$qmpname=$v";
1698         }
1699         if (my $v = $drive->{"iops${dir}_max"}) {
1700             $opts .= ",throttling.iops$qmpname-max=$v";
1701         }
1702         if (my $v = $drive->{"iops${dir}_max_length"}) {
1703             $opts .= ",throttling.iops$qmpname-max-length=$v";
1704         }
1705     }
1706
1707     if ($pbs_name) {
1708         $format = "rbd" if $is_rbd;
1709         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1710             if !$format;
1711         $opts .= ",format=alloc-track,file.driver=$format";
1712     } elsif ($format) {
1713         $opts .= ",format=$format";
1714     }
1715
1716     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1717
1718     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1719
1720     if (!$drive->{aio}) {
1721         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1722             # io_uring supports all cache modes
1723             $opts .= ",aio=io_uring";
1724         } else {
1725             # aio native works only with O_DIRECT
1726             if($cache_direct) {
1727                 $opts .= ",aio=native";
1728             } else {
1729                 $opts .= ",aio=threads";
1730             }
1731         }
1732     }
1733
1734     if (!drive_is_cdrom($drive)) {
1735         my $detectzeroes;
1736         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1737             $detectzeroes = 'off';
1738         } elsif ($drive->{discard}) {
1739             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1740         } else {
1741             # This used to be our default with discard not being specified:
1742             $detectzeroes = 'on';
1743         }
1744
1745         # note: 'detect-zeroes' works per blockdev and we want it to persist
1746         # after the alloc-track is removed, so put it on 'file' directly
1747         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1748         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1749     }
1750
1751     if ($pbs_name) {
1752         $opts .= ",backing=$pbs_name";
1753         $opts .= ",auto-remove=on";
1754     }
1755
1756     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1757     my $file_param = "file";
1758     if ($pbs_name) {
1759         # non-rbd drivers require the underlying file to be a seperate block
1760         # node, so add a second .file indirection
1761         $file_param .= ".file" if !$is_rbd;
1762         $file_param .= ".filename";
1763     }
1764     my $pathinfo = $path ? "$file_param=$path," : '';
1765
1766     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1767 }
1768
1769 sub print_pbs_blockdev {
1770     my ($pbs_conf, $pbs_name) = @_;
1771     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1772     $blockdev .= ",repository=$pbs_conf->{repository}";
1773     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1774     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1775     $blockdev .= ",archive=$pbs_conf->{archive}";
1776     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1777     return $blockdev;
1778 }
1779
1780 sub print_netdevice_full {
1781     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1782
1783     my $device = $net->{model};
1784     if ($net->{model} eq 'virtio') {
1785          $device = 'virtio-net-pci';
1786      };
1787
1788     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1789     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1790     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1791         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1792         # and out of each queue plus one config interrupt and control vector queue
1793         my $vectors = $net->{queues} * 2 + 2;
1794         $tmpstr .= ",vectors=$vectors,mq=on";
1795         if (min_version($machine_version, 7, 1)) {
1796             $tmpstr .= ",packed=on";
1797         }
1798     }
1799
1800     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1801         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1802     }
1803
1804     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1805
1806     if (my $mtu = $net->{mtu}) {
1807         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1808             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1809             if ($mtu == 1) {
1810                  $mtu = $bridge_mtu;
1811             } elsif ($mtu < 576) {
1812                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1813             } elsif ($mtu > $bridge_mtu) {
1814                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1815             }
1816             $tmpstr .= ",host_mtu=$mtu";
1817         } else {
1818             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1819         }
1820     }
1821
1822     if ($use_old_bios_files) {
1823         my $romfile;
1824         if ($device eq 'virtio-net-pci') {
1825             $romfile = 'pxe-virtio.rom';
1826         } elsif ($device eq 'e1000') {
1827             $romfile = 'pxe-e1000.rom';
1828         } elsif ($device eq 'e1000e') {
1829             $romfile = 'pxe-e1000e.rom';
1830         } elsif ($device eq 'ne2k') {
1831             $romfile = 'pxe-ne2k_pci.rom';
1832         } elsif ($device eq 'pcnet') {
1833             $romfile = 'pxe-pcnet.rom';
1834         } elsif ($device eq 'rtl8139') {
1835             $romfile = 'pxe-rtl8139.rom';
1836         }
1837         $tmpstr .= ",romfile=$romfile" if $romfile;
1838     }
1839
1840     return $tmpstr;
1841 }
1842
1843 sub print_netdev_full {
1844     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1845
1846     my $i = '';
1847     if ($netid =~ m/^net(\d+)$/) {
1848         $i = int($1);
1849     }
1850
1851     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1852
1853     my $ifname = "tap${vmid}i$i";
1854
1855     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1856     die "interface name '$ifname' is too long (max 15 character)\n"
1857         if length($ifname) >= 16;
1858
1859     my $vhostparam = '';
1860     if (is_native($arch)) {
1861         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1862     }
1863
1864     my $vmname = $conf->{name} || "vm$vmid";
1865
1866     my $netdev = "";
1867     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1868
1869     if ($net->{bridge}) {
1870         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1871             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1872     } else {
1873         $netdev = "type=user,id=$netid,hostname=$vmname";
1874     }
1875
1876     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1877
1878     return $netdev;
1879 }
1880
1881 my $vga_map = {
1882     'cirrus' => 'cirrus-vga',
1883     'std' => 'VGA',
1884     'vmware' => 'vmware-svga',
1885     'virtio' => 'virtio-vga',
1886     'virtio-gl' => 'virtio-vga-gl',
1887 };
1888
1889 sub print_vga_device {
1890     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1891
1892     my $type = $vga_map->{$vga->{type}};
1893     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1894         $type = 'virtio-gpu';
1895     }
1896     my $vgamem_mb = $vga->{memory};
1897
1898     my $max_outputs = '';
1899     if ($qxlnum) {
1900         $type = $id ? 'qxl' : 'qxl-vga';
1901
1902         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1903             # set max outputs so linux can have up to 4 qxl displays with one device
1904             if (min_version($machine_version, 4, 1)) {
1905                 $max_outputs = ",max_outputs=4";
1906             }
1907         }
1908     }
1909
1910     die "no devicetype for $vga->{type}\n" if !$type;
1911
1912     my $memory = "";
1913     if ($vgamem_mb) {
1914         if ($vga->{type} =~ /^virtio/) {
1915             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1916             $memory = ",max_hostmem=$bytes";
1917         } elsif ($qxlnum) {
1918             # from https://www.spice-space.org/multiple-monitors.html
1919             $memory = ",vgamem_mb=$vga->{memory}";
1920             my $ram = $vgamem_mb * 4;
1921             my $vram = $vgamem_mb * 2;
1922             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1923         } else {
1924             $memory = ",vgamem_mb=$vga->{memory}";
1925         }
1926     } elsif ($qxlnum && $id) {
1927         $memory = ",ram_size=67108864,vram_size=33554432";
1928     }
1929
1930     my $edidoff = "";
1931     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1932         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1933     }
1934
1935     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1936     my $vgaid = "vga" . ($id // '');
1937     my $pciaddr;
1938     if ($q35 && $vgaid eq 'vga') {
1939         # the first display uses pcie.0 bus on q35 machines
1940         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1941     } else {
1942         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1943     }
1944
1945     if ($vga->{type} eq 'virtio-gl') {
1946         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1947         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1948             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1949
1950         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1951             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1952     }
1953
1954     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1955 }
1956
1957 sub parse_number_sets {
1958     my ($set) = @_;
1959     my $res = [];
1960     foreach my $part (split(/;/, $set)) {
1961         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1962             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1963             push @$res, [ $1, $2 ];
1964         } else {
1965             die "invalid range: $part\n";
1966         }
1967     }
1968     return $res;
1969 }
1970
1971 sub parse_numa {
1972     my ($data) = @_;
1973
1974     my $res = parse_property_string($numa_fmt, $data);
1975     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1976     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1977     return $res;
1978 }
1979
1980 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1981 sub parse_net {
1982     my ($data, $disable_mac_autogen) = @_;
1983
1984     my $res = eval { parse_property_string($net_fmt, $data) };
1985     if ($@) {
1986         warn $@;
1987         return;
1988     }
1989     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1990         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1991         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1992     }
1993     return $res;
1994 }
1995
1996 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1997 sub parse_ipconfig {
1998     my ($data) = @_;
1999
2000     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
2001     if ($@) {
2002         warn $@;
2003         return;
2004     }
2005
2006     if ($res->{gw} && !$res->{ip}) {
2007         warn 'gateway specified without specifying an IP address';
2008         return;
2009     }
2010     if ($res->{gw6} && !$res->{ip6}) {
2011         warn 'IPv6 gateway specified without specifying an IPv6 address';
2012         return;
2013     }
2014     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2015         warn 'gateway specified together with DHCP';
2016         return;
2017     }
2018     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2019         # gw6 + auto/dhcp
2020         warn "IPv6 gateway specified together with $res->{ip6} address";
2021         return;
2022     }
2023
2024     if (!$res->{ip} && !$res->{ip6}) {
2025         return { ip => 'dhcp', ip6 => 'dhcp' };
2026     }
2027
2028     return $res;
2029 }
2030
2031 sub print_net {
2032     my $net = shift;
2033
2034     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2035 }
2036
2037 sub add_random_macs {
2038     my ($settings) = @_;
2039
2040     foreach my $opt (keys %$settings) {
2041         next if $opt !~ m/^net(\d+)$/;
2042         my $net = parse_net($settings->{$opt});
2043         next if !$net;
2044         $settings->{$opt} = print_net($net);
2045     }
2046 }
2047
2048 sub vm_is_volid_owner {
2049     my ($storecfg, $vmid, $volid) = @_;
2050
2051     if ($volid !~  m|^/|) {
2052         my ($path, $owner);
2053         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2054         if ($owner && ($owner == $vmid)) {
2055             return 1;
2056         }
2057     }
2058
2059     return;
2060 }
2061
2062 sub vmconfig_register_unused_drive {
2063     my ($storecfg, $vmid, $conf, $drive) = @_;
2064
2065     if (drive_is_cloudinit($drive)) {
2066         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2067         warn $@ if $@;
2068         delete $conf->{cloudinit};
2069     } elsif (!drive_is_cdrom($drive)) {
2070         my $volid = $drive->{file};
2071         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2072             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2073         }
2074     }
2075 }
2076
2077 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2078 my $smbios1_fmt = {
2079     uuid => {
2080         type => 'string',
2081         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2082         format_description => 'UUID',
2083         description => "Set SMBIOS1 UUID.",
2084         optional => 1,
2085     },
2086     version => {
2087         type => 'string',
2088         pattern => '[A-Za-z0-9+\/]+={0,2}',
2089         format_description => 'Base64 encoded string',
2090         description => "Set SMBIOS1 version.",
2091         optional => 1,
2092     },
2093     serial => {
2094         type => 'string',
2095         pattern => '[A-Za-z0-9+\/]+={0,2}',
2096         format_description => 'Base64 encoded string',
2097         description => "Set SMBIOS1 serial number.",
2098         optional => 1,
2099     },
2100     manufacturer => {
2101         type => 'string',
2102         pattern => '[A-Za-z0-9+\/]+={0,2}',
2103         format_description => 'Base64 encoded string',
2104         description => "Set SMBIOS1 manufacturer.",
2105         optional => 1,
2106     },
2107     product => {
2108         type => 'string',
2109         pattern => '[A-Za-z0-9+\/]+={0,2}',
2110         format_description => 'Base64 encoded string',
2111         description => "Set SMBIOS1 product ID.",
2112         optional => 1,
2113     },
2114     sku => {
2115         type => 'string',
2116         pattern => '[A-Za-z0-9+\/]+={0,2}',
2117         format_description => 'Base64 encoded string',
2118         description => "Set SMBIOS1 SKU string.",
2119         optional => 1,
2120     },
2121     family => {
2122         type => 'string',
2123         pattern => '[A-Za-z0-9+\/]+={0,2}',
2124         format_description => 'Base64 encoded string',
2125         description => "Set SMBIOS1 family string.",
2126         optional => 1,
2127     },
2128     base64 => {
2129         type => 'boolean',
2130         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2131         optional => 1,
2132     },
2133 };
2134
2135 sub parse_smbios1 {
2136     my ($data) = @_;
2137
2138     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2139     warn $@ if $@;
2140     return $res;
2141 }
2142
2143 sub print_smbios1 {
2144     my ($smbios1) = @_;
2145     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2146 }
2147
2148 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2149
2150 sub parse_watchdog {
2151     my ($value) = @_;
2152
2153     return if !$value;
2154
2155     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2156     warn $@ if $@;
2157     return $res;
2158 }
2159
2160 sub parse_guest_agent {
2161     my ($conf) = @_;
2162
2163     return {} if !defined($conf->{agent});
2164
2165     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2166     warn $@ if $@;
2167
2168     # if the agent is disabled ignore the other potentially set properties
2169     return {} if !$res->{enabled};
2170     return $res;
2171 }
2172
2173 sub get_qga_key {
2174     my ($conf, $key) = @_;
2175     return undef if !defined($conf->{agent});
2176
2177     my $agent = parse_guest_agent($conf);
2178     return $agent->{$key};
2179 }
2180
2181 sub parse_vga {
2182     my ($value) = @_;
2183
2184     return {} if !$value;
2185     my $res = eval { parse_property_string($vga_fmt, $value) };
2186     warn $@ if $@;
2187     return $res;
2188 }
2189
2190 sub parse_rng {
2191     my ($value) = @_;
2192
2193     return if !$value;
2194
2195     my $res = eval { parse_property_string($rng_fmt, $value) };
2196     warn $@ if $@;
2197     return $res;
2198 }
2199
2200 sub parse_meta_info {
2201     my ($value) = @_;
2202
2203     return if !$value;
2204
2205     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2206     warn $@ if $@;
2207     return $res;
2208 }
2209
2210 sub new_meta_info_string {
2211     my () = @_; # for now do not allow to override any value
2212
2213     return PVE::JSONSchema::print_property_string(
2214         {
2215             'creation-qemu' => kvm_user_version(),
2216             ctime => "". int(time()),
2217         },
2218         $meta_info_fmt
2219     );
2220 }
2221
2222 sub qemu_created_version_fixups {
2223     my ($conf, $forcemachine, $kvmver) = @_;
2224
2225     my $meta = parse_meta_info($conf->{meta}) // {};
2226     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2227
2228     # check if we need to apply some handling for VMs that always use the latest machine version but
2229     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2230     # would be required (we do not want to pin machine version for non-windows OS type)
2231     if (
2232         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2233         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2234         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2235         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2236     ) {
2237         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2238         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2239             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2240             # and thus with the predictable interface naming of systemd
2241             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2242         }
2243     }
2244     return;
2245 }
2246
2247 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2248 sub verify_usb_device {
2249     my ($value, $noerr) = @_;
2250
2251     return $value if parse_usb_device($value);
2252
2253     return if $noerr;
2254
2255     die "unable to parse usb device\n";
2256 }
2257
2258 # add JSON properties for create and set function
2259 sub json_config_properties {
2260     my ($prop, $with_disk_alloc) = @_;
2261
2262     my $skip_json_config_opts = {
2263         parent => 1,
2264         snaptime => 1,
2265         vmstate => 1,
2266         runningmachine => 1,
2267         runningcpu => 1,
2268         meta => 1,
2269     };
2270
2271     foreach my $opt (keys %$confdesc) {
2272         next if $skip_json_config_opts->{$opt};
2273
2274         if ($with_disk_alloc && is_valid_drivename($opt)) {
2275             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2276         } else {
2277             $prop->{$opt} = $confdesc->{$opt};
2278         }
2279     }
2280
2281     return $prop;
2282 }
2283
2284 # Properties that we can read from an OVF file
2285 sub json_ovf_properties {
2286     my $prop = {};
2287
2288     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2289         $prop->{$device} = {
2290             type => 'string',
2291             format => 'pve-volume-id-or-absolute-path',
2292             description => "Disk image that gets imported to $device",
2293             optional => 1,
2294         };
2295     }
2296
2297     $prop->{cores} = {
2298         type => 'integer',
2299         description => "The number of CPU cores.",
2300         optional => 1,
2301     };
2302     $prop->{memory} = {
2303         type => 'integer',
2304         description => "Amount of RAM for the VM in MB.",
2305         optional => 1,
2306     };
2307     $prop->{name} = {
2308         type => 'string',
2309         description => "Name of the VM.",
2310         optional => 1,
2311     };
2312
2313     return $prop;
2314 }
2315
2316 # return copy of $confdesc_cloudinit to generate documentation
2317 sub cloudinit_config_properties {
2318
2319     return dclone($confdesc_cloudinit);
2320 }
2321
2322 sub cloudinit_pending_properties {
2323     my $p = {
2324         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2325         name => 1,
2326     };
2327     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2328     return $p;
2329 }
2330
2331 sub check_type {
2332     my ($key, $value) = @_;
2333
2334     die "unknown setting '$key'\n" if !$confdesc->{$key};
2335
2336     my $type = $confdesc->{$key}->{type};
2337
2338     if (!defined($value)) {
2339         die "got undefined value\n";
2340     }
2341
2342     if ($value =~ m/[\n\r]/) {
2343         die "property contains a line feed\n";
2344     }
2345
2346     if ($type eq 'boolean') {
2347         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2348         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2349         die "type check ('boolean') failed - got '$value'\n";
2350     } elsif ($type eq 'integer') {
2351         return int($1) if $value =~ m/^(\d+)$/;
2352         die "type check ('integer') failed - got '$value'\n";
2353     } elsif ($type eq 'number') {
2354         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2355         die "type check ('number') failed - got '$value'\n";
2356     } elsif ($type eq 'string') {
2357         if (my $fmt = $confdesc->{$key}->{format}) {
2358             PVE::JSONSchema::check_format($fmt, $value);
2359             return $value;
2360         }
2361         $value =~ s/^\"(.*)\"$/$1/;
2362         return $value;
2363     } else {
2364         die "internal error"
2365     }
2366 }
2367
2368 sub destroy_vm {
2369     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2370
2371     my $conf = PVE::QemuConfig->load_config($vmid);
2372
2373     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2374         PVE::QemuConfig->check_lock($conf);
2375     }
2376
2377     if ($conf->{template}) {
2378         # check if any base image is still used by a linked clone
2379         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2380                 my ($ds, $drive) = @_;
2381                 return if drive_is_cdrom($drive);
2382
2383                 my $volid = $drive->{file};
2384                 return if !$volid || $volid =~ m|^/|;
2385
2386                 die "base volume '$volid' is still in use by linked cloned\n"
2387                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2388
2389         });
2390     }
2391
2392     my $volids = {};
2393     my $remove_owned_drive = sub {
2394         my ($ds, $drive) = @_;
2395         return if drive_is_cdrom($drive, 1);
2396
2397         my $volid = $drive->{file};
2398         return if !$volid || $volid =~ m|^/|;
2399         return if $volids->{$volid};
2400
2401         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2402         return if !$path || !$owner || ($owner != $vmid);
2403
2404         $volids->{$volid} = 1;
2405         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2406         warn "Could not remove disk '$volid', check manually: $@" if $@;
2407     };
2408
2409     # only remove disks owned by this VM (referenced in the config)
2410     my $include_opts = {
2411         include_unused => 1,
2412         extra_keys => ['vmstate'],
2413     };
2414     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2415
2416     for my $snap (values %{$conf->{snapshots}}) {
2417         next if !defined($snap->{vmstate});
2418         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2419         next if !defined($drive);
2420         $remove_owned_drive->('vmstate', $drive);
2421     }
2422
2423     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2424
2425     if ($purge_unreferenced) { # also remove unreferenced disk
2426         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2427         PVE::Storage::foreach_volid($vmdisks, sub {
2428             my ($volid, $sid, $volname, $d) = @_;
2429             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2430             warn $@ if $@;
2431         });
2432     }
2433
2434     if (defined $replacement_conf) {
2435         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2436     } else {
2437         PVE::QemuConfig->destroy_config($vmid);
2438     }
2439 }
2440
2441 sub parse_vm_config {
2442     my ($filename, $raw, $strict) = @_;
2443
2444     return if !defined($raw);
2445
2446     my $res = {
2447         digest => Digest::SHA::sha1_hex($raw),
2448         snapshots => {},
2449         pending => {},
2450         cloudinit => {},
2451     };
2452
2453     my $handle_error = sub {
2454         my ($msg) = @_;
2455
2456         if ($strict) {
2457             die $msg;
2458         } else {
2459             warn $msg;
2460         }
2461     };
2462
2463     $filename =~ m|/qemu-server/(\d+)\.conf$|
2464         || die "got strange filename '$filename'";
2465
2466     my $vmid = $1;
2467
2468     my $conf = $res;
2469     my $descr;
2470     my $finish_description = sub {
2471         if (defined($descr)) {
2472             $descr =~ s/\s+$//;
2473             $conf->{description} = $descr;
2474         }
2475         $descr = undef;
2476     };
2477     my $section = '';
2478
2479     my @lines = split(/\n/, $raw);
2480     foreach my $line (@lines) {
2481         next if $line =~ m/^\s*$/;
2482
2483         if ($line =~ m/^\[PENDING\]\s*$/i) {
2484             $section = 'pending';
2485             $finish_description->();
2486             $conf = $res->{$section} = {};
2487             next;
2488         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2489             $section = 'cloudinit';
2490             $finish_description->();
2491             $conf = $res->{$section} = {};
2492             next;
2493
2494         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2495             $section = $1;
2496             $finish_description->();
2497             $conf = $res->{snapshots}->{$section} = {};
2498             next;
2499         }
2500
2501         if ($line =~ m/^\#(.*)$/) {
2502             $descr = '' if !defined($descr);
2503             $descr .= PVE::Tools::decode_text($1) . "\n";
2504             next;
2505         }
2506
2507         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2508             $descr = '' if !defined($descr);
2509             $descr .= PVE::Tools::decode_text($2);
2510         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2511             $conf->{snapstate} = $1;
2512         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2513             my $key = $1;
2514             my $value = $2;
2515             $conf->{$key} = $value;
2516         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2517             my $value = $1;
2518             if ($section eq 'pending') {
2519                 $conf->{delete} = $value; # we parse this later
2520             } else {
2521                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2522             }
2523         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2524             my $key = $1;
2525             my $value = $2;
2526             if ($section eq 'cloudinit') {
2527                 # ignore validation only used for informative purpose
2528                 $conf->{$key} = $value;
2529                 next;
2530             }
2531             eval { $value = check_type($key, $value); };
2532             if ($@) {
2533                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2534             } else {
2535                 $key = 'ide2' if $key eq 'cdrom';
2536                 my $fmt = $confdesc->{$key}->{format};
2537                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2538                     my $v = parse_drive($key, $value);
2539                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2540                         $v->{file} = $volid;
2541                         $value = print_drive($v);
2542                     } else {
2543                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2544                         next;
2545                     }
2546                 }
2547
2548                 $conf->{$key} = $value;
2549             }
2550         } else {
2551             $handle_error->("vm $vmid - unable to parse config: $line\n");
2552         }
2553     }
2554
2555     $finish_description->();
2556     delete $res->{snapstate}; # just to be sure
2557
2558     return $res;
2559 }
2560
2561 sub write_vm_config {
2562     my ($filename, $conf) = @_;
2563
2564     delete $conf->{snapstate}; # just to be sure
2565
2566     if ($conf->{cdrom}) {
2567         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2568         $conf->{ide2} = $conf->{cdrom};
2569         delete $conf->{cdrom};
2570     }
2571
2572     # we do not use 'smp' any longer
2573     if ($conf->{sockets}) {
2574         delete $conf->{smp};
2575     } elsif ($conf->{smp}) {
2576         $conf->{sockets} = $conf->{smp};
2577         delete $conf->{cores};
2578         delete $conf->{smp};
2579     }
2580
2581     my $used_volids = {};
2582
2583     my $cleanup_config = sub {
2584         my ($cref, $pending, $snapname) = @_;
2585
2586         foreach my $key (keys %$cref) {
2587             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2588                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2589             my $value = $cref->{$key};
2590             if ($key eq 'delete') {
2591                 die "propertry 'delete' is only allowed in [PENDING]\n"
2592                     if !$pending;
2593                 # fixme: check syntax?
2594                 next;
2595             }
2596             eval { $value = check_type($key, $value); };
2597             die "unable to parse value of '$key' - $@" if $@;
2598
2599             $cref->{$key} = $value;
2600
2601             if (!$snapname && is_valid_drivename($key)) {
2602                 my $drive = parse_drive($key, $value);
2603                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2604             }
2605         }
2606     };
2607
2608     &$cleanup_config($conf);
2609
2610     &$cleanup_config($conf->{pending}, 1);
2611
2612     foreach my $snapname (keys %{$conf->{snapshots}}) {
2613         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2614         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2615     }
2616
2617     # remove 'unusedX' settings if we re-add a volume
2618     foreach my $key (keys %$conf) {
2619         my $value = $conf->{$key};
2620         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2621             delete $conf->{$key};
2622         }
2623     }
2624
2625     my $generate_raw_config = sub {
2626         my ($conf, $pending) = @_;
2627
2628         my $raw = '';
2629
2630         # add description as comment to top of file
2631         if (defined(my $descr = $conf->{description})) {
2632             if ($descr) {
2633                 foreach my $cl (split(/\n/, $descr)) {
2634                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2635                 }
2636             } else {
2637                 $raw .= "#\n" if $pending;
2638             }
2639         }
2640
2641         foreach my $key (sort keys %$conf) {
2642             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2643             $raw .= "$key: $conf->{$key}\n";
2644         }
2645         return $raw;
2646     };
2647
2648     my $raw = &$generate_raw_config($conf);
2649
2650     if (scalar(keys %{$conf->{pending}})){
2651         $raw .= "\n[PENDING]\n";
2652         $raw .= &$generate_raw_config($conf->{pending}, 1);
2653     }
2654
2655     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2656         $raw .= "\n[special:cloudinit]\n";
2657         $raw .= &$generate_raw_config($conf->{cloudinit});
2658     }
2659
2660     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2661         $raw .= "\n[$snapname]\n";
2662         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2663     }
2664
2665     return $raw;
2666 }
2667
2668 sub load_defaults {
2669
2670     my $res = {};
2671
2672     # we use static defaults from our JSON schema configuration
2673     foreach my $key (keys %$confdesc) {
2674         if (defined(my $default = $confdesc->{$key}->{default})) {
2675             $res->{$key} = $default;
2676         }
2677     }
2678
2679     return $res;
2680 }
2681
2682 sub config_list {
2683     my $vmlist = PVE::Cluster::get_vmlist();
2684     my $res = {};
2685     return $res if !$vmlist || !$vmlist->{ids};
2686     my $ids = $vmlist->{ids};
2687     my $nodename = nodename();
2688
2689     foreach my $vmid (keys %$ids) {
2690         my $d = $ids->{$vmid};
2691         next if !$d->{node} || $d->{node} ne $nodename;
2692         next if !$d->{type} || $d->{type} ne 'qemu';
2693         $res->{$vmid}->{exists} = 1;
2694     }
2695     return $res;
2696 }
2697
2698 # test if VM uses local resources (to prevent migration)
2699 sub check_local_resources {
2700     my ($conf, $noerr) = @_;
2701
2702     my @loc_res = ();
2703
2704     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2705     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2706
2707     push @loc_res, "ivshmem" if $conf->{ivshmem};
2708
2709     foreach my $k (keys %$conf) {
2710         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2711         # sockets are safe: they will recreated be on the target side post-migrate
2712         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2713         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2714     }
2715
2716     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2717
2718     return \@loc_res;
2719 }
2720
2721 # check if used storages are available on all nodes (use by migrate)
2722 sub check_storage_availability {
2723     my ($storecfg, $conf, $node) = @_;
2724
2725     PVE::QemuConfig->foreach_volume($conf, sub {
2726         my ($ds, $drive) = @_;
2727
2728         my $volid = $drive->{file};
2729         return if !$volid;
2730
2731         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2732         return if !$sid;
2733
2734         # check if storage is available on both nodes
2735         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2736         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2737
2738         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2739
2740         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2741             if !$scfg->{content}->{$vtype};
2742    });
2743 }
2744
2745 # list nodes where all VM images are available (used by has_feature API)
2746 sub shared_nodes {
2747     my ($conf, $storecfg) = @_;
2748
2749     my $nodelist = PVE::Cluster::get_nodelist();
2750     my $nodehash = { map { $_ => 1 } @$nodelist };
2751     my $nodename = nodename();
2752
2753     PVE::QemuConfig->foreach_volume($conf, sub {
2754         my ($ds, $drive) = @_;
2755
2756         my $volid = $drive->{file};
2757         return if !$volid;
2758
2759         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2760         if ($storeid) {
2761             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2762             if ($scfg->{disable}) {
2763                 $nodehash = {};
2764             } elsif (my $avail = $scfg->{nodes}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     delete $nodehash->{$node} if !$avail->{$node};
2767                 }
2768             } elsif (!$scfg->{shared}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     delete $nodehash->{$node} if $node ne $nodename
2771                 }
2772             }
2773         }
2774     });
2775
2776     return $nodehash
2777 }
2778
2779 sub check_local_storage_availability {
2780     my ($conf, $storecfg) = @_;
2781
2782     my $nodelist = PVE::Cluster::get_nodelist();
2783     my $nodehash = { map { $_ => {} } @$nodelist };
2784
2785     PVE::QemuConfig->foreach_volume($conf, sub {
2786         my ($ds, $drive) = @_;
2787
2788         my $volid = $drive->{file};
2789         return if !$volid;
2790
2791         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2792         if ($storeid) {
2793             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2794
2795             if ($scfg->{disable}) {
2796                 foreach my $node (keys %$nodehash) {
2797                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2798                 }
2799             } elsif (my $avail = $scfg->{nodes}) {
2800                 foreach my $node (keys %$nodehash) {
2801                     if (!$avail->{$node}) {
2802                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2803                     }
2804                 }
2805             }
2806         }
2807     });
2808
2809     foreach my $node (values %$nodehash) {
2810         if (my $unavail = $node->{unavailable_storages}) {
2811             $node->{unavailable_storages} = [ sort keys %$unavail ];
2812         }
2813     }
2814
2815     return $nodehash
2816 }
2817
2818 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2819 sub check_running {
2820     my ($vmid, $nocheck, $node) = @_;
2821
2822     # $nocheck is set when called during a migration, in which case the config
2823     # file might still or already reside on the *other* node
2824     # - because rename has already happened, and current node is source
2825     # - because rename hasn't happened yet, and current node is target
2826     # - because rename has happened, current node is target, but hasn't yet
2827     # processed it yet
2828     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2829     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2830 }
2831
2832 sub vzlist {
2833
2834     my $vzlist = config_list();
2835
2836     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2837
2838     while (defined(my $de = $fd->read)) {
2839         next if $de !~ m/^(\d+)\.pid$/;
2840         my $vmid = $1;
2841         next if !defined($vzlist->{$vmid});
2842         if (my $pid = check_running($vmid)) {
2843             $vzlist->{$vmid}->{pid} = $pid;
2844         }
2845     }
2846
2847     return $vzlist;
2848 }
2849
2850 our $vmstatus_return_properties = {
2851     vmid => get_standard_option('pve-vmid'),
2852     status => {
2853         description => "QEMU process status.",
2854         type => 'string',
2855         enum => ['stopped', 'running'],
2856     },
2857     maxmem => {
2858         description => "Maximum memory in bytes.",
2859         type => 'integer',
2860         optional => 1,
2861         renderer => 'bytes',
2862     },
2863     maxdisk => {
2864         description => "Root disk size in bytes.",
2865         type => 'integer',
2866         optional => 1,
2867         renderer => 'bytes',
2868     },
2869     name => {
2870         description => "VM name.",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     qmpstatus => {
2875         description => "QEMU QMP agent status.",
2876         type => 'string',
2877         optional => 1,
2878     },
2879     pid => {
2880         description => "PID of running qemu process.",
2881         type => 'integer',
2882         optional => 1,
2883     },
2884     uptime => {
2885         description => "Uptime.",
2886         type => 'integer',
2887         optional => 1,
2888         renderer => 'duration',
2889     },
2890     cpus => {
2891         description => "Maximum usable CPUs.",
2892         type => 'number',
2893         optional => 1,
2894     },
2895     lock => {
2896         description => "The current config lock, if any.",
2897         type => 'string',
2898         optional => 1,
2899     },
2900     tags => {
2901         description  => "The current configured tags, if any",
2902         type => 'string',
2903         optional => 1,
2904     },
2905     'running-machine' => {
2906         description => "The currently running machine type (if running).",
2907         type => 'string',
2908         optional => 1,
2909     },
2910     'running-qemu' => {
2911         description => "The currently running QEMU version (if running).",
2912         type => 'string',
2913         optional => 1,
2914     },
2915 };
2916
2917 my $last_proc_pid_stat;
2918
2919 # get VM status information
2920 # This must be fast and should not block ($full == false)
2921 # We only query KVM using QMP if $full == true (this can be slow)
2922 sub vmstatus {
2923     my ($opt_vmid, $full) = @_;
2924
2925     my $res = {};
2926
2927     my $storecfg = PVE::Storage::config();
2928
2929     my $list = vzlist();
2930     my $defaults = load_defaults();
2931
2932     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2933
2934     my $cpucount = $cpuinfo->{cpus} || 1;
2935
2936     foreach my $vmid (keys %$list) {
2937         next if $opt_vmid && ($vmid ne $opt_vmid);
2938
2939         my $conf = PVE::QemuConfig->load_config($vmid);
2940
2941         my $d = { vmid => int($vmid) };
2942         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2943
2944         # fixme: better status?
2945         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2946
2947         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2948         if (defined($size)) {
2949             $d->{disk} = 0; # no info available
2950             $d->{maxdisk} = $size;
2951         } else {
2952             $d->{disk} = 0;
2953             $d->{maxdisk} = 0;
2954         }
2955
2956         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2957             * ($conf->{cores} || $defaults->{cores});
2958         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2959         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2960
2961         $d->{name} = $conf->{name} || "VM $vmid";
2962         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2963             : $defaults->{memory}*(1024*1024);
2964
2965         if ($conf->{balloon}) {
2966             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2967             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2968                 : $defaults->{shares};
2969         }
2970
2971         $d->{uptime} = 0;
2972         $d->{cpu} = 0;
2973         $d->{mem} = 0;
2974
2975         $d->{netout} = 0;
2976         $d->{netin} = 0;
2977
2978         $d->{diskread} = 0;
2979         $d->{diskwrite} = 0;
2980
2981         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2982
2983         $d->{serial} = 1 if conf_has_serial($conf);
2984         $d->{lock} = $conf->{lock} if $conf->{lock};
2985         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2986
2987         $res->{$vmid} = $d;
2988     }
2989
2990     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2991     foreach my $dev (keys %$netdev) {
2992         next if $dev !~ m/^tap([1-9]\d*)i/;
2993         my $vmid = $1;
2994         my $d = $res->{$vmid};
2995         next if !$d;
2996
2997         $d->{netout} += $netdev->{$dev}->{receive};
2998         $d->{netin} += $netdev->{$dev}->{transmit};
2999
3000         if ($full) {
3001             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
3002             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
3003         }
3004
3005     }
3006
3007     my $ctime = gettimeofday;
3008
3009     foreach my $vmid (keys %$list) {
3010
3011         my $d = $res->{$vmid};
3012         my $pid = $d->{pid};
3013         next if !$pid;
3014
3015         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3016         next if !$pstat; # not running
3017
3018         my $used = $pstat->{utime} + $pstat->{stime};
3019
3020         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3021
3022         if ($pstat->{vsize}) {
3023             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3024         }
3025
3026         my $old = $last_proc_pid_stat->{$pid};
3027         if (!$old) {
3028             $last_proc_pid_stat->{$pid} = {
3029                 time => $ctime,
3030                 used => $used,
3031                 cpu => 0,
3032             };
3033             next;
3034         }
3035
3036         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3037
3038         if ($dtime > 1000) {
3039             my $dutime = $used -  $old->{used};
3040
3041             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3042             $last_proc_pid_stat->{$pid} = {
3043                 time => $ctime,
3044                 used => $used,
3045                 cpu => $d->{cpu},
3046             };
3047         } else {
3048             $d->{cpu} = $old->{cpu};
3049         }
3050     }
3051
3052     return $res if !$full;
3053
3054     my $qmpclient = PVE::QMPClient->new();
3055
3056     my $ballooncb = sub {
3057         my ($vmid, $resp) = @_;
3058
3059         my $info = $resp->{'return'};
3060         return if !$info->{max_mem};
3061
3062         my $d = $res->{$vmid};
3063
3064         # use memory assigned to VM
3065         $d->{maxmem} = $info->{max_mem};
3066         $d->{balloon} = $info->{actual};
3067
3068         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3069             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3070             $d->{freemem} = $info->{free_mem};
3071         }
3072
3073         $d->{ballooninfo} = $info;
3074     };
3075
3076     my $blockstatscb = sub {
3077         my ($vmid, $resp) = @_;
3078         my $data = $resp->{'return'} || [];
3079         my $totalrdbytes = 0;
3080         my $totalwrbytes = 0;
3081
3082         for my $blockstat (@$data) {
3083             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3084             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3085
3086             $blockstat->{device} =~ s/drive-//;
3087             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3088         }
3089         $res->{$vmid}->{diskread} = $totalrdbytes;
3090         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3091     };
3092
3093     my $machinecb = sub {
3094         my ($vmid, $resp) = @_;
3095         my $data = $resp->{'return'} || [];
3096
3097         $res->{$vmid}->{'running-machine'} =
3098             PVE::QemuServer::Machine::current_from_query_machines($data);
3099     };
3100
3101     my $versioncb = sub {
3102         my ($vmid, $resp) = @_;
3103         my $data = $resp->{'return'} // {};
3104         my $version = 'unknown';
3105
3106         if (my $v = $data->{qemu}) {
3107             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3108         }
3109
3110         $res->{$vmid}->{'running-qemu'} = $version;
3111     };
3112
3113     my $statuscb = sub {
3114         my ($vmid, $resp) = @_;
3115
3116         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3117         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3118         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3119         # this fails if ballon driver is not loaded, so this must be
3120         # the last commnand (following command are aborted if this fails).
3121         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3122
3123         my $status = 'unknown';
3124         if (!defined($status = $resp->{'return'}->{status})) {
3125             warn "unable to get VM status\n";
3126             return;
3127         }
3128
3129         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3130     };
3131
3132     foreach my $vmid (keys %$list) {
3133         next if $opt_vmid && ($vmid ne $opt_vmid);
3134         next if !$res->{$vmid}->{pid}; # not running
3135         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3136     }
3137
3138     $qmpclient->queue_execute(undef, 2);
3139
3140     foreach my $vmid (keys %$list) {
3141         next if $opt_vmid && ($vmid ne $opt_vmid);
3142         next if !$res->{$vmid}->{pid}; #not running
3143
3144         # we can't use the $qmpclient since it might have already aborted on
3145         # 'query-balloon', but this might also fail for older versions...
3146         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3147         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3148     }
3149
3150     foreach my $vmid (keys %$list) {
3151         next if $opt_vmid && ($vmid ne $opt_vmid);
3152         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3153     }
3154
3155     return $res;
3156 }
3157
3158 sub conf_has_serial {
3159     my ($conf) = @_;
3160
3161     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3162         if ($conf->{"serial$i"}) {
3163             return 1;
3164         }
3165     }
3166
3167     return 0;
3168 }
3169
3170 sub conf_has_audio {
3171     my ($conf, $id) = @_;
3172
3173     $id //= 0;
3174     my $audio = $conf->{"audio$id"};
3175     return if !defined($audio);
3176
3177     my $audioproperties = parse_property_string($audio_fmt, $audio);
3178     my $audiodriver = $audioproperties->{driver} // 'spice';
3179
3180     return {
3181         dev => $audioproperties->{device},
3182         dev_id => "audiodev$id",
3183         backend => $audiodriver,
3184         backend_id => "$audiodriver-backend${id}",
3185     };
3186 }
3187
3188 sub audio_devs {
3189     my ($audio, $audiopciaddr, $machine_version) = @_;
3190
3191     my $devs = [];
3192
3193     my $id = $audio->{dev_id};
3194     my $audiodev = "";
3195     if (min_version($machine_version, 4, 2)) {
3196         $audiodev = ",audiodev=$audio->{backend_id}";
3197     }
3198
3199     if ($audio->{dev} eq 'AC97') {
3200         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3201     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3202         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3203         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3204         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3205     } else {
3206         die "unkown audio device '$audio->{dev}', implement me!";
3207     }
3208
3209     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3210
3211     return $devs;
3212 }
3213
3214 sub get_tpm_paths {
3215     my ($vmid) = @_;
3216     return {
3217         socket => "/var/run/qemu-server/$vmid.swtpm",
3218         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3219     };
3220 }
3221
3222 sub add_tpm_device {
3223     my ($vmid, $devices, $conf) = @_;
3224
3225     return if !$conf->{tpmstate0};
3226
3227     my $paths = get_tpm_paths($vmid);
3228
3229     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3230     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3231     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3232 }
3233
3234 sub start_swtpm {
3235     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3236
3237     return if !$tpmdrive;
3238
3239     my $state;
3240     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3241     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3242     if ($storeid) {
3243         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3244     } else {
3245         $state = $tpm->{file};
3246     }
3247
3248     my $paths = get_tpm_paths($vmid);
3249
3250     # during migration, we will get state from remote
3251     #
3252     if (!$migration) {
3253         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3254         my $setup_cmd = [
3255             "swtpm_setup",
3256             "--tpmstate",
3257             "file://$state",
3258             "--createek",
3259             "--create-ek-cert",
3260             "--create-platform-cert",
3261             "--lock-nvram",
3262             "--config",
3263             "/etc/swtpm_setup.conf", # do not use XDG configs
3264             "--runas",
3265             "0", # force creation as root, error if not possible
3266             "--not-overwrite", # ignore existing state, do not modify
3267         ];
3268
3269         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3270         # TPM 2.0 supports ECC crypto, use if possible
3271         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3272
3273         run_command($setup_cmd, outfunc => sub {
3274             print "swtpm_setup: $1\n";
3275         });
3276     }
3277
3278     # Used to distinguish different invocations in the log.
3279     my $log_prefix = "[id=" . int(time()) . "] ";
3280
3281     my $emulator_cmd = [
3282         "swtpm",
3283         "socket",
3284         "--tpmstate",
3285         "backend-uri=file://$state,mode=0600",
3286         "--ctrl",
3287         "type=unixio,path=$paths->{socket},mode=0600",
3288         "--pid",
3289         "file=$paths->{pid}",
3290         "--terminate", # terminate on QEMU disconnect
3291         "--daemon",
3292         "--log",
3293         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3294     ];
3295     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3296     run_command($emulator_cmd, outfunc => sub { print $1; });
3297
3298     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3299     while (! -e $paths->{pid}) {
3300         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3301         usleep(50_000);
3302     }
3303
3304     # return untainted PID of swtpm daemon so it can be killed on error
3305     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3306     return $1;
3307 }
3308
3309 sub vga_conf_has_spice {
3310     my ($vga) = @_;
3311
3312     my $vgaconf = parse_vga($vga);
3313     my $vgatype = $vgaconf->{type};
3314     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3315
3316     return $1 || 1;
3317 }
3318
3319 sub is_native($) {
3320     my ($arch) = @_;
3321     return get_host_arch() eq $arch;
3322 }
3323
3324 sub get_vm_arch {
3325     my ($conf) = @_;
3326     return $conf->{arch} // get_host_arch();
3327 }
3328
3329 my $default_machines = {
3330     x86_64 => 'pc',
3331     aarch64 => 'virt',
3332 };
3333
3334 sub get_installed_machine_version {
3335     my ($kvmversion) = @_;
3336     $kvmversion = kvm_user_version() if !defined($kvmversion);
3337     $kvmversion =~ m/^(\d+\.\d+)/;
3338     return $1;
3339 }
3340
3341 sub windows_get_pinned_machine_version {
3342     my ($machine, $base_version, $kvmversion) = @_;
3343
3344     my $pin_version = $base_version;
3345     if (!defined($base_version) ||
3346         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3347     ) {
3348         $pin_version = get_installed_machine_version($kvmversion);
3349     }
3350     if (!$machine || $machine eq 'pc') {
3351         $machine = "pc-i440fx-$pin_version";
3352     } elsif ($machine eq 'q35') {
3353         $machine = "pc-q35-$pin_version";
3354     } elsif ($machine eq 'virt') {
3355         $machine = "virt-$pin_version";
3356     } else {
3357         warn "unknown machine type '$machine', not touching that!\n";
3358     }
3359
3360     return $machine;
3361 }
3362
3363 sub get_vm_machine {
3364     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3365
3366     my $machine = $forcemachine || $conf->{machine};
3367
3368     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3369         $kvmversion //= kvm_user_version();
3370         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3371         # layout which confuses windows quite a bit and may result in various regressions..
3372         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3373         if (windows_version($conf->{ostype})) {
3374             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3375         }
3376         $arch //= 'x86_64';
3377         $machine ||= $default_machines->{$arch};
3378         if ($add_pve_version) {
3379             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3380             $machine .= "+pve$pvever";
3381         }
3382     }
3383
3384     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3385         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3386         $machine = $1 if $is_pxe;
3387
3388         # for version-pinned machines that do not include a pve-version (e.g.
3389         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3390         $machine .= '+pve0';
3391
3392         $machine .= '.pxe' if $is_pxe;
3393     }
3394
3395     return $machine;
3396 }
3397
3398 sub get_ovmf_files($$$) {
3399     my ($arch, $efidisk, $smm) = @_;
3400
3401     my $types = $OVMF->{$arch}
3402         or die "no OVMF images known for architecture '$arch'\n";
3403
3404     my $type = 'default';
3405     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3406         $type = $smm ? "4m" : "4m-no-smm";
3407         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3408     }
3409
3410     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3411     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3412     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3413
3414     return ($ovmf_code, $ovmf_vars);
3415 }
3416
3417 my $Arch2Qemu = {
3418     aarch64 => '/usr/bin/qemu-system-aarch64',
3419     x86_64 => '/usr/bin/qemu-system-x86_64',
3420 };
3421 sub get_command_for_arch($) {
3422     my ($arch) = @_;
3423     return '/usr/bin/kvm' if is_native($arch);
3424
3425     my $cmd = $Arch2Qemu->{$arch}
3426         or die "don't know how to emulate architecture '$arch'\n";
3427     return $cmd;
3428 }
3429
3430 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3431 # to use in a QEMU command line (-cpu element), first array_intersect the result
3432 # of query_supported_ with query_understood_. This is necessary because:
3433 #
3434 # a) query_understood_ returns flags the host cannot use and
3435 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3436 #    flags, but CPU settings - with most of them being flags. Those settings
3437 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3438 #
3439 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3440 # expensive. If you need the value returned from this, you can get it much
3441 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3442 # $accel being 'kvm' or 'tcg'.
3443 #
3444 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3445 # changes, automatically populating pmxcfs.
3446 #
3447 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3448 # since kvm and tcg machines support different flags
3449 #
3450 sub query_supported_cpu_flags {
3451     my ($arch) = @_;
3452
3453     $arch //= get_host_arch();
3454     my $default_machine = $default_machines->{$arch};
3455
3456     my $flags = {};
3457
3458     # FIXME: Once this is merged, the code below should work for ARM as well:
3459     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3460     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3461         $arch eq "aarch64";
3462
3463     my $kvm_supported = defined(kvm_version());
3464     my $qemu_cmd = get_command_for_arch($arch);
3465     my $fakevmid = -1;
3466     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3467
3468     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3469     my $query_supported_run_qemu = sub {
3470         my ($kvm) = @_;
3471
3472         my $flags = {};
3473         my $cmd = [
3474             $qemu_cmd,
3475             '-machine', $default_machine,
3476             '-display', 'none',
3477             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3478             '-mon', 'chardev=qmp,mode=control',
3479             '-pidfile', $pidfile,
3480             '-S', '-daemonize'
3481         ];
3482
3483         if (!$kvm) {
3484             push @$cmd, '-accel', 'tcg';
3485         }
3486
3487         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3488         die "QEMU flag querying VM exited with code " . $rc if $rc;
3489
3490         eval {
3491             my $cmd_result = mon_cmd(
3492                 $fakevmid,
3493                 'query-cpu-model-expansion',
3494                 type => 'full',
3495                 model => { name => 'host' }
3496             );
3497
3498             my $props = $cmd_result->{model}->{props};
3499             foreach my $prop (keys %$props) {
3500                 next if $props->{$prop} ne '1';
3501                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3502                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3503                 # We only keep those with underscores, to match /proc/cpuinfo
3504                 $prop =~ s/\.|-/_/g;
3505                 $flags->{$prop} = 1;
3506             }
3507         };
3508         my $err = $@;
3509
3510         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3511         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3512
3513         die $err if $err;
3514
3515         return [ sort keys %$flags ];
3516     };
3517
3518     # We need to query QEMU twice, since KVM and TCG have different supported flags
3519     PVE::QemuConfig->lock_config($fakevmid, sub {
3520         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3521         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3522
3523         if ($kvm_supported) {
3524             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3525             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3526         }
3527     });
3528
3529     return $flags;
3530 }
3531
3532 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3533 my $understood_cpu_flag_dir = "/usr/share/kvm";
3534 sub query_understood_cpu_flags {
3535     my $arch = get_host_arch();
3536     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3537
3538     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3539         if ! -e $filepath;
3540
3541     my $raw = file_get_contents($filepath);
3542     $raw =~ s/^\s+|\s+$//g;
3543     my @flags = split(/\s+/, $raw);
3544
3545     return \@flags;
3546 }
3547
3548 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3549 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3550 my sub should_disable_smm {
3551     my ($conf, $vga, $machine) = @_;
3552
3553     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3554
3555     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3556         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3557 }
3558
3559 my sub print_ovmf_drive_commandlines {
3560     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3561
3562     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3563
3564     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3565
3566     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3567     if ($d) {
3568         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3569         my ($path, $format) = $d->@{'file', 'format'};
3570         if ($storeid) {
3571             $path = PVE::Storage::path($storecfg, $d->{file});
3572             if (!defined($format)) {
3573                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3574                 $format = qemu_img_format($scfg, $volname);
3575             }
3576         } elsif (!defined($format)) {
3577             die "efidisk format must be specified\n";
3578         }
3579         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3580         if ($path =~ m/^rbd:/) {
3581             $var_drive_str .= ',cache=writeback';
3582             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3583         }
3584         $var_drive_str .= ",format=$format,file=$path";
3585
3586         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3587         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3588     } else {
3589         log_warn("no efidisk configured! Using temporary efivars disk.");
3590         my $path = "/tmp/$vmid-ovmf.fd";
3591         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3592         $var_drive_str .= ",format=raw,file=$path";
3593         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3594     }
3595
3596     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3597 }
3598
3599 sub config_to_command {
3600     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3601         $pbs_backing) = @_;
3602
3603     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3604     my $devices = [];
3605     my $bridges = {};
3606     my $ostype = $conf->{ostype};
3607     my $winversion = windows_version($ostype);
3608     my $kvm = $conf->{kvm};
3609     my $nodename = nodename();
3610
3611     my $arch = get_vm_arch($conf);
3612     my $kvm_binary = get_command_for_arch($arch);
3613     my $kvmver = kvm_user_version($kvm_binary);
3614
3615     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3616         $kvmver //= "undefined";
3617         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3618     }
3619
3620     my $add_pve_version = min_version($kvmver, 4, 1);
3621
3622     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3623     my $machine_version = extract_version($machine_type, $kvmver);
3624     $kvm //= 1 if is_native($arch);
3625
3626     $machine_version =~ m/(\d+)\.(\d+)/;
3627     my ($machine_major, $machine_minor) = ($1, $2);
3628
3629     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3630         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3631     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3632         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3633             ." please upgrade node '$nodename'\n"
3634     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3635         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3636         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3637             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3638             ." node '$nodename'\n";
3639     }
3640
3641     # if a specific +pve version is required for a feature, use $version_guard
3642     # instead of min_version to allow machines to be run with the minimum
3643     # required version
3644     my $required_pve_version = 0;
3645     my $version_guard = sub {
3646         my ($major, $minor, $pve) = @_;
3647         return 0 if !min_version($machine_version, $major, $minor, $pve);
3648         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3649         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3650         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3651         return 1;
3652     };
3653
3654     if ($kvm && !defined kvm_version()) {
3655         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3656             ." or enable in BIOS.\n";
3657     }
3658
3659     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3660     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3661     my $use_old_bios_files = undef;
3662     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3663
3664     my $cmd = [];
3665     if ($conf->{affinity}) {
3666         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3667     }
3668
3669     push @$cmd, $kvm_binary;
3670
3671     push @$cmd, '-id', $vmid;
3672
3673     my $vmname = $conf->{name} || "vm$vmid";
3674
3675     push @$cmd, '-name', "$vmname,debug-threads=on";
3676
3677     push @$cmd, '-no-shutdown';
3678
3679     my $use_virtio = 0;
3680
3681     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3682     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3683     push @$cmd, '-mon', "chardev=qmp,mode=control";
3684
3685     if (min_version($machine_version, 2, 12)) {
3686         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3687         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3688     }
3689
3690     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3691
3692     push @$cmd, '-daemonize';
3693
3694     if ($conf->{smbios1}) {
3695         my $smbios_conf = parse_smbios1($conf->{smbios1});
3696         if ($smbios_conf->{base64}) {
3697             # Do not pass base64 flag to qemu
3698             delete $smbios_conf->{base64};
3699             my $smbios_string = "";
3700             foreach my $key (keys %$smbios_conf) {
3701                 my $value;
3702                 if ($key eq "uuid") {
3703                     $value = $smbios_conf->{uuid}
3704                 } else {
3705                     $value = decode_base64($smbios_conf->{$key});
3706                 }
3707                 # qemu accepts any binary data, only commas need escaping by double comma
3708                 $value =~ s/,/,,/g;
3709                 $smbios_string .= "," . $key . "=" . $value if $value;
3710             }
3711             push @$cmd, '-smbios', "type=1" . $smbios_string;
3712         } else {
3713             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3714         }
3715     }
3716
3717     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3718         my ($code_drive_str, $var_drive_str) =
3719             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3720         push $cmd->@*, '-drive', $code_drive_str;
3721         push $cmd->@*, '-drive', $var_drive_str;
3722     }
3723
3724     if ($q35) { # tell QEMU to load q35 config early
3725         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3726         if (min_version($machine_version, 4, 0)) {
3727             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3728         } else {
3729             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3730         }
3731     }
3732
3733     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3734         push @$cmd, $fixups->@*;
3735     }
3736
3737     if ($conf->{vmgenid}) {
3738         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3739     }
3740
3741     # add usb controllers
3742     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3743         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3744     push @$devices, @usbcontrollers if @usbcontrollers;
3745     my $vga = parse_vga($conf->{vga});
3746
3747     my $qxlnum = vga_conf_has_spice($conf->{vga});
3748     $vga->{type} = 'qxl' if $qxlnum;
3749
3750     if (!$vga->{type}) {
3751         if ($arch eq 'aarch64') {
3752             $vga->{type} = 'virtio';
3753         } elsif (min_version($machine_version, 2, 9)) {
3754             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3755         } else {
3756             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3757         }
3758     }
3759
3760     # enable absolute mouse coordinates (needed by vnc)
3761     my $tablet = $conf->{tablet};
3762     if (!defined($tablet)) {
3763         $tablet = $defaults->{tablet};
3764         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3765         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3766     }
3767
3768     if ($tablet) {
3769         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3770         my $kbd = print_keyboarddevice_full($conf, $arch);
3771         push @$devices, '-device', $kbd if defined($kbd);
3772     }
3773
3774     my $bootorder = device_bootorder($conf);
3775
3776     # host pci device passthrough
3777     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3778         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3779
3780     # usb devices
3781     my $usb_dev_features = {};
3782     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3783
3784     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3785         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3786     push @$devices, @usbdevices if @usbdevices;
3787
3788     # serial devices
3789     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3790         my $path = $conf->{"serial$i"} or next;
3791         if ($path eq 'socket') {
3792             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3793             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3794             # On aarch64, serial0 is the UART device. QEMU only allows
3795             # connecting UART devices via the '-serial' command line, as
3796             # the device has a fixed slot on the hardware...
3797             if ($arch eq 'aarch64' && $i == 0) {
3798                 push @$devices, '-serial', "chardev:serial$i";
3799             } else {
3800                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3801             }
3802         } else {
3803             die "no such serial device\n" if ! -c $path;
3804             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3805             push @$devices, '-device', "isa-serial,chardev=serial$i";
3806         }
3807     }
3808
3809     # parallel devices
3810     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3811         if (my $path = $conf->{"parallel$i"}) {
3812             die "no such parallel device\n" if ! -c $path;
3813             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3814             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3815             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3816         }
3817     }
3818
3819     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3820         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3821         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3822         push @$devices, @$audio_devs;
3823     }
3824
3825     add_tpm_device($vmid, $devices, $conf);
3826
3827     my $sockets = 1;
3828     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3829     $sockets = $conf->{sockets} if  $conf->{sockets};
3830
3831     my $cores = $conf->{cores} || 1;
3832
3833     my $maxcpus = $sockets * $cores;
3834
3835     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3836
3837     my $allowed_vcpus = $cpuinfo->{cpus};
3838
3839     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3840
3841     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3842         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3843         for (my $i = 2; $i <= $vcpus; $i++)  {
3844             my $cpustr = print_cpu_device($conf,$i);
3845             push @$cmd, '-device', $cpustr;
3846         }
3847
3848     } else {
3849
3850         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3851     }
3852     push @$cmd, '-nodefaults';
3853
3854     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3855
3856     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3857
3858     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3859
3860     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3861         push @$devices, '-device', print_vga_device(
3862             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3863
3864         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3865
3866         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3867         push @$cmd,  '-vnc', "unix:$socket,password=on";
3868     } else {
3869         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3870         push @$cmd, '-nographic';
3871     }
3872
3873     # time drift fix
3874     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3875     my $useLocaltime = $conf->{localtime};
3876
3877     if ($winversion >= 5) { # windows
3878         $useLocaltime = 1 if !defined($conf->{localtime});
3879
3880         # use time drift fix when acpi is enabled
3881         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3882             $tdf = 1 if !defined($conf->{tdf});
3883         }
3884     }
3885
3886     if ($winversion >= 6) {
3887         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3888         push @$machineFlags, 'hpet=off';
3889     }
3890
3891     push @$rtcFlags, 'driftfix=slew' if $tdf;
3892
3893     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3894         push @$rtcFlags, "base=$conf->{startdate}";
3895     } elsif ($useLocaltime) {
3896         push @$rtcFlags, 'base=localtime';
3897     }
3898
3899     if ($forcecpu) {
3900         push @$cmd, '-cpu', $forcecpu;
3901     } else {
3902         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3903     }
3904
3905     PVE::QemuServer::Memory::config(
3906         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3907
3908     push @$cmd, '-S' if $conf->{freeze};
3909
3910     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3911
3912     my $guest_agent = parse_guest_agent($conf);
3913
3914     if ($guest_agent->{enabled}) {
3915         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3916         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3917
3918         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3919             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3920             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3921             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3922         } elsif ($guest_agent->{type} eq 'isa') {
3923             push @$devices, '-device', "isa-serial,chardev=qga0";
3924         }
3925     }
3926
3927     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3928     if ($rng && $version_guard->(4, 1, 2)) {
3929         check_rng_source($rng->{source});
3930
3931         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3932         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3933         my $limiter_str = "";
3934         if ($max_bytes) {
3935             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3936         }
3937
3938         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3939         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3940         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3941     }
3942
3943     my $spice_port;
3944
3945     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3946         if ($qxlnum > 1) {
3947             if ($winversion){
3948                 for (my $i = 1; $i < $qxlnum; $i++){
3949                     push @$devices, '-device', print_vga_device(
3950                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3951                 }
3952             } else {
3953                 # assume other OS works like Linux
3954                 my ($ram, $vram) = ("134217728", "67108864");
3955                 if ($vga->{memory}) {
3956                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3957                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3958                 }
3959                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3960                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3961             }
3962         }
3963
3964         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3965
3966         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3967         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3968         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3969
3970         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3971         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3972         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3973
3974         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3975         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3976
3977         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3978         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3979         if ($spice_enhancement->{foldersharing}) {
3980             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3981             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3982         }
3983
3984         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3985         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3986             if $spice_enhancement->{videostreaming};
3987
3988         push @$devices, '-spice', "$spice_opts";
3989     }
3990
3991     # enable balloon by default, unless explicitly disabled
3992     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3993         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3994         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3995         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3996         push @$devices, '-device', $ballooncmd;
3997     }
3998
3999     if ($conf->{watchdog}) {
4000         my $wdopts = parse_watchdog($conf->{watchdog});
4001         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4002         my $watchdog = $wdopts->{model} || 'i6300esb';
4003         push @$devices, '-device', "$watchdog$pciaddr";
4004         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4005     }
4006
4007     my $vollist = [];
4008     my $scsicontroller = {};
4009     my $ahcicontroller = {};
4010     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4011
4012     # Add iscsi initiator name if available
4013     if (my $initiator = get_initiator_name()) {
4014         push @$devices, '-iscsi', "initiator-name=$initiator";
4015     }
4016
4017     PVE::QemuConfig->foreach_volume($conf, sub {
4018         my ($ds, $drive) = @_;
4019
4020         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4021             check_volume_storage_type($storecfg, $drive->{file});
4022             push @$vollist, $drive->{file};
4023         }
4024
4025         # ignore efidisk here, already added in bios/fw handling code above
4026         return if $drive->{interface} eq 'efidisk';
4027         # similar for TPM
4028         return if $drive->{interface} eq 'tpmstate';
4029
4030         $use_virtio = 1 if $ds =~ m/^virtio/;
4031
4032         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4033
4034         if ($drive->{interface} eq 'virtio'){
4035            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4036         }
4037
4038         if ($drive->{interface} eq 'scsi') {
4039
4040             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4041
4042             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4043                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4044
4045             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4046             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4047
4048             my $iothread = '';
4049             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4050                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4051                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4052             } elsif ($drive->{iothread}) {
4053                 log_warn(
4054                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4055                 );
4056             }
4057
4058             my $queues = '';
4059             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4060                 $queues = ",num_queues=$drive->{queues}";
4061             }
4062
4063             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4064                 if !$scsicontroller->{$controller};
4065             $scsicontroller->{$controller}=1;
4066         }
4067
4068         if ($drive->{interface} eq 'sata') {
4069             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4070             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4071             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4072                 if !$ahcicontroller->{$controller};
4073             $ahcicontroller->{$controller}=1;
4074         }
4075
4076         my $pbs_conf = $pbs_backing->{$ds};
4077         my $pbs_name = undef;
4078         if ($pbs_conf) {
4079             $pbs_name = "drive-$ds-pbs";
4080             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4081         }
4082
4083         my $drive_cmd = print_drive_commandline_full(
4084             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4085
4086         # extra protection for templates, but SATA and IDE don't support it..
4087         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4088
4089         push @$devices, '-drive',$drive_cmd;
4090         push @$devices, '-device', print_drivedevice_full(
4091             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4092     });
4093
4094     for (my $i = 0; $i < $MAX_NETS; $i++) {
4095         my $netname = "net$i";
4096
4097         next if !$conf->{$netname};
4098         my $d = parse_net($conf->{$netname});
4099         next if !$d;
4100         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4101
4102         $use_virtio = 1 if $d->{model} eq 'virtio';
4103
4104         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4105
4106         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4107         push @$devices, '-netdev', $netdevfull;
4108
4109         my $netdevicefull = print_netdevice_full(
4110             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4111
4112         push @$devices, '-device', $netdevicefull;
4113     }
4114
4115     if ($conf->{ivshmem}) {
4116         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4117
4118         my $bus;
4119         if ($q35) {
4120             $bus = print_pcie_addr("ivshmem");
4121         } else {
4122             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4123         }
4124
4125         my $ivshmem_name = $ivshmem->{name} // $vmid;
4126         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4127
4128         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4129         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4130             .",size=$ivshmem->{size}M";
4131     }
4132
4133     # pci.4 is nested in pci.1
4134     $bridges->{1} = 1 if $bridges->{4};
4135
4136     if (!$q35) { # add pci bridges
4137         if (min_version($machine_version, 2, 3)) {
4138            $bridges->{1} = 1;
4139            $bridges->{2} = 1;
4140         }
4141         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4142     }
4143
4144     for my $k (sort {$b cmp $a} keys %$bridges) {
4145         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4146
4147         my $k_name = $k;
4148         if ($k == 2 && $legacy_igd) {
4149             $k_name = "$k-igd";
4150         }
4151         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4152         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4153
4154         if ($q35) { # add after -readconfig pve-q35.cfg
4155             splice @$devices, 2, 0, '-device', $devstr;
4156         } else {
4157             unshift @$devices, '-device', $devstr if $k > 0;
4158         }
4159     }
4160
4161     if (!$kvm) {
4162         push @$machineFlags, 'accel=tcg';
4163     }
4164
4165     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4166
4167     my $machine_type_min = $machine_type;
4168     if ($add_pve_version) {
4169         $machine_type_min =~ s/\+pve\d+$//;
4170         $machine_type_min .= "+pve$required_pve_version";
4171     }
4172     push @$machineFlags, "type=${machine_type_min}";
4173
4174     push @$cmd, @$devices;
4175     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4176     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4177     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4178
4179     if (my $vmstate = $conf->{vmstate}) {
4180         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4181         push @$vollist, $vmstate;
4182         push @$cmd, '-loadstate', $statepath;
4183         print "activating and using '$vmstate' as vmstate\n";
4184     }
4185
4186     if (PVE::QemuConfig->is_template($conf)) {
4187         # needed to workaround base volumes being read-only
4188         push @$cmd, '-snapshot';
4189     }
4190
4191     # add custom args
4192     if ($conf->{args}) {
4193         my $aa = PVE::Tools::split_args($conf->{args});
4194         push @$cmd, @$aa;
4195     }
4196
4197     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4198 }
4199
4200 sub check_rng_source {
4201     my ($source) = @_;
4202
4203     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4204     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4205         if ! -e $source;
4206
4207     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4208     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4209         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4210         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4211         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4212             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4213             ." to the host.\n";
4214     }
4215 }
4216
4217 sub spice_port {
4218     my ($vmid) = @_;
4219
4220     my $res = mon_cmd($vmid, 'query-spice');
4221
4222     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4223 }
4224
4225 sub vm_devices_list {
4226     my ($vmid) = @_;
4227
4228     my $res = mon_cmd($vmid, 'query-pci');
4229     my $devices_to_check = [];
4230     my $devices = {};
4231     foreach my $pcibus (@$res) {
4232         push @$devices_to_check, @{$pcibus->{devices}},
4233     }
4234
4235     while (@$devices_to_check) {
4236         my $to_check = [];
4237         for my $d (@$devices_to_check) {
4238             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4239             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4240
4241             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4242             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4243         }
4244         $devices_to_check = $to_check;
4245     }
4246
4247     my $resblock = mon_cmd($vmid, 'query-block');
4248     foreach my $block (@$resblock) {
4249         if($block->{device} =~ m/^drive-(\S+)/){
4250                 $devices->{$1} = 1;
4251         }
4252     }
4253
4254     my $resmice = mon_cmd($vmid, 'query-mice');
4255     foreach my $mice (@$resmice) {
4256         if ($mice->{name} eq 'QEMU HID Tablet') {
4257             $devices->{tablet} = 1;
4258             last;
4259         }
4260     }
4261
4262     # for usb devices there is no query-usb
4263     # but we can iterate over the entries in
4264     # qom-list path=/machine/peripheral
4265     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4266     foreach my $per (@$resperipheral) {
4267         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4268             $devices->{$per->{name}} = 1;
4269         }
4270     }
4271
4272     return $devices;
4273 }
4274
4275 sub vm_deviceplug {
4276     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4277
4278     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4279
4280     my $devices_list = vm_devices_list($vmid);
4281     return 1 if defined($devices_list->{$deviceid});
4282
4283     # add PCI bridge if we need it for the device
4284     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4285
4286     if ($deviceid eq 'tablet') {
4287         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4288     } elsif ($deviceid eq 'keyboard') {
4289         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4290     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4291         my $id = $1;
4292         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4293         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4294     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4295         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4296     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4297         qemu_iothread_add($vmid, $deviceid, $device);
4298
4299         qemu_driveadd($storecfg, $vmid, $device);
4300         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4301
4302         qemu_deviceadd($vmid, $devicefull);
4303         eval { qemu_deviceaddverify($vmid, $deviceid); };
4304         if (my $err = $@) {
4305             eval { qemu_drivedel($vmid, $deviceid); };
4306             warn $@ if $@;
4307             die $err;
4308         }
4309     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4310         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4311         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4312         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4313
4314         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4315
4316         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4317             qemu_iothread_add($vmid, $deviceid, $device);
4318             $devicefull .= ",iothread=iothread-$deviceid";
4319         }
4320
4321         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4322             $devicefull .= ",num_queues=$device->{queues}";
4323         }
4324
4325         qemu_deviceadd($vmid, $devicefull);
4326         qemu_deviceaddverify($vmid, $deviceid);
4327     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4328         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4329         qemu_driveadd($storecfg, $vmid, $device);
4330
4331         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4332         eval { qemu_deviceadd($vmid, $devicefull); };
4333         if (my $err = $@) {
4334             eval { qemu_drivedel($vmid, $deviceid); };
4335             warn $@ if $@;
4336             die $err;
4337         }
4338     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4339         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4340
4341         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4342         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4343         my $use_old_bios_files = undef;
4344         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4345
4346         my $netdevicefull = print_netdevice_full(
4347             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4348         qemu_deviceadd($vmid, $netdevicefull);
4349         eval {
4350             qemu_deviceaddverify($vmid, $deviceid);
4351             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4352         };
4353         if (my $err = $@) {
4354             eval { qemu_netdevdel($vmid, $deviceid); };
4355             warn $@ if $@;
4356             die $err;
4357         }
4358     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4359         my $bridgeid = $2;
4360         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4361         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4362
4363         qemu_deviceadd($vmid, $devicefull);
4364         qemu_deviceaddverify($vmid, $deviceid);
4365     } else {
4366         die "can't hotplug device '$deviceid'\n";
4367     }
4368
4369     return 1;
4370 }
4371
4372 # fixme: this should raise exceptions on error!
4373 sub vm_deviceunplug {
4374     my ($vmid, $conf, $deviceid) = @_;
4375
4376     my $devices_list = vm_devices_list($vmid);
4377     return 1 if !defined($devices_list->{$deviceid});
4378
4379     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4380     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4381
4382     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4383         qemu_devicedel($vmid, $deviceid);
4384     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4385         qemu_devicedel($vmid, $deviceid);
4386         qemu_devicedelverify($vmid, $deviceid);
4387     } elsif ($deviceid =~ m/^usb\d+$/) {
4388         qemu_devicedel($vmid, $deviceid);
4389         qemu_devicedelverify($vmid, $deviceid);
4390     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4391         my $device = parse_drive($deviceid, $conf->{$deviceid});
4392
4393         qemu_devicedel($vmid, $deviceid);
4394         qemu_devicedelverify($vmid, $deviceid);
4395         qemu_drivedel($vmid, $deviceid);
4396         qemu_iothread_del($vmid, $deviceid, $device);
4397     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4398         qemu_devicedel($vmid, $deviceid);
4399         qemu_devicedelverify($vmid, $deviceid);
4400     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4401         my $device = parse_drive($deviceid, $conf->{$deviceid});
4402
4403         qemu_devicedel($vmid, $deviceid);
4404         qemu_devicedelverify($vmid, $deviceid);
4405         qemu_drivedel($vmid, $deviceid);
4406         qemu_deletescsihw($conf, $vmid, $deviceid);
4407
4408         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4409             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4410     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4411         qemu_devicedel($vmid, $deviceid);
4412         qemu_devicedelverify($vmid, $deviceid);
4413         qemu_netdevdel($vmid, $deviceid);
4414     } else {
4415         die "can't unplug device '$deviceid'\n";
4416     }
4417
4418     return 1;
4419 }
4420
4421 sub qemu_spice_usbredir_chardev_add {
4422     my ($vmid, $id) = @_;
4423
4424     mon_cmd($vmid, "chardev-add" , (
4425         id => $id,
4426         backend => {
4427             type => 'spicevmc',
4428             data => {
4429                 type => "usbredir",
4430             },
4431         },
4432     ));
4433 }
4434
4435 sub qemu_deviceadd {
4436     my ($vmid, $devicefull) = @_;
4437
4438     $devicefull = "driver=".$devicefull;
4439     my %options =  split(/[=,]/, $devicefull);
4440
4441     mon_cmd($vmid, "device_add" , %options);
4442 }
4443
4444 sub qemu_devicedel {
4445     my ($vmid, $deviceid) = @_;
4446
4447     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4448 }
4449
4450 sub qemu_iothread_add {
4451     my ($vmid, $deviceid, $device) = @_;
4452
4453     if ($device->{iothread}) {
4454         my $iothreads = vm_iothreads_list($vmid);
4455         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4456     }
4457 }
4458
4459 sub qemu_iothread_del {
4460     my ($vmid, $deviceid, $device) = @_;
4461
4462     if ($device->{iothread}) {
4463         my $iothreads = vm_iothreads_list($vmid);
4464         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4465     }
4466 }
4467
4468 sub qemu_objectadd {
4469     my ($vmid, $objectid, $qomtype) = @_;
4470
4471     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4472
4473     return 1;
4474 }
4475
4476 sub qemu_objectdel {
4477     my ($vmid, $objectid) = @_;
4478
4479     mon_cmd($vmid, "object-del", id => $objectid);
4480
4481     return 1;
4482 }
4483
4484 sub qemu_driveadd {
4485     my ($storecfg, $vmid, $device) = @_;
4486
4487     my $kvmver = get_running_qemu_version($vmid);
4488     my $io_uring = min_version($kvmver, 6, 0);
4489     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4490     $drive =~ s/\\/\\\\/g;
4491     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4492
4493     # If the command succeeds qemu prints: "OK"
4494     return 1 if $ret =~ m/OK/s;
4495
4496     die "adding drive failed: $ret\n";
4497 }
4498
4499 sub qemu_drivedel {
4500     my ($vmid, $deviceid) = @_;
4501
4502     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4503     $ret =~ s/^\s+//;
4504
4505     return 1 if $ret eq "";
4506
4507     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4508     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4509
4510     die "deleting drive $deviceid failed : $ret\n";
4511 }
4512
4513 sub qemu_deviceaddverify {
4514     my ($vmid, $deviceid) = @_;
4515
4516     for (my $i = 0; $i <= 5; $i++) {
4517          my $devices_list = vm_devices_list($vmid);
4518          return 1 if defined($devices_list->{$deviceid});
4519          sleep 1;
4520     }
4521
4522     die "error on hotplug device '$deviceid'\n";
4523 }
4524
4525
4526 sub qemu_devicedelverify {
4527     my ($vmid, $deviceid) = @_;
4528
4529     # need to verify that the device is correctly removed as device_del
4530     # is async and empty return is not reliable
4531
4532     for (my $i = 0; $i <= 5; $i++) {
4533          my $devices_list = vm_devices_list($vmid);
4534          return 1 if !defined($devices_list->{$deviceid});
4535          sleep 1;
4536     }
4537
4538     die "error on hot-unplugging device '$deviceid'\n";
4539 }
4540
4541 sub qemu_findorcreatescsihw {
4542     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4543
4544     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4545
4546     my $scsihwid="$controller_prefix$controller";
4547     my $devices_list = vm_devices_list($vmid);
4548
4549     if (!defined($devices_list->{$scsihwid})) {
4550         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4551     }
4552
4553     return 1;
4554 }
4555
4556 sub qemu_deletescsihw {
4557     my ($conf, $vmid, $opt) = @_;
4558
4559     my $device = parse_drive($opt, $conf->{$opt});
4560
4561     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4562         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4563         return 1;
4564     }
4565
4566     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4567
4568     my $devices_list = vm_devices_list($vmid);
4569     foreach my $opt (keys %{$devices_list}) {
4570         if (is_valid_drivename($opt)) {
4571             my $drive = parse_drive($opt, $conf->{$opt});
4572             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4573                 return 1;
4574             }
4575         }
4576     }
4577
4578     my $scsihwid="scsihw$controller";
4579
4580     vm_deviceunplug($vmid, $conf, $scsihwid);
4581
4582     return 1;
4583 }
4584
4585 sub qemu_add_pci_bridge {
4586     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4587
4588     my $bridges = {};
4589
4590     my $bridgeid;
4591
4592     print_pci_addr($device, $bridges, $arch, $machine_type);
4593
4594     while (my ($k, $v) = each %$bridges) {
4595         $bridgeid = $k;
4596     }
4597     return 1 if !defined($bridgeid) || $bridgeid < 1;
4598
4599     my $bridge = "pci.$bridgeid";
4600     my $devices_list = vm_devices_list($vmid);
4601
4602     if (!defined($devices_list->{$bridge})) {
4603         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4604     }
4605
4606     return 1;
4607 }
4608
4609 sub qemu_set_link_status {
4610     my ($vmid, $device, $up) = @_;
4611
4612     mon_cmd($vmid, "set_link", name => $device,
4613                up => $up ? JSON::true : JSON::false);
4614 }
4615
4616 sub qemu_netdevadd {
4617     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4618
4619     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4620     my %options =  split(/[=,]/, $netdev);
4621
4622     if (defined(my $vhost = $options{vhost})) {
4623         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4624     }
4625
4626     if (defined(my $queues = $options{queues})) {
4627         $options{queues} = $queues + 0;
4628     }
4629
4630     mon_cmd($vmid, "netdev_add",  %options);
4631     return 1;
4632 }
4633
4634 sub qemu_netdevdel {
4635     my ($vmid, $deviceid) = @_;
4636
4637     mon_cmd($vmid, "netdev_del", id => $deviceid);
4638 }
4639
4640 sub qemu_usb_hotplug {
4641     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4642
4643     return if !$device;
4644
4645     # remove the old one first
4646     vm_deviceunplug($vmid, $conf, $deviceid);
4647
4648     # check if xhci controller is necessary and available
4649     my $devicelist = vm_devices_list($vmid);
4650
4651     if (!$devicelist->{xhci}) {
4652         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4653         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4654     }
4655
4656     # print_usbdevice_full expects the parsed device
4657     my $d = parse_usb_device($device->{host});
4658     $d->{usb3} = $device->{usb3};
4659
4660     # add the new one
4661     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4662 }
4663
4664 sub qemu_cpu_hotplug {
4665     my ($vmid, $conf, $vcpus) = @_;
4666
4667     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4668
4669     my $sockets = 1;
4670     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4671     $sockets = $conf->{sockets} if  $conf->{sockets};
4672     my $cores = $conf->{cores} || 1;
4673     my $maxcpus = $sockets * $cores;
4674
4675     $vcpus = $maxcpus if !$vcpus;
4676
4677     die "you can't add more vcpus than maxcpus\n"
4678         if $vcpus > $maxcpus;
4679
4680     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4681
4682     if ($vcpus < $currentvcpus) {
4683
4684         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4685
4686             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4687                 qemu_devicedel($vmid, "cpu$i");
4688                 my $retry = 0;
4689                 my $currentrunningvcpus = undef;
4690                 while (1) {
4691                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4692                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4693                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4694                     $retry++;
4695                     sleep 1;
4696                 }
4697                 #update conf after each succesfull cpu unplug
4698                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4699                 PVE::QemuConfig->write_config($vmid, $conf);
4700             }
4701         } else {
4702             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4703         }
4704
4705         return;
4706     }
4707
4708     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4709     die "vcpus in running vm does not match its configuration\n"
4710         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4711
4712     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4713
4714         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4715             my $cpustr = print_cpu_device($conf, $i);
4716             qemu_deviceadd($vmid, $cpustr);
4717
4718             my $retry = 0;
4719             my $currentrunningvcpus = undef;
4720             while (1) {
4721                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4722                 last if scalar(@{$currentrunningvcpus}) == $i;
4723                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4724                 sleep 1;
4725                 $retry++;
4726             }
4727             #update conf after each succesfull cpu hotplug
4728             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4729             PVE::QemuConfig->write_config($vmid, $conf);
4730         }
4731     } else {
4732
4733         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4734             mon_cmd($vmid, "cpu-add", id => int($i));
4735         }
4736     }
4737 }
4738
4739 sub qemu_block_set_io_throttle {
4740     my ($vmid, $deviceid,
4741         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4742         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4743         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4744         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4745
4746     return if !check_running($vmid) ;
4747
4748     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4749         bps => int($bps),
4750         bps_rd => int($bps_rd),
4751         bps_wr => int($bps_wr),
4752         iops => int($iops),
4753         iops_rd => int($iops_rd),
4754         iops_wr => int($iops_wr),
4755         bps_max => int($bps_max),
4756         bps_rd_max => int($bps_rd_max),
4757         bps_wr_max => int($bps_wr_max),
4758         iops_max => int($iops_max),
4759         iops_rd_max => int($iops_rd_max),
4760         iops_wr_max => int($iops_wr_max),
4761         bps_max_length => int($bps_max_length),
4762         bps_rd_max_length => int($bps_rd_max_length),
4763         bps_wr_max_length => int($bps_wr_max_length),
4764         iops_max_length => int($iops_max_length),
4765         iops_rd_max_length => int($iops_rd_max_length),
4766         iops_wr_max_length => int($iops_wr_max_length),
4767     );
4768
4769 }
4770
4771 sub qemu_block_resize {
4772     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4773
4774     my $running = check_running($vmid);
4775
4776     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4777
4778     return if !$running;
4779
4780     my $padding = (1024 - $size % 1024) % 1024;
4781     $size = $size + $padding;
4782
4783     mon_cmd(
4784         $vmid,
4785         "block_resize",
4786         device => $deviceid,
4787         size => int($size),
4788         timeout => 60,
4789     );
4790 }
4791
4792 sub qemu_volume_snapshot {
4793     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4794
4795     my $running = check_running($vmid);
4796
4797     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4798         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4799     } else {
4800         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4801     }
4802 }
4803
4804 sub qemu_volume_snapshot_delete {
4805     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4806
4807     my $running = check_running($vmid);
4808
4809     if($running) {
4810
4811         $running = undef;
4812         my $conf = PVE::QemuConfig->load_config($vmid);
4813         PVE::QemuConfig->foreach_volume($conf, sub {
4814             my ($ds, $drive) = @_;
4815             $running = 1 if $drive->{file} eq $volid;
4816         });
4817     }
4818
4819     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4820         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4821     } else {
4822         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4823     }
4824 }
4825
4826 sub set_migration_caps {
4827     my ($vmid, $savevm) = @_;
4828
4829     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4830
4831     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4832     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4833
4834     my $cap_ref = [];
4835
4836     my $enabled_cap = {
4837         "auto-converge" => 1,
4838         "xbzrle" => 1,
4839         "x-rdma-pin-all" => 0,
4840         "zero-blocks" => 0,
4841         "compress" => 0,
4842         "dirty-bitmaps" => $dirty_bitmaps,
4843     };
4844
4845     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4846
4847     for my $supported_capability (@$supported_capabilities) {
4848         push @$cap_ref, {
4849             capability => $supported_capability->{capability},
4850             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4851         };
4852     }
4853
4854     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4855 }
4856
4857 sub foreach_volid {
4858     my ($conf, $func, @param) = @_;
4859
4860     my $volhash = {};
4861
4862     my $test_volid = sub {
4863         my ($key, $drive, $snapname) = @_;
4864
4865         my $volid = $drive->{file};
4866         return if !$volid;
4867
4868         $volhash->{$volid}->{cdrom} //= 1;
4869         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4870
4871         my $replicate = $drive->{replicate} // 1;
4872         $volhash->{$volid}->{replicate} //= 0;
4873         $volhash->{$volid}->{replicate} = 1 if $replicate;
4874
4875         $volhash->{$volid}->{shared} //= 0;
4876         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4877
4878         $volhash->{$volid}->{referenced_in_config} //= 0;
4879         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4880
4881         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4882             if defined($snapname);
4883
4884         my $size = $drive->{size};
4885         $volhash->{$volid}->{size} //= $size if $size;
4886
4887         $volhash->{$volid}->{is_vmstate} //= 0;
4888         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4889
4890         $volhash->{$volid}->{is_tpmstate} //= 0;
4891         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4892
4893         $volhash->{$volid}->{is_unused} //= 0;
4894         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4895
4896         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4897     };
4898
4899     my $include_opts = {
4900         extra_keys => ['vmstate'],
4901         include_unused => 1,
4902     };
4903
4904     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4905     foreach my $snapname (keys %{$conf->{snapshots}}) {
4906         my $snap = $conf->{snapshots}->{$snapname};
4907         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4908     }
4909
4910     foreach my $volid (keys %$volhash) {
4911         &$func($volid, $volhash->{$volid}, @param);
4912     }
4913 }
4914
4915 my $fast_plug_option = {
4916     'lock' => 1,
4917     'name' => 1,
4918     'onboot' => 1,
4919     'shares' => 1,
4920     'startup' => 1,
4921     'description' => 1,
4922     'protection' => 1,
4923     'vmstatestorage' => 1,
4924     'hookscript' => 1,
4925     'tags' => 1,
4926 };
4927
4928 for my $opt (keys %$confdesc_cloudinit) {
4929     $fast_plug_option->{$opt} = 1;
4930 };
4931
4932 # hotplug changes in [PENDING]
4933 # $selection hash can be used to only apply specified options, for
4934 # example: { cores => 1 } (only apply changed 'cores')
4935 # $errors ref is used to return error messages
4936 sub vmconfig_hotplug_pending {
4937     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4938
4939     my $defaults = load_defaults();
4940     my $arch = get_vm_arch($conf);
4941     my $machine_type = get_vm_machine($conf, undef, $arch);
4942
4943     # commit values which do not have any impact on running VM first
4944     # Note: those option cannot raise errors, we we do not care about
4945     # $selection and always apply them.
4946
4947     my $add_error = sub {
4948         my ($opt, $msg) = @_;
4949         $errors->{$opt} = "hotplug problem - $msg";
4950     };
4951
4952     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4953
4954     my $cloudinit_record_changed = sub {
4955         my ($conf, $opt, $old, $new) = @_;
4956         return if !$cloudinit_pending_properties->{$opt};
4957
4958         my $ci = ($conf->{cloudinit} //= {});
4959
4960         my $recorded = $ci->{$opt};
4961         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4962
4963         if (defined($new)) {
4964             if (defined($old)) {
4965                 # an existing value is being modified
4966                 if (defined($recorded)) {
4967                     # the value was already not in sync
4968                     if ($new eq $recorded) {
4969                         # a value is being reverted to the cloud-init state:
4970                         delete $ci->{$opt};
4971                         delete $added{$opt};
4972                     } else {
4973                         # the value was changed multiple times, do nothing
4974                     }
4975                 } elsif ($added{$opt}) {
4976                     # the value had been marked as added and is being changed, do nothing
4977                 } else {
4978                     # the value is new, record it:
4979                     $ci->{$opt} = $old;
4980                 }
4981             } else {
4982                 # a new value is being added
4983                 if (defined($recorded)) {
4984                     # it was already not in sync
4985                     if ($new eq $recorded) {
4986                         # a value is being reverted to the cloud-init state:
4987                         delete $ci->{$opt};
4988                         delete $added{$opt};
4989                     } else {
4990                         # the value had temporarily been removed, do nothing
4991                     }
4992                 } elsif ($added{$opt}) {
4993                     # the value had been marked as added already, do nothing
4994                 } else {
4995                     # the value is new, add it
4996                     $added{$opt} = 1;
4997                 }
4998             }
4999         } elsif (!defined($old)) {
5000             # a non-existent value is being removed? ignore...
5001         } else {
5002             # a value is being deleted
5003             if (defined($recorded)) {
5004                 # a value was already recorded, just keep it
5005             } elsif ($added{$opt}) {
5006                 # the value was marked as added, remove it
5007                 delete $added{$opt};
5008             } else {
5009                 # a previously unrecorded value is being removed, record the old value:
5010                 $ci->{$opt} = $old;
5011             }
5012         }
5013
5014         my $added = join(',', sort keys %added);
5015         $ci->{added} = $added if length($added);
5016     };
5017
5018     my $changes = 0;
5019     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5020         if ($fast_plug_option->{$opt}) {
5021             my $new = delete $conf->{pending}->{$opt};
5022             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5023             $conf->{$opt} = $new;
5024             $changes = 1;
5025         }
5026     }
5027
5028     if ($changes) {
5029         PVE::QemuConfig->write_config($vmid, $conf);
5030     }
5031
5032     my $ostype = $conf->{ostype};
5033     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5034     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5035     my $usb_hotplug = $hotplug_features->{usb}
5036         && min_version($version, 7, 1)
5037         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5038
5039     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5040     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5041
5042     foreach my $opt (sort keys %$pending_delete_hash) {
5043         next if $selection && !$selection->{$opt};
5044         my $force = $pending_delete_hash->{$opt}->{force};
5045         eval {
5046             if ($opt eq 'hotplug') {
5047                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5048             } elsif ($opt eq 'tablet') {
5049                 die "skip\n" if !$hotplug_features->{usb};
5050                 if ($defaults->{tablet}) {
5051                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5052                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5053                         if $arch eq 'aarch64';
5054                 } else {
5055                     vm_deviceunplug($vmid, $conf, 'tablet');
5056                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5057                 }
5058             } elsif ($opt =~ m/^usb(\d+)$/) {
5059                 my $index = $1;
5060                 die "skip\n" if !$usb_hotplug;
5061                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5062                 vm_deviceunplug($vmid, $conf, $opt);
5063             } elsif ($opt eq 'vcpus') {
5064                 die "skip\n" if !$hotplug_features->{cpu};
5065                 qemu_cpu_hotplug($vmid, $conf, undef);
5066             } elsif ($opt eq 'balloon') {
5067                 # enable balloon device is not hotpluggable
5068                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5069                 # here we reset the ballooning value to memory
5070                 my $balloon = $conf->{memory} || $defaults->{memory};
5071                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5072             } elsif ($fast_plug_option->{$opt}) {
5073                 # do nothing
5074             } elsif ($opt =~ m/^net(\d+)$/) {
5075                 die "skip\n" if !$hotplug_features->{network};
5076                 vm_deviceunplug($vmid, $conf, $opt);
5077             } elsif (is_valid_drivename($opt)) {
5078                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5079                 vm_deviceunplug($vmid, $conf, $opt);
5080                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5081             } elsif ($opt =~ m/^memory$/) {
5082                 die "skip\n" if !$hotplug_features->{memory};
5083                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5084             } elsif ($opt eq 'cpuunits') {
5085                 $cgroup->change_cpu_shares(undef);
5086             } elsif ($opt eq 'cpulimit') {
5087                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5088             } else {
5089                 die "skip\n";
5090             }
5091         };
5092         if (my $err = $@) {
5093             &$add_error($opt, $err) if $err ne "skip\n";
5094         } else {
5095             my $old = delete $conf->{$opt};
5096             $cloudinit_record_changed->($conf, $opt, $old, undef);
5097             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5098         }
5099     }
5100
5101     my $cloudinit_opt;
5102     foreach my $opt (keys %{$conf->{pending}}) {
5103         next if $selection && !$selection->{$opt};
5104         my $value = $conf->{pending}->{$opt};
5105         eval {
5106             if ($opt eq 'hotplug') {
5107                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5108             } elsif ($opt eq 'tablet') {
5109                 die "skip\n" if !$hotplug_features->{usb};
5110                 if ($value == 1) {
5111                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5112                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5113                         if $arch eq 'aarch64';
5114                 } elsif ($value == 0) {
5115                     vm_deviceunplug($vmid, $conf, 'tablet');
5116                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5117                 }
5118             } elsif ($opt =~ m/^usb(\d+)$/) {
5119                 my $index = $1;
5120                 die "skip\n" if !$usb_hotplug;
5121                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5122                 my $id = $opt;
5123                 if ($d->{host} eq 'spice')  {
5124                     $id = "usbredirdev$index";
5125                 }
5126                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5127             } elsif ($opt eq 'vcpus') {
5128                 die "skip\n" if !$hotplug_features->{cpu};
5129                 qemu_cpu_hotplug($vmid, $conf, $value);
5130             } elsif ($opt eq 'balloon') {
5131                 # enable/disable balloning device is not hotpluggable
5132                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5133                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5134                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5135
5136                 # allow manual ballooning if shares is set to zero
5137                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5138                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5139                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5140                 }
5141             } elsif ($opt =~ m/^net(\d+)$/) {
5142                 # some changes can be done without hotplug
5143                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5144                                     $vmid, $opt, $value, $arch, $machine_type);
5145             } elsif (is_valid_drivename($opt)) {
5146                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5147                 # some changes can be done without hotplug
5148                 my $drive = parse_drive($opt, $value);
5149                 if (drive_is_cloudinit($drive)) {
5150                     $cloudinit_opt = [$opt, $drive];
5151                     # apply all the other changes first, then generate the cloudinit disk
5152                     die "skip\n";
5153                 }
5154                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5155                                      $vmid, $opt, $value, $arch, $machine_type);
5156             } elsif ($opt =~ m/^memory$/) { #dimms
5157                 die "skip\n" if !$hotplug_features->{memory};
5158                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5159             } elsif ($opt eq 'cpuunits') {
5160                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5161                 $cgroup->change_cpu_shares($new_cpuunits);
5162             } elsif ($opt eq 'cpulimit') {
5163                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5164                 $cgroup->change_cpu_quota($cpulimit, 100000);
5165             } elsif ($opt eq 'agent') {
5166                 vmconfig_update_agent($conf, $opt, $value);
5167             } else {
5168                 die "skip\n";  # skip non-hot-pluggable options
5169             }
5170         };
5171         if (my $err = $@) {
5172             &$add_error($opt, $err) if $err ne "skip\n";
5173         } else {
5174             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5175             $conf->{$opt} = $value;
5176             delete $conf->{pending}->{$opt};
5177         }
5178     }
5179
5180     if (defined($cloudinit_opt)) {
5181         my ($opt, $drive) = @$cloudinit_opt;
5182         my $value = $conf->{pending}->{$opt};
5183         eval {
5184             my $temp = {%$conf, $opt => $value};
5185             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5186             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5187                                  $vmid, $opt, $value, $arch, $machine_type);
5188         };
5189         if (my $err = $@) {
5190             &$add_error($opt, $err) if $err ne "skip\n";
5191         } else {
5192             $conf->{$opt} = $value;
5193             delete $conf->{pending}->{$opt};
5194         }
5195     }
5196
5197     # unplug xhci controller if no usb device is left
5198     if ($usb_hotplug) {
5199         my $has_usb = 0;
5200         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5201             next if !defined($conf->{"usb$i"});
5202             $has_usb = 1;
5203             last;
5204         }
5205         if (!$has_usb) {
5206             vm_deviceunplug($vmid, $conf, 'xhci');
5207         }
5208     }
5209
5210     PVE::QemuConfig->write_config($vmid, $conf);
5211
5212     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5213         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5214     }
5215 }
5216
5217 sub try_deallocate_drive {
5218     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5219
5220     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5221         my $volid = $drive->{file};
5222         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5223             my $sid = PVE::Storage::parse_volume_id($volid);
5224             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5225
5226             # check if the disk is really unused
5227             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5228                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5229             PVE::Storage::vdisk_free($storecfg, $volid);
5230             return 1;
5231         } else {
5232             # If vm is not owner of this disk remove from config
5233             return 1;
5234         }
5235     }
5236
5237     return;
5238 }
5239
5240 sub vmconfig_delete_or_detach_drive {
5241     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5242
5243     my $drive = parse_drive($opt, $conf->{$opt});
5244
5245     my $rpcenv = PVE::RPCEnvironment::get();
5246     my $authuser = $rpcenv->get_user();
5247
5248     if ($force) {
5249         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5250         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5251     } else {
5252         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5253     }
5254 }
5255
5256
5257
5258 sub vmconfig_apply_pending {
5259     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5260
5261     return if !scalar(keys %{$conf->{pending}});
5262
5263     my $add_apply_error = sub {
5264         my ($opt, $msg) = @_;
5265         my $err_msg = "unable to apply pending change $opt : $msg";
5266         $errors->{$opt} = $err_msg;
5267         warn $err_msg;
5268     };
5269
5270     # cold plug
5271
5272     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5273     foreach my $opt (sort keys %$pending_delete_hash) {
5274         my $force = $pending_delete_hash->{$opt}->{force};
5275         eval {
5276             if ($opt =~ m/^unused/) {
5277                 die "internal error";
5278             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5279                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5280             }
5281         };
5282         if (my $err = $@) {
5283             $add_apply_error->($opt, $err);
5284         } else {
5285             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5286             delete $conf->{$opt};
5287         }
5288     }
5289
5290     PVE::QemuConfig->cleanup_pending($conf);
5291
5292     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5293
5294     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5295         next if $opt eq 'delete'; # just to be sure
5296         eval {
5297             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5298                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5299             }
5300         };
5301         if (my $err = $@) {
5302             $add_apply_error->($opt, $err);
5303         } else {
5304
5305             if (is_valid_drivename($opt)) {
5306                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5307                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5308             }
5309
5310             $conf->{$opt} = delete $conf->{pending}->{$opt};
5311         }
5312     }
5313
5314     # write all changes at once to avoid unnecessary i/o
5315     PVE::QemuConfig->write_config($vmid, $conf);
5316     if ($generate_cloudinit) {
5317         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5318             # After successful generation and if there were changes to be applied, update the
5319             # config to drop the {cloudinit} entry.
5320             PVE::QemuConfig->write_config($vmid, $conf);
5321         }
5322     }
5323 }
5324
5325 sub vmconfig_update_net {
5326     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5327
5328     my $newnet = parse_net($value);
5329
5330     if ($conf->{$opt}) {
5331         my $oldnet = parse_net($conf->{$opt});
5332
5333         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5334             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5335             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5336             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5337             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5338
5339             # for non online change, we try to hot-unplug
5340             die "skip\n" if !$hotplug;
5341             vm_deviceunplug($vmid, $conf, $opt);
5342         } else {
5343
5344             die "internal error" if $opt !~ m/net(\d+)/;
5345             my $iface = "tap${vmid}i$1";
5346
5347             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5348                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5349                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5350                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5351                 PVE::Network::tap_unplug($iface);
5352
5353                 if ($have_sdn) {
5354                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5355                 } else {
5356                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5357                 }
5358             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5359                 # Rate can be applied on its own but any change above needs to
5360                 # include the rate in tap_plug since OVS resets everything.
5361                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5362             }
5363
5364             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5365                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5366             }
5367
5368             return 1;
5369         }
5370     }
5371
5372     if ($hotplug) {
5373         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5374     } else {
5375         die "skip\n";
5376     }
5377 }
5378
5379 sub vmconfig_update_agent {
5380     my ($conf, $opt, $value) = @_;
5381
5382     die "skip\n" if !$conf->{$opt};
5383
5384     my $hotplug_options = { fstrim_cloned_disks => 1 };
5385
5386     my $old_agent = parse_guest_agent($conf);
5387     my $agent = parse_guest_agent({$opt => $value});
5388
5389     for my $option (keys %$agent) { # added/changed options
5390         next if defined($hotplug_options->{$option});
5391         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5392     }
5393
5394     for my $option (keys %$old_agent) { # removed options
5395         next if defined($hotplug_options->{$option});
5396         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5397     }
5398
5399     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5400 }
5401
5402 sub vmconfig_update_disk {
5403     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5404
5405     my $drive = parse_drive($opt, $value);
5406
5407     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5408         my $media = $drive->{media} || 'disk';
5409         my $oldmedia = $old_drive->{media} || 'disk';
5410         die "unable to change media type\n" if $media ne $oldmedia;
5411
5412         if (!drive_is_cdrom($old_drive)) {
5413
5414             if ($drive->{file} ne $old_drive->{file}) {
5415
5416                 die "skip\n" if !$hotplug;
5417
5418                 # unplug and register as unused
5419                 vm_deviceunplug($vmid, $conf, $opt);
5420                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5421
5422             } else {
5423                 # update existing disk
5424
5425                 # skip non hotpluggable value
5426                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5427                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5428                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5429                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5430                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5431                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5432                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5433                     die "skip\n";
5434                 }
5435
5436                 # apply throttle
5437                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5438                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5439                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5440                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5441                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5442                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5443                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5444                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5445                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5446                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5447                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5448                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5449                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5450                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5451                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5452                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5453                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5454                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5455
5456                     qemu_block_set_io_throttle(
5457                         $vmid,"drive-$opt",
5458                         ($drive->{mbps} || 0)*1024*1024,
5459                         ($drive->{mbps_rd} || 0)*1024*1024,
5460                         ($drive->{mbps_wr} || 0)*1024*1024,
5461                          $drive->{iops} || 0,
5462                          $drive->{iops_rd} || 0,
5463                          $drive->{iops_wr} || 0,
5464                         ($drive->{mbps_max} || 0)*1024*1024,
5465                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5466                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5467                          $drive->{iops_max} || 0,
5468                          $drive->{iops_rd_max} || 0,
5469                          $drive->{iops_wr_max} || 0,
5470                          $drive->{bps_max_length} || 1,
5471                          $drive->{bps_rd_max_length} || 1,
5472                          $drive->{bps_wr_max_length} || 1,
5473                          $drive->{iops_max_length} || 1,
5474                          $drive->{iops_rd_max_length} || 1,
5475                          $drive->{iops_wr_max_length} || 1,
5476                     );
5477
5478                 }
5479
5480                 return 1;
5481             }
5482
5483         } else { # cdrom
5484
5485             if ($drive->{file} eq 'none') {
5486                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5487                 if (drive_is_cloudinit($old_drive)) {
5488                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5489                 }
5490             } else {
5491                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5492
5493                 # force eject if locked
5494                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5495
5496                 if ($path) {
5497                     mon_cmd($vmid, "blockdev-change-medium",
5498                         id => "$opt", filename => "$path");
5499                 }
5500             }
5501
5502             return 1;
5503         }
5504     }
5505
5506     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5507     # hotplug new disks
5508     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5509     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5510 }
5511
5512 sub vmconfig_update_cloudinit_drive {
5513     my ($storecfg, $conf, $vmid) = @_;
5514
5515     my $cloudinit_ds = undef;
5516     my $cloudinit_drive = undef;
5517
5518     PVE::QemuConfig->foreach_volume($conf, sub {
5519         my ($ds, $drive) = @_;
5520         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5521             $cloudinit_ds = $ds;
5522             $cloudinit_drive = $drive;
5523         }
5524     });
5525
5526     return if !$cloudinit_drive;
5527
5528     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5529         PVE::QemuConfig->write_config($vmid, $conf);
5530     }
5531
5532     my $running = PVE::QemuServer::check_running($vmid);
5533
5534     if ($running) {
5535         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5536         if ($path) {
5537             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5538             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5539         }
5540     }
5541 }
5542
5543 # called in locked context by incoming migration
5544 sub vm_migrate_get_nbd_disks {
5545     my ($storecfg, $conf, $replicated_volumes) = @_;
5546
5547     my $local_volumes = {};
5548     PVE::QemuConfig->foreach_volume($conf, sub {
5549         my ($ds, $drive) = @_;
5550
5551         return if drive_is_cdrom($drive);
5552         return if $ds eq 'tpmstate0';
5553
5554         my $volid = $drive->{file};
5555
5556         return if !$volid;
5557
5558         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5559
5560         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5561         return if $scfg->{shared};
5562
5563         # replicated disks re-use existing state via bitmap
5564         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5565         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5566     });
5567     return $local_volumes;
5568 }
5569
5570 # called in locked context by incoming migration
5571 sub vm_migrate_alloc_nbd_disks {
5572     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5573
5574     my $nbd = {};
5575     foreach my $opt (sort keys %$source_volumes) {
5576         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5577
5578         if ($use_existing) {
5579             $nbd->{$opt}->{drivestr} = print_drive($drive);
5580             $nbd->{$opt}->{volid} = $volid;
5581             $nbd->{$opt}->{replicated} = 1;
5582             next;
5583         }
5584
5585         # storage mapping + volname = regular migration
5586         # storage mapping + format = remote migration
5587         # order of precedence, filtered by whether storage supports it:
5588         # 1. explicit requested format
5589         # 2. format of current volume
5590         # 3. default format of storage
5591         if (!$storagemap->{identity}) {
5592             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5593             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5594             if (!$format || !grep { $format eq $_ } @$validFormats) {
5595                 if ($volname) {
5596                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5597                     my $fileFormat = qemu_img_format($scfg, $volname);
5598                     $format = $fileFormat
5599                         if grep { $fileFormat eq $_ } @$validFormats;
5600                 }
5601                 $format //= $defFormat;
5602             }
5603         } else {
5604             # can't happen for remote migration, so $volname is always defined
5605             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5606             $format = qemu_img_format($scfg, $volname);
5607         }
5608
5609         my $size = $drive->{size} / 1024;
5610         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5611         my $newdrive = $drive;
5612         $newdrive->{format} = $format;
5613         $newdrive->{file} = $newvolid;
5614         my $drivestr = print_drive($newdrive);
5615         $nbd->{$opt}->{drivestr} = $drivestr;
5616         $nbd->{$opt}->{volid} = $newvolid;
5617     }
5618
5619     return $nbd;
5620 }
5621
5622 # see vm_start_nolock for parameters, additionally:
5623 # migrate_opts:
5624 #   storagemap = parsed storage map for allocating NBD disks
5625 sub vm_start {
5626     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5627
5628     return PVE::QemuConfig->lock_config($vmid, sub {
5629         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5630
5631         die "you can't start a vm if it's a template\n"
5632             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5633
5634         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5635         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5636
5637         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5638
5639         if ($has_backup_lock && $running) {
5640             # a backup is currently running, attempt to start the guest in the
5641             # existing QEMU instance
5642             return vm_resume($vmid);
5643         }
5644
5645         PVE::QemuConfig->check_lock($conf)
5646             if !($params->{skiplock} || $has_suspended_lock);
5647
5648         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5649
5650         die "VM $vmid already running\n" if $running;
5651
5652         if (my $storagemap = $migrate_opts->{storagemap}) {
5653             my $replicated = $migrate_opts->{replicated_volumes};
5654             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5655             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5656
5657             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5658                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5659             }
5660         }
5661
5662         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5663     });
5664 }
5665
5666
5667 # params:
5668 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5669 #   skiplock => 0/1, skip checking for config lock
5670 #   skiptemplate => 0/1, skip checking whether VM is template
5671 #   forcemachine => to force QEMU machine (rollback/migration)
5672 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5673 #   timeout => in seconds
5674 #   paused => start VM in paused state (backup)
5675 #   resume => resume from hibernation
5676 #   pbs-backing => {
5677 #      sata0 => {
5678 #         repository
5679 #         snapshot
5680 #         keyfile
5681 #         archive
5682 #      },
5683 #      virtio2 => ...
5684 #   }
5685 # migrate_opts:
5686 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5687 #   migratedfrom => source node
5688 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5689 #   network => CIDR of migration network
5690 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5691 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5692 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5693 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5694 #       contained in config
5695 sub vm_start_nolock {
5696     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5697
5698     my $statefile = $params->{statefile};
5699     my $resume = $params->{resume};
5700
5701     my $migratedfrom = $migrate_opts->{migratedfrom};
5702     my $migration_type = $migrate_opts->{type};
5703
5704     my $res = {};
5705
5706     # clean up leftover reboot request files
5707     eval { clear_reboot_request($vmid); };
5708     warn $@ if $@;
5709
5710     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5711         vmconfig_apply_pending($vmid, $conf, $storecfg);
5712         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5713     }
5714
5715     # don't regenerate the ISO if the VM is started as part of a live migration
5716     # this way we can reuse the old ISO with the correct config
5717     if (!$migratedfrom) {
5718         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5719             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5720             # $conf->{cloudinit}, so we could just not do this?
5721             # But we do it above, so for now let's be consistent.
5722             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5723         }
5724     }
5725
5726     # override offline migrated volumes, conf is out of date still
5727     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5728         for my $key (sort keys $offline_volumes->%*) {
5729             my $parsed = parse_drive($key, $conf->{$key});
5730             $parsed->{file} = $offline_volumes->{$key};
5731             $conf->{$key} = print_drive($parsed);
5732         }
5733     }
5734
5735     my $defaults = load_defaults();
5736
5737     # set environment variable useful inside network script
5738     # for remote migration the config is available on the target node!
5739     if (!$migrate_opts->{remote_node}) {
5740         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5741     }
5742
5743     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5744
5745     my $forcemachine = $params->{forcemachine};
5746     my $forcecpu = $params->{forcecpu};
5747     if ($resume) {
5748         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5749         $forcemachine = $conf->{runningmachine};
5750         $forcecpu = $conf->{runningcpu};
5751         print "Resuming suspended VM\n";
5752     }
5753
5754     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5755         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5756
5757     my $migration_ip;
5758     my $get_migration_ip = sub {
5759         my ($nodename) = @_;
5760
5761         return $migration_ip if defined($migration_ip);
5762
5763         my $cidr = $migrate_opts->{network};
5764
5765         if (!defined($cidr)) {
5766             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5767             $cidr = $dc_conf->{migration}->{network};
5768         }
5769
5770         if (defined($cidr)) {
5771             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5772
5773             die "could not get IP: no address configured on local " .
5774                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5775
5776             die "could not get IP: multiple addresses configured on local " .
5777                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5778
5779             $migration_ip = @$ips[0];
5780         }
5781
5782         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5783             if !defined($migration_ip);
5784
5785         return $migration_ip;
5786     };
5787
5788     if ($statefile) {
5789         if ($statefile eq 'tcp') {
5790             my $migrate = $res->{migrate} = { proto => 'tcp' };
5791             $migrate->{addr} = "localhost";
5792             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5793             my $nodename = nodename();
5794
5795             if (!defined($migration_type)) {
5796                 if (defined($datacenterconf->{migration}->{type})) {
5797                     $migration_type = $datacenterconf->{migration}->{type};
5798                 } else {
5799                     $migration_type = 'secure';
5800                 }
5801             }
5802
5803             if ($migration_type eq 'insecure') {
5804                 $migrate->{addr} = $get_migration_ip->($nodename);
5805                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5806             }
5807
5808             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5809             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5810             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5811             push @$cmd, '-incoming', $migrate->{uri};
5812             push @$cmd, '-S';
5813
5814         } elsif ($statefile eq 'unix') {
5815             # should be default for secure migrations as a ssh TCP forward
5816             # tunnel is not deterministic reliable ready and fails regurarly
5817             # to set up in time, so use UNIX socket forwards
5818             my $migrate = $res->{migrate} = { proto => 'unix' };
5819             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5820             unlink $migrate->{addr};
5821
5822             $migrate->{uri} = "unix:$migrate->{addr}";
5823             push @$cmd, '-incoming', $migrate->{uri};
5824             push @$cmd, '-S';
5825
5826         } elsif (-e $statefile) {
5827             push @$cmd, '-loadstate', $statefile;
5828         } else {
5829             my $statepath = PVE::Storage::path($storecfg, $statefile);
5830             push @$vollist, $statefile;
5831             push @$cmd, '-loadstate', $statepath;
5832         }
5833     } elsif ($params->{paused}) {
5834         push @$cmd, '-S';
5835     }
5836
5837     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5838
5839     my $pci_devices = {}; # host pci devices
5840     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5841         my $dev = $conf->{"hostpci$i"} or next;
5842         $pci_devices->{$i} = parse_hostpci($dev);
5843     }
5844
5845     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5846     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5847
5848     # map to a flat list of pci ids
5849     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5850
5851     # reserve all PCI IDs before actually doing anything with them
5852     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5853
5854     eval {
5855         my $uuid;
5856         for my $id (sort keys %$pci_devices) {
5857             my $d = $pci_devices->{$id};
5858             for my $dev ($d->{pciid}->@*) {
5859                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5860
5861                 # nvidia grid needs the qemu parameter '-uuid' set
5862                 # use smbios uuid or mdev uuid as fallback for that
5863                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5864                     if (defined($conf->{smbios1})) {
5865                         my $smbios_conf = parse_smbios1($conf->{smbios1});
5866                         $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5867                     }
5868                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id) if !defined($uuid);
5869                 }
5870             }
5871         }
5872         push @$cmd, '-uuid', $uuid if defined($uuid);
5873     };
5874     if (my $err = $@) {
5875         eval { cleanup_pci_devices($vmid, $conf) };
5876         warn $@ if $@;
5877         die $err;
5878     }
5879
5880     PVE::Storage::activate_volumes($storecfg, $vollist);
5881
5882     eval {
5883         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5884     };
5885     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5886     # timeout should be more than enough here...
5887     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5888
5889     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5890
5891     my %run_params = (
5892         timeout => $statefile ? undef : $start_timeout,
5893         umask => 0077,
5894         noerr => 1,
5895     );
5896
5897     # when migrating, prefix QEMU output so other side can pick up any
5898     # errors that might occur and show the user
5899     if ($migratedfrom) {
5900         $run_params{quiet} = 1;
5901         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5902     }
5903
5904     my %systemd_properties = (
5905         Slice => 'qemu.slice',
5906         KillMode => 'process',
5907         SendSIGKILL => 0,
5908         TimeoutStopUSec => ULONG_MAX, # infinity
5909     );
5910
5911     if (PVE::CGroup::cgroup_mode() == 2) {
5912         $systemd_properties{CPUWeight} = $cpuunits;
5913     } else {
5914         $systemd_properties{CPUShares} = $cpuunits;
5915     }
5916
5917     if (my $cpulimit = $conf->{cpulimit}) {
5918         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5919     }
5920     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5921
5922     my $run_qemu = sub {
5923         PVE::Tools::run_fork sub {
5924             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5925
5926             my $tpmpid;
5927             if (my $tpm = $conf->{tpmstate0}) {
5928                 # start the TPM emulator so QEMU can connect on start
5929                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5930             }
5931
5932             my $exitcode = run_command($cmd, %run_params);
5933             if ($exitcode) {
5934                 if ($tpmpid) {
5935                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5936                     kill 'TERM', $tpmpid;
5937                 }
5938                 die "QEMU exited with code $exitcode\n";
5939             }
5940         };
5941     };
5942
5943     if ($conf->{hugepages}) {
5944
5945         my $code = sub {
5946             my $hotplug_features =
5947                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5948             my $hugepages_topology =
5949                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5950
5951             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5952
5953             PVE::QemuServer::Memory::hugepages_mount();
5954             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5955
5956             eval { $run_qemu->() };
5957             if (my $err = $@) {
5958                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5959                     if !$conf->{keephugepages};
5960                 die $err;
5961             }
5962
5963             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5964                 if !$conf->{keephugepages};
5965         };
5966         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5967
5968     } else {
5969         eval { $run_qemu->() };
5970     }
5971
5972     if (my $err = $@) {
5973         # deactivate volumes if start fails
5974         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5975         warn $@ if $@;
5976         eval { cleanup_pci_devices($vmid, $conf) };
5977         warn $@ if $@;
5978
5979         die "start failed: $err";
5980     }
5981
5982     # re-reserve all PCI IDs now that we can know the actual VM PID
5983     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5984     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5985     warn $@ if $@;
5986
5987     if (defined($res->{migrate})) {
5988         print "migration listens on $res->{migrate}->{uri}\n";
5989     } elsif ($statefile) {
5990         eval { mon_cmd($vmid, "cont"); };
5991         warn $@ if $@;
5992     }
5993
5994     #start nbd server for storage migration
5995     if (my $nbd = $migrate_opts->{nbd}) {
5996         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5997
5998         my $migrate_storage_uri;
5999         # nbd_protocol_version > 0 for unix socket support
6000         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
6001             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
6002             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
6003             $migrate_storage_uri = "nbd:unix:$socket_path";
6004             $res->{migrate}->{unix_sockets} = [$socket_path];
6005         } else {
6006             my $nodename = nodename();
6007             my $localip = $get_migration_ip->($nodename);
6008             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6009             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6010
6011             mon_cmd($vmid, "nbd-server-start", addr => {
6012                 type => 'inet',
6013                 data => {
6014                     host => "${localip}",
6015                     port => "${storage_migrate_port}",
6016                 },
6017             });
6018             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6019             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6020         }
6021
6022         my $block_info = mon_cmd($vmid, "query-block");
6023         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6024
6025         foreach my $opt (sort keys %$nbd) {
6026             my $drivestr = $nbd->{$opt}->{drivestr};
6027             my $volid = $nbd->{$opt}->{volid};
6028
6029             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6030
6031             mon_cmd(
6032                 $vmid,
6033                 "block-export-add",
6034                 id => "drive-$opt",
6035                 'node-name' => $block_node,
6036                 writable => JSON::true,
6037                 type => "nbd",
6038                 name => "drive-$opt", # NBD export name
6039             );
6040
6041             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6042             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6043             print "re-using replicated volume: $opt - $volid\n"
6044                 if $nbd->{$opt}->{replicated};
6045
6046             $res->{drives}->{$opt} = $nbd->{$opt};
6047             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6048         }
6049     }
6050
6051     if ($migratedfrom) {
6052         eval {
6053             set_migration_caps($vmid);
6054         };
6055         warn $@ if $@;
6056
6057         if ($spice_port) {
6058             print "spice listens on port $spice_port\n";
6059             $res->{spice_port} = $spice_port;
6060             if ($migrate_opts->{spice_ticket}) {
6061                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6062                     $migrate_opts->{spice_ticket});
6063                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6064             }
6065         }
6066
6067     } else {
6068         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6069             if !$statefile && $conf->{balloon};
6070
6071         foreach my $opt (keys %$conf) {
6072             next if $opt !~  m/^net\d+$/;
6073             my $nicconf = parse_net($conf->{$opt});
6074             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6075         }
6076         add_nets_bridge_fdb($conf, $vmid);
6077     }
6078
6079    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6080         eval {
6081             mon_cmd(
6082                 $vmid,
6083                 'qom-set',
6084                 path => "machine/peripheral/balloon0",
6085                 property => "guest-stats-polling-interval",
6086                 value => 2
6087             );
6088         };
6089         log_warn("could not set polling interval for ballooning - $@") if $@;
6090     }
6091
6092     if ($resume) {
6093         print "Resumed VM, removing state\n";
6094         if (my $vmstate = $conf->{vmstate}) {
6095             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6096             PVE::Storage::vdisk_free($storecfg, $vmstate);
6097         }
6098         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6099         PVE::QemuConfig->write_config($vmid, $conf);
6100     }
6101
6102     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6103
6104     return $res;
6105 }
6106
6107 sub vm_commandline {
6108     my ($storecfg, $vmid, $snapname) = @_;
6109
6110     my $conf = PVE::QemuConfig->load_config($vmid);
6111
6112     my ($forcemachine, $forcecpu);
6113     if ($snapname) {
6114         my $snapshot = $conf->{snapshots}->{$snapname};
6115         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6116
6117         # check for machine or CPU overrides in snapshot
6118         $forcemachine = $snapshot->{runningmachine};
6119         $forcecpu = $snapshot->{runningcpu};
6120
6121         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6122
6123         $conf = $snapshot;
6124     }
6125
6126     my $defaults = load_defaults();
6127
6128     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6129
6130     return PVE::Tools::cmd2string($cmd);
6131 }
6132
6133 sub vm_reset {
6134     my ($vmid, $skiplock) = @_;
6135
6136     PVE::QemuConfig->lock_config($vmid, sub {
6137
6138         my $conf = PVE::QemuConfig->load_config($vmid);
6139
6140         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6141
6142         mon_cmd($vmid, "system_reset");
6143     });
6144 }
6145
6146 sub get_vm_volumes {
6147     my ($conf) = @_;
6148
6149     my $vollist = [];
6150     foreach_volid($conf, sub {
6151         my ($volid, $attr) = @_;
6152
6153         return if $volid =~ m|^/|;
6154
6155         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6156         return if !$sid;
6157
6158         push @$vollist, $volid;
6159     });
6160
6161     return $vollist;
6162 }
6163
6164 sub cleanup_pci_devices {
6165     my ($vmid, $conf) = @_;
6166
6167     foreach my $key (keys %$conf) {
6168         next if $key !~ m/^hostpci(\d+)$/;
6169         my $hostpciindex = $1;
6170         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6171         my $d = parse_hostpci($conf->{$key});
6172         if ($d->{mdev}) {
6173             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6174             # don't want to break ABI just for this two liner
6175             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6176
6177             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6178             # out when we do it first. so wait for 10 seconds and then try it
6179             my $pciid = $d->{pciid}->[0]->{id};
6180             my $info = PVE::SysFSTools::pci_device_info("$pciid");
6181             if ($info->{vendor} eq '10de') {
6182                 sleep 10;
6183             }
6184
6185             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6186         }
6187     }
6188     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6189 }
6190
6191 sub vm_stop_cleanup {
6192     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6193
6194     eval {
6195
6196         if (!$keepActive) {
6197             my $vollist = get_vm_volumes($conf);
6198             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6199
6200             if (my $tpmdrive = $conf->{tpmstate0}) {
6201                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6202                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6203                 if ($storeid) {
6204                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6205                 }
6206             }
6207         }
6208
6209         foreach my $ext (qw(mon qmp pid vnc qga)) {
6210             unlink "/var/run/qemu-server/${vmid}.$ext";
6211         }
6212
6213         if ($conf->{ivshmem}) {
6214             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6215             # just delete it for now, VMs which have this already open do not
6216             # are affected, but new VMs will get a separated one. If this
6217             # becomes an issue we either add some sort of ref-counting or just
6218             # add a "don't delete on stop" flag to the ivshmem format.
6219             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6220         }
6221
6222         cleanup_pci_devices($vmid, $conf);
6223
6224         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6225     };
6226     warn $@ if $@; # avoid errors - just warn
6227 }
6228
6229 # call only in locked context
6230 sub _do_vm_stop {
6231     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6232
6233     my $pid = check_running($vmid, $nocheck);
6234     return if !$pid;
6235
6236     my $conf;
6237     if (!$nocheck) {
6238         $conf = PVE::QemuConfig->load_config($vmid);
6239         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6240         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6241             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6242             $timeout = $opts->{down} if $opts->{down};
6243         }
6244         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6245     }
6246
6247     eval {
6248         if ($shutdown) {
6249             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6250                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6251             } else {
6252                 mon_cmd($vmid, "system_powerdown");
6253             }
6254         } else {
6255             mon_cmd($vmid, "quit");
6256         }
6257     };
6258     my $err = $@;
6259
6260     if (!$err) {
6261         $timeout = 60 if !defined($timeout);
6262
6263         my $count = 0;
6264         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6265             $count++;
6266             sleep 1;
6267         }
6268
6269         if ($count >= $timeout) {
6270             if ($force) {
6271                 warn "VM still running - terminating now with SIGTERM\n";
6272                 kill 15, $pid;
6273             } else {
6274                 die "VM quit/powerdown failed - got timeout\n";
6275             }
6276         } else {
6277             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6278             return;
6279         }
6280     } else {
6281        if (!check_running($vmid, $nocheck)) {
6282             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6283             return;
6284        }
6285        if ($force) {
6286             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6287             kill 15, $pid;
6288         } else {
6289             die "VM quit/powerdown failed\n";
6290         }
6291     }
6292
6293     # wait again
6294     $timeout = 10;
6295
6296     my $count = 0;
6297     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6298         $count++;
6299         sleep 1;
6300     }
6301
6302     if ($count >= $timeout) {
6303         warn "VM still running - terminating now with SIGKILL\n";
6304         kill 9, $pid;
6305         sleep 1;
6306     }
6307
6308     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6309 }
6310
6311 # Note: use $nocheck to skip tests if VM configuration file exists.
6312 # We need that when migration VMs to other nodes (files already moved)
6313 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6314 sub vm_stop {
6315     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6316
6317     $force = 1 if !defined($force) && !$shutdown;
6318
6319     if ($migratedfrom){
6320         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6321         kill 15, $pid if $pid;
6322         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6323         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6324         return;
6325     }
6326
6327     PVE::QemuConfig->lock_config($vmid, sub {
6328         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6329    });
6330 }
6331
6332 sub vm_reboot {
6333     my ($vmid, $timeout) = @_;
6334
6335     PVE::QemuConfig->lock_config($vmid, sub {
6336         eval {
6337
6338             # only reboot if running, as qmeventd starts it again on a stop event
6339             return if !check_running($vmid);
6340
6341             create_reboot_request($vmid);
6342
6343             my $storecfg = PVE::Storage::config();
6344             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6345
6346         };
6347         if (my $err = $@) {
6348             # avoid that the next normal shutdown will be confused for a reboot
6349             clear_reboot_request($vmid);
6350             die $err;
6351         }
6352    });
6353 }
6354
6355 # note: if using the statestorage parameter, the caller has to check privileges
6356 sub vm_suspend {
6357     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6358
6359     my $conf;
6360     my $path;
6361     my $storecfg;
6362     my $vmstate;
6363
6364     PVE::QemuConfig->lock_config($vmid, sub {
6365
6366         $conf = PVE::QemuConfig->load_config($vmid);
6367
6368         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6369         PVE::QemuConfig->check_lock($conf)
6370             if !($skiplock || $is_backing_up);
6371
6372         die "cannot suspend to disk during backup\n"
6373             if $is_backing_up && $includestate;
6374
6375         if ($includestate) {
6376             $conf->{lock} = 'suspending';
6377             my $date = strftime("%Y-%m-%d", localtime(time()));
6378             $storecfg = PVE::Storage::config();
6379             if (!$statestorage) {
6380                 $statestorage = find_vmstate_storage($conf, $storecfg);
6381                 # check permissions for the storage
6382                 my $rpcenv = PVE::RPCEnvironment::get();
6383                 if ($rpcenv->{type} ne 'cli') {
6384                     my $authuser = $rpcenv->get_user();
6385                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6386                 }
6387             }
6388
6389
6390             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6391                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6392             $path = PVE::Storage::path($storecfg, $vmstate);
6393             PVE::QemuConfig->write_config($vmid, $conf);
6394         } else {
6395             mon_cmd($vmid, "stop");
6396         }
6397     });
6398
6399     if ($includestate) {
6400         # save vm state
6401         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6402
6403         eval {
6404             set_migration_caps($vmid, 1);
6405             mon_cmd($vmid, "savevm-start", statefile => $path);
6406             for(;;) {
6407                 my $state = mon_cmd($vmid, "query-savevm");
6408                 if (!$state->{status}) {
6409                     die "savevm not active\n";
6410                 } elsif ($state->{status} eq 'active') {
6411                     sleep(1);
6412                     next;
6413                 } elsif ($state->{status} eq 'completed') {
6414                     print "State saved, quitting\n";
6415                     last;
6416                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6417                     die "query-savevm failed with error '$state->{error}'\n"
6418                 } else {
6419                     die "query-savevm returned status '$state->{status}'\n";
6420                 }
6421             }
6422         };
6423         my $err = $@;
6424
6425         PVE::QemuConfig->lock_config($vmid, sub {
6426             $conf = PVE::QemuConfig->load_config($vmid);
6427             if ($err) {
6428                 # cleanup, but leave suspending lock, to indicate something went wrong
6429                 eval {
6430                     mon_cmd($vmid, "savevm-end");
6431                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6432                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6433                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6434                     PVE::QemuConfig->write_config($vmid, $conf);
6435                 };
6436                 warn $@ if $@;
6437                 die $err;
6438             }
6439
6440             die "lock changed unexpectedly\n"
6441                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6442
6443             mon_cmd($vmid, "quit");
6444             $conf->{lock} = 'suspended';
6445             PVE::QemuConfig->write_config($vmid, $conf);
6446         });
6447     }
6448 }
6449
6450 # $nocheck is set when called as part of a migration - in this context the
6451 # location of the config file (source or target node) is not deterministic,
6452 # since migration cannot wait for pmxcfs to process the rename
6453 sub vm_resume {
6454     my ($vmid, $skiplock, $nocheck) = @_;
6455
6456     PVE::QemuConfig->lock_config($vmid, sub {
6457         my $res = mon_cmd($vmid, 'query-status');
6458         my $resume_cmd = 'cont';
6459         my $reset = 0;
6460         my $conf;
6461         if ($nocheck) {
6462             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6463             if ($@) {
6464                 my $vmlist = PVE::Cluster::get_vmlist();
6465                 if (exists($vmlist->{ids}->{$vmid})) {
6466                     my $node = $vmlist->{ids}->{$vmid}->{node};
6467                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6468                 }
6469                 if (!$conf) {
6470                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6471                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6472                 }
6473             }
6474         } else {
6475             $conf = PVE::QemuConfig->load_config($vmid);
6476         }
6477
6478         if ($res->{status}) {
6479             return if $res->{status} eq 'running'; # job done, go home
6480             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6481             $reset = 1 if $res->{status} eq 'shutdown';
6482         }
6483
6484         if (!$nocheck) {
6485             PVE::QemuConfig->check_lock($conf)
6486                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6487         }
6488
6489         if ($reset) {
6490             # required if a VM shuts down during a backup and we get a resume
6491             # request before the backup finishes for example
6492             mon_cmd($vmid, "system_reset");
6493         }
6494
6495         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6496
6497         mon_cmd($vmid, $resume_cmd);
6498     });
6499 }
6500
6501 sub vm_sendkey {
6502     my ($vmid, $skiplock, $key) = @_;
6503
6504     PVE::QemuConfig->lock_config($vmid, sub {
6505
6506         my $conf = PVE::QemuConfig->load_config($vmid);
6507
6508         # there is no qmp command, so we use the human monitor command
6509         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6510         die $res if $res ne '';
6511     });
6512 }
6513
6514 # vzdump restore implementaion
6515
6516 sub tar_archive_read_firstfile {
6517     my $archive = shift;
6518
6519     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6520
6521     # try to detect archive type first
6522     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6523         die "unable to open file '$archive'\n";
6524     my $firstfile = <$fh>;
6525     kill 15, $pid;
6526     close $fh;
6527
6528     die "ERROR: archive contaions no data\n" if !$firstfile;
6529     chomp $firstfile;
6530
6531     return $firstfile;
6532 }
6533
6534 sub tar_restore_cleanup {
6535     my ($storecfg, $statfile) = @_;
6536
6537     print STDERR "starting cleanup\n";
6538
6539     if (my $fd = IO::File->new($statfile, "r")) {
6540         while (defined(my $line = <$fd>)) {
6541             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6542                 my $volid = $2;
6543                 eval {
6544                     if ($volid =~ m|^/|) {
6545                         unlink $volid || die 'unlink failed\n';
6546                     } else {
6547                         PVE::Storage::vdisk_free($storecfg, $volid);
6548                     }
6549                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6550                 };
6551                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6552             } else {
6553                 print STDERR "unable to parse line in statfile - $line";
6554             }
6555         }
6556         $fd->close();
6557     }
6558 }
6559
6560 sub restore_file_archive {
6561     my ($archive, $vmid, $user, $opts) = @_;
6562
6563     return restore_vma_archive($archive, $vmid, $user, $opts)
6564         if $archive eq '-';
6565
6566     my $info = PVE::Storage::archive_info($archive);
6567     my $format = $opts->{format} // $info->{format};
6568     my $comp = $info->{compression};
6569
6570     # try to detect archive format
6571     if ($format eq 'tar') {
6572         return restore_tar_archive($archive, $vmid, $user, $opts);
6573     } else {
6574         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6575     }
6576 }
6577
6578 # hepler to remove disks that will not be used after restore
6579 my $restore_cleanup_oldconf = sub {
6580     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6581
6582     my $kept_disks = {};
6583
6584     PVE::QemuConfig->foreach_volume($oldconf, sub {
6585         my ($ds, $drive) = @_;
6586
6587         return if drive_is_cdrom($drive, 1);
6588
6589         my $volid = $drive->{file};
6590         return if !$volid || $volid =~ m|^/|;
6591
6592         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6593         return if !$path || !$owner || ($owner != $vmid);
6594
6595         # Note: only delete disk we want to restore
6596         # other volumes will become unused
6597         if ($virtdev_hash->{$ds}) {
6598             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6599             if (my $err = $@) {
6600                 warn $err;
6601             }
6602         } else {
6603             $kept_disks->{$volid} = 1;
6604         }
6605     });
6606
6607     # after the restore we have no snapshots anymore
6608     for my $snapname (keys $oldconf->{snapshots}->%*) {
6609         my $snap = $oldconf->{snapshots}->{$snapname};
6610         if ($snap->{vmstate}) {
6611             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6612             if (my $err = $@) {
6613                 warn $err;
6614             }
6615         }
6616
6617         for my $volid (keys $kept_disks->%*) {
6618             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6619             warn $@ if $@;
6620         }
6621     }
6622 };
6623
6624 # Helper to parse vzdump backup device hints
6625 #
6626 # $rpcenv: Environment, used to ckeck storage permissions
6627 # $user: User ID, to check storage permissions
6628 # $storecfg: Storage configuration
6629 # $fh: the file handle for reading the configuration
6630 # $devinfo: should contain device sizes for all backu-up'ed devices
6631 # $options: backup options (pool, default storage)
6632 #
6633 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6634 my $parse_backup_hints = sub {
6635     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6636
6637     my $check_storage = sub { # assert if an image can be allocate
6638         my ($storeid, $scfg) = @_;
6639         die "Content type 'images' is not available on storage '$storeid'\n"
6640             if !$scfg->{content}->{images};
6641         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6642             if $user ne 'root@pam';
6643     };
6644
6645     my $virtdev_hash = {};
6646     while (defined(my $line = <$fh>)) {
6647         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6648             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6649             die "archive does not contain data for drive '$virtdev'\n"
6650                 if !$devinfo->{$devname};
6651
6652             if (defined($options->{storage})) {
6653                 $storeid = $options->{storage} || 'local';
6654             } elsif (!$storeid) {
6655                 $storeid = 'local';
6656             }
6657             $format = 'raw' if !$format;
6658             $devinfo->{$devname}->{devname} = $devname;
6659             $devinfo->{$devname}->{virtdev} = $virtdev;
6660             $devinfo->{$devname}->{format} = $format;
6661             $devinfo->{$devname}->{storeid} = $storeid;
6662
6663             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6664             $check_storage->($storeid, $scfg); # permission and content type check
6665
6666             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6667         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6668             my $virtdev = $1;
6669             my $drive = parse_drive($virtdev, $2);
6670
6671             if (drive_is_cloudinit($drive)) {
6672                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6673                 $storeid = $options->{storage} if defined ($options->{storage});
6674                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6675                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6676
6677                 $check_storage->($storeid, $scfg); # permission and content type check
6678
6679                 $virtdev_hash->{$virtdev} = {
6680                     format => $format,
6681                     storeid => $storeid,
6682                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6683                     is_cloudinit => 1,
6684                 };
6685             }
6686         }
6687     }
6688
6689     return $virtdev_hash;
6690 };
6691
6692 # Helper to allocate and activate all volumes required for a restore
6693 #
6694 # $storecfg: Storage configuration
6695 # $virtdev_hash: as returned by parse_backup_hints()
6696 #
6697 # Returns: { $virtdev => $volid }
6698 my $restore_allocate_devices = sub {
6699     my ($storecfg, $virtdev_hash, $vmid) = @_;
6700
6701     my $map = {};
6702     foreach my $virtdev (sort keys %$virtdev_hash) {
6703         my $d = $virtdev_hash->{$virtdev};
6704         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6705         my $storeid = $d->{storeid};
6706         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6707
6708         # test if requested format is supported
6709         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6710         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6711         $d->{format} = $defFormat if !$supported;
6712
6713         my $name;
6714         if ($d->{is_cloudinit}) {
6715             $name = "vm-$vmid-cloudinit";
6716             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6717             if ($scfg->{path}) {
6718                 $name .= ".$d->{format}";
6719             }
6720         }
6721
6722         my $volid = PVE::Storage::vdisk_alloc(
6723             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6724
6725         print STDERR "new volume ID is '$volid'\n";
6726         $d->{volid} = $volid;
6727
6728         PVE::Storage::activate_volumes($storecfg, [$volid]);
6729
6730         $map->{$virtdev} = $volid;
6731     }
6732
6733     return $map;
6734 };
6735
6736 sub restore_update_config_line {
6737     my ($cookie, $map, $line, $unique) = @_;
6738
6739     return '' if $line =~ m/^\#qmdump\#/;
6740     return '' if $line =~ m/^\#vzdump\#/;
6741     return '' if $line =~ m/^lock:/;
6742     return '' if $line =~ m/^unused\d+:/;
6743     return '' if $line =~ m/^parent:/;
6744
6745     my $res = '';
6746
6747     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6748     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6749         # try to convert old 1.X settings
6750         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6751         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6752             my ($model, $macaddr) = split(/\=/, $devconfig);
6753             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6754             my $net = {
6755                 model => $model,
6756                 bridge => "vmbr$ind",
6757                 macaddr => $macaddr,
6758             };
6759             my $netstr = print_net($net);
6760
6761             $res .= "net$cookie->{netcount}: $netstr\n";
6762             $cookie->{netcount}++;
6763         }
6764     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6765         my ($id, $netstr) = ($1, $2);
6766         my $net = parse_net($netstr);
6767         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6768         $netstr = print_net($net);
6769         $res .= "$id: $netstr\n";
6770     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6771         my $virtdev = $1;
6772         my $value = $3;
6773         my $di = parse_drive($virtdev, $value);
6774         if (defined($di->{backup}) && !$di->{backup}) {
6775             $res .= "#$line";
6776         } elsif ($map->{$virtdev}) {
6777             delete $di->{format}; # format can change on restore
6778             $di->{file} = $map->{$virtdev};
6779             $value = print_drive($di);
6780             $res .= "$virtdev: $value\n";
6781         } else {
6782             $res .= $line;
6783         }
6784     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6785         my $vmgenid = $1;
6786         if ($vmgenid ne '0') {
6787             # always generate a new vmgenid if there was a valid one setup
6788             $vmgenid = generate_uuid();
6789         }
6790         $res .= "vmgenid: $vmgenid\n";
6791     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6792         my ($uuid, $uuid_str);
6793         UUID::generate($uuid);
6794         UUID::unparse($uuid, $uuid_str);
6795         my $smbios1 = parse_smbios1($2);
6796         $smbios1->{uuid} = $uuid_str;
6797         $res .= $1.print_smbios1($smbios1)."\n";
6798     } else {
6799         $res .= $line;
6800     }
6801
6802     return $res;
6803 }
6804
6805 my $restore_deactivate_volumes = sub {
6806     my ($storecfg, $virtdev_hash) = @_;
6807
6808     my $vollist = [];
6809     for my $dev (values $virtdev_hash->%*) {
6810         push $vollist->@*, $dev->{volid} if $dev->{volid};
6811     }
6812
6813     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6814     print STDERR $@ if $@;
6815 };
6816
6817 my $restore_destroy_volumes = sub {
6818     my ($storecfg, $virtdev_hash) = @_;
6819
6820     for my $dev (values $virtdev_hash->%*) {
6821         my $volid = $dev->{volid} or next;
6822         eval {
6823             PVE::Storage::vdisk_free($storecfg, $volid);
6824             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6825         };
6826         print STDERR "unable to cleanup '$volid' - $@" if $@;
6827     }
6828 };
6829
6830 my $restore_merge_config = sub {
6831     my ($filename, $backup_conf_raw, $override_conf) = @_;
6832
6833     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6834     for my $key (keys $override_conf->%*) {
6835         $backup_conf->{$key} = $override_conf->{$key};
6836     }
6837
6838     return $backup_conf;
6839 };
6840
6841 sub scan_volids {
6842     my ($cfg, $vmid) = @_;
6843
6844     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6845
6846     my $volid_hash = {};
6847     foreach my $storeid (keys %$info) {
6848         foreach my $item (@{$info->{$storeid}}) {
6849             next if !($item->{volid} && $item->{size});
6850             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6851             $volid_hash->{$item->{volid}} = $item;
6852         }
6853     }
6854
6855     return $volid_hash;
6856 }
6857
6858 sub update_disk_config {
6859     my ($vmid, $conf, $volid_hash) = @_;
6860
6861     my $changes;
6862     my $prefix = "VM $vmid";
6863
6864     # used and unused disks
6865     my $referenced = {};
6866
6867     # Note: it is allowed to define multiple storages with same path (alias), so
6868     # we need to check both 'volid' and real 'path' (two different volid can point
6869     # to the same path).
6870
6871     my $referencedpath = {};
6872
6873     # update size info
6874     PVE::QemuConfig->foreach_volume($conf, sub {
6875         my ($opt, $drive) = @_;
6876
6877         my $volid = $drive->{file};
6878         return if !$volid;
6879         my $volume = $volid_hash->{$volid};
6880
6881         # mark volid as "in-use" for next step
6882         $referenced->{$volid} = 1;
6883         if ($volume && (my $path = $volume->{path})) {
6884             $referencedpath->{$path} = 1;
6885         }
6886
6887         return if drive_is_cdrom($drive);
6888         return if !$volume;
6889
6890         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6891         if (defined($updated)) {
6892             $changes = 1;
6893             $conf->{$opt} = print_drive($updated);
6894             print "$prefix ($opt): $msg\n";
6895         }
6896     });
6897
6898     # remove 'unusedX' entry if volume is used
6899     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6900         my ($opt, $drive) = @_;
6901
6902         my $volid = $drive->{file};
6903         return if !$volid;
6904
6905         my $path;
6906         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6907         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6908             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6909             $changes = 1;
6910             delete $conf->{$opt};
6911         }
6912
6913         $referenced->{$volid} = 1;
6914         $referencedpath->{$path} = 1 if $path;
6915     });
6916
6917     foreach my $volid (sort keys %$volid_hash) {
6918         next if $volid =~ m/vm-$vmid-state-/;
6919         next if $referenced->{$volid};
6920         my $path = $volid_hash->{$volid}->{path};
6921         next if !$path; # just to be sure
6922         next if $referencedpath->{$path};
6923         $changes = 1;
6924         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6925         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6926         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6927     }
6928
6929     return $changes;
6930 }
6931
6932 sub rescan {
6933     my ($vmid, $nolock, $dryrun) = @_;
6934
6935     my $cfg = PVE::Storage::config();
6936
6937     print "rescan volumes...\n";
6938     my $volid_hash = scan_volids($cfg, $vmid);
6939
6940     my $updatefn =  sub {
6941         my ($vmid) = @_;
6942
6943         my $conf = PVE::QemuConfig->load_config($vmid);
6944
6945         PVE::QemuConfig->check_lock($conf);
6946
6947         my $vm_volids = {};
6948         foreach my $volid (keys %$volid_hash) {
6949             my $info = $volid_hash->{$volid};
6950             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6951         }
6952
6953         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6954
6955         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6956     };
6957
6958     if (defined($vmid)) {
6959         if ($nolock) {
6960             &$updatefn($vmid);
6961         } else {
6962             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6963         }
6964     } else {
6965         my $vmlist = config_list();
6966         foreach my $vmid (keys %$vmlist) {
6967             if ($nolock) {
6968                 &$updatefn($vmid);
6969             } else {
6970                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6971             }
6972         }
6973     }
6974 }
6975
6976 sub restore_proxmox_backup_archive {
6977     my ($archive, $vmid, $user, $options) = @_;
6978
6979     my $storecfg = PVE::Storage::config();
6980
6981     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6982     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6983
6984     my $fingerprint = $scfg->{fingerprint};
6985     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6986
6987     my $repo = PVE::PBSClient::get_repository($scfg);
6988     my $namespace = $scfg->{namespace};
6989
6990     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6991     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6992     local $ENV{PBS_PASSWORD} = $password;
6993     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6994
6995     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6996         PVE::Storage::parse_volname($storecfg, $archive);
6997
6998     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6999
7000     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7001
7002     my $tmpdir = "/var/tmp/vzdumptmp$$";
7003     rmtree $tmpdir;
7004     mkpath $tmpdir;
7005
7006     my $conffile = PVE::QemuConfig->config_file($vmid);
7007      # disable interrupts (always do cleanups)
7008     local $SIG{INT} =
7009         local $SIG{TERM} =
7010         local $SIG{QUIT} =
7011         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7012
7013     # Note: $oldconf is undef if VM does not exists
7014     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7015     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7016     my $new_conf_raw = '';
7017
7018     my $rpcenv = PVE::RPCEnvironment::get();
7019     my $devinfo = {}; # info about drives included in backup
7020     my $virtdev_hash = {}; # info about allocated drives
7021
7022     eval {
7023         # enable interrupts
7024         local $SIG{INT} =
7025             local $SIG{TERM} =
7026             local $SIG{QUIT} =
7027             local $SIG{HUP} =
7028             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7029
7030         my $cfgfn = "$tmpdir/qemu-server.conf";
7031         my $firewall_config_fn = "$tmpdir/fw.conf";
7032         my $index_fn = "$tmpdir/index.json";
7033
7034         my $cmd = "restore";
7035
7036         my $param = [$pbs_backup_name, "index.json", $index_fn];
7037         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7038         my $index = PVE::Tools::file_get_contents($index_fn);
7039         $index = decode_json($index);
7040
7041         foreach my $info (@{$index->{files}}) {
7042             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7043                 my $devname = $1;
7044                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7045                     $devinfo->{$devname}->{size} = $1;
7046                 } else {
7047                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7048                 }
7049             }
7050         }
7051
7052         my $is_qemu_server_backup = scalar(
7053             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7054         );
7055         if (!$is_qemu_server_backup) {
7056             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7057         }
7058         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7059
7060         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7061         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7062
7063         if ($has_firewall_config) {
7064             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7065             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7066
7067             my $pve_firewall_dir = '/etc/pve/firewall';
7068             mkdir $pve_firewall_dir; # make sure the dir exists
7069             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7070         }
7071
7072         my $fh = IO::File->new($cfgfn, "r") ||
7073             die "unable to read qemu-server.conf - $!\n";
7074
7075         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7076
7077         # fixme: rate limit?
7078
7079         # create empty/temp config
7080         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7081
7082         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7083
7084         # allocate volumes
7085         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7086
7087         foreach my $virtdev (sort keys %$virtdev_hash) {
7088             my $d = $virtdev_hash->{$virtdev};
7089             next if $d->{is_cloudinit}; # no need to restore cloudinit
7090
7091             # this fails if storage is unavailable
7092             my $volid = $d->{volid};
7093             my $path = PVE::Storage::path($storecfg, $volid);
7094
7095             # for live-restore we only want to preload the efidisk and TPM state
7096             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7097
7098             my @ns_arg;
7099             if (defined(my $ns = $scfg->{namespace})) {
7100                 @ns_arg = ('--ns', $ns);
7101             }
7102
7103             my $pbs_restore_cmd = [
7104                 '/usr/bin/pbs-restore',
7105                 '--repository', $repo,
7106                 @ns_arg,
7107                 $pbs_backup_name,
7108                 "$d->{devname}.img.fidx",
7109                 $path,
7110                 '--verbose',
7111                 ];
7112
7113             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7114             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7115
7116             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7117                 push @$pbs_restore_cmd, '--skip-zero';
7118             }
7119
7120             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7121             print "restore proxmox backup image: $dbg_cmdstring\n";
7122             run_command($pbs_restore_cmd);
7123         }
7124
7125         $fh->seek(0, 0) || die "seek failed - $!\n";
7126
7127         my $cookie = { netcount => 0 };
7128         while (defined(my $line = <$fh>)) {
7129             $new_conf_raw .= restore_update_config_line(
7130                 $cookie,
7131                 $map,
7132                 $line,
7133                 $options->{unique},
7134             );
7135         }
7136
7137         $fh->close();
7138     };
7139     my $err = $@;
7140
7141     if ($err || !$options->{live}) {
7142         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7143     }
7144
7145     rmtree $tmpdir;
7146
7147     if ($err) {
7148         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7149         die $err;
7150     }
7151
7152     if ($options->{live}) {
7153         # keep lock during live-restore
7154         $new_conf_raw .= "\nlock: create";
7155     }
7156
7157     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7158     PVE::QemuConfig->write_config($vmid, $new_conf);
7159
7160     eval { rescan($vmid, 1); };
7161     warn $@ if $@;
7162
7163     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7164
7165     if ($options->{live}) {
7166         # enable interrupts
7167         local $SIG{INT} =
7168             local $SIG{TERM} =
7169             local $SIG{QUIT} =
7170             local $SIG{HUP} =
7171             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7172
7173         my $conf = PVE::QemuConfig->load_config($vmid);
7174         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7175
7176         # these special drives are already restored before start
7177         delete $devinfo->{'drive-efidisk0'};
7178         delete $devinfo->{'drive-tpmstate0-backup'};
7179
7180         my $pbs_opts = {
7181             repo => $repo,
7182             keyfile => $keyfile,
7183             snapshot => $pbs_backup_name,
7184             namespace => $namespace,
7185         };
7186         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7187
7188         PVE::QemuConfig->remove_lock($vmid, "create");
7189     }
7190 }
7191
7192 sub pbs_live_restore {
7193     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7194
7195     print "starting VM for live-restore\n";
7196     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7197
7198     my $pbs_backing = {};
7199     for my $ds (keys %$restored_disks) {
7200         $ds =~ m/^drive-(.*)$/;
7201         my $confname = $1;
7202         $pbs_backing->{$confname} = {
7203             repository => $opts->{repo},
7204             snapshot => $opts->{snapshot},
7205             archive => "$ds.img.fidx",
7206         };
7207         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7208         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7209
7210         my $drive = parse_drive($confname, $conf->{$confname});
7211         print "restoring '$ds' to '$drive->{file}'\n";
7212     }
7213
7214     my $drives_streamed = 0;
7215     eval {
7216         # make sure HA doesn't interrupt our restore by stopping the VM
7217         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7218             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7219         }
7220
7221         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7222         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7223         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7224
7225         my $qmeventd_fd = register_qmeventd_handle($vmid);
7226
7227         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7228         # this will effectively collapse the backing image chain consisting of
7229         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7230         # removes itself once all backing images vanish with 'auto-remove=on')
7231         my $jobs = {};
7232         for my $ds (sort keys %$restored_disks) {
7233             my $job_id = "restore-$ds";
7234             mon_cmd($vmid, 'block-stream',
7235                 'job-id' => $job_id,
7236                 device => "$ds",
7237             );
7238             $jobs->{$job_id} = {};
7239         }
7240
7241         mon_cmd($vmid, 'cont');
7242         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7243
7244         print "restore-drive jobs finished successfully, removing all tracking block devices"
7245             ." to disconnect from Proxmox Backup Server\n";
7246
7247         for my $ds (sort keys %$restored_disks) {
7248             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7249         }
7250
7251         close($qmeventd_fd);
7252     };
7253
7254     my $err = $@;
7255
7256     if ($err) {
7257         warn "An error occurred during live-restore: $err\n";
7258         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7259         die "live-restore failed\n";
7260     }
7261 }
7262
7263 sub restore_vma_archive {
7264     my ($archive, $vmid, $user, $opts, $comp) = @_;
7265
7266     my $readfrom = $archive;
7267
7268     my $cfg = PVE::Storage::config();
7269     my $commands = [];
7270     my $bwlimit = $opts->{bwlimit};
7271
7272     my $dbg_cmdstring = '';
7273     my $add_pipe = sub {
7274         my ($cmd) = @_;
7275         push @$commands, $cmd;
7276         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7277         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7278         $readfrom = '-';
7279     };
7280
7281     my $input = undef;
7282     if ($archive eq '-') {
7283         $input = '<&STDIN';
7284     } else {
7285         # If we use a backup from a PVE defined storage we also consider that
7286         # storage's rate limit:
7287         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7288         if (defined($volid)) {
7289             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7290             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7291             if ($readlimit) {
7292                 print STDERR "applying read rate limit: $readlimit\n";
7293                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7294                 $add_pipe->($cstream);
7295             }
7296         }
7297     }
7298
7299     if ($comp) {
7300         my $info = PVE::Storage::decompressor_info('vma', $comp);
7301         my $cmd = $info->{decompressor};
7302         push @$cmd, $readfrom;
7303         $add_pipe->($cmd);
7304     }
7305
7306     my $tmpdir = "/var/tmp/vzdumptmp$$";
7307     rmtree $tmpdir;
7308
7309     # disable interrupts (always do cleanups)
7310     local $SIG{INT} =
7311         local $SIG{TERM} =
7312         local $SIG{QUIT} =
7313         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7314
7315     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7316     POSIX::mkfifo($mapfifo, 0600);
7317     my $fifofh;
7318     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7319
7320     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7321
7322     my $oldtimeout;
7323     my $timeout = 5;
7324
7325     my $devinfo = {}; # info about drives included in backup
7326     my $virtdev_hash = {}; # info about allocated drives
7327
7328     my $rpcenv = PVE::RPCEnvironment::get();
7329
7330     my $conffile = PVE::QemuConfig->config_file($vmid);
7331
7332     # Note: $oldconf is undef if VM does not exist
7333     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7334     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7335     my $new_conf_raw = '';
7336
7337     my %storage_limits;
7338
7339     my $print_devmap = sub {
7340         my $cfgfn = "$tmpdir/qemu-server.conf";
7341
7342         # we can read the config - that is already extracted
7343         my $fh = IO::File->new($cfgfn, "r") ||
7344             die "unable to read qemu-server.conf - $!\n";
7345
7346         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7347         if (-f $fwcfgfn) {
7348             my $pve_firewall_dir = '/etc/pve/firewall';
7349             mkdir $pve_firewall_dir; # make sure the dir exists
7350             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7351         }
7352
7353         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7354
7355         foreach my $info (values %{$virtdev_hash}) {
7356             my $storeid = $info->{storeid};
7357             next if defined($storage_limits{$storeid});
7358
7359             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7360             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7361             $storage_limits{$storeid} = $limit * 1024;
7362         }
7363
7364         foreach my $devname (keys %$devinfo) {
7365             die "found no device mapping information for device '$devname'\n"
7366                 if !$devinfo->{$devname}->{virtdev};
7367         }
7368
7369         # create empty/temp config
7370         if ($oldconf) {
7371             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7372             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7373         }
7374
7375         # allocate volumes
7376         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7377
7378         # print restore information to $fifofh
7379         foreach my $virtdev (sort keys %$virtdev_hash) {
7380             my $d = $virtdev_hash->{$virtdev};
7381             next if $d->{is_cloudinit}; # no need to restore cloudinit
7382
7383             my $storeid = $d->{storeid};
7384             my $volid = $d->{volid};
7385
7386             my $map_opts = '';
7387             if (my $limit = $storage_limits{$storeid}) {
7388                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7389             }
7390
7391             my $write_zeros = 1;
7392             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7393                 $write_zeros = 0;
7394             }
7395
7396             my $path = PVE::Storage::path($cfg, $volid);
7397
7398             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7399
7400             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7401         }
7402
7403         $fh->seek(0, 0) || die "seek failed - $!\n";
7404
7405         my $cookie = { netcount => 0 };
7406         while (defined(my $line = <$fh>)) {
7407             $new_conf_raw .= restore_update_config_line(
7408                 $cookie,
7409                 $map,
7410                 $line,
7411                 $opts->{unique},
7412             );
7413         }
7414
7415         $fh->close();
7416     };
7417
7418     eval {
7419         # enable interrupts
7420         local $SIG{INT} =
7421             local $SIG{TERM} =
7422             local $SIG{QUIT} =
7423             local $SIG{HUP} =
7424             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7425         local $SIG{ALRM} = sub { die "got timeout\n"; };
7426
7427         $oldtimeout = alarm($timeout);
7428
7429         my $parser = sub {
7430             my $line = shift;
7431
7432             print "$line\n";
7433
7434             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7435                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7436                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7437             } elsif ($line =~ m/^CTIME: /) {
7438                 # we correctly received the vma config, so we can disable
7439                 # the timeout now for disk allocation (set to 10 minutes, so
7440                 # that we always timeout if something goes wrong)
7441                 alarm(600);
7442                 &$print_devmap();
7443                 print $fifofh "done\n";
7444                 my $tmp = $oldtimeout || 0;
7445                 $oldtimeout = undef;
7446                 alarm($tmp);
7447                 close($fifofh);
7448                 $fifofh = undef;
7449             }
7450         };
7451
7452         print "restore vma archive: $dbg_cmdstring\n";
7453         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7454     };
7455     my $err = $@;
7456
7457     alarm($oldtimeout) if $oldtimeout;
7458
7459     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7460
7461     close($fifofh) if $fifofh;
7462     unlink $mapfifo;
7463     rmtree $tmpdir;
7464
7465     if ($err) {
7466         $restore_destroy_volumes->($cfg, $virtdev_hash);
7467         die $err;
7468     }
7469
7470     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7471     PVE::QemuConfig->write_config($vmid, $new_conf);
7472
7473     eval { rescan($vmid, 1); };
7474     warn $@ if $@;
7475
7476     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7477 }
7478
7479 sub restore_tar_archive {
7480     my ($archive, $vmid, $user, $opts) = @_;
7481
7482     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7483         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7484         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7485     }
7486
7487     if ($archive ne '-') {
7488         my $firstfile = tar_archive_read_firstfile($archive);
7489         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7490             if $firstfile ne 'qemu-server.conf';
7491     }
7492
7493     my $storecfg = PVE::Storage::config();
7494
7495     # avoid zombie disks when restoring over an existing VM -> cleanup first
7496     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7497     # skiplock=1 because qmrestore has set the 'create' lock itself already
7498     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7499     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7500
7501     my $tocmd = "/usr/lib/qemu-server/qmextract";
7502
7503     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7504     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7505     $tocmd .= ' --prealloc' if $opts->{prealloc};
7506     $tocmd .= ' --info' if $opts->{info};
7507
7508     # tar option "xf" does not autodetect compression when read from STDIN,
7509     # so we pipe to zcat
7510     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7511         PVE::Tools::shellquote("--to-command=$tocmd");
7512
7513     my $tmpdir = "/var/tmp/vzdumptmp$$";
7514     mkpath $tmpdir;
7515
7516     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7517     local $ENV{VZDUMP_VMID} = $vmid;
7518     local $ENV{VZDUMP_USER} = $user;
7519
7520     my $conffile = PVE::QemuConfig->config_file($vmid);
7521     my $new_conf_raw = '';
7522
7523     # disable interrupts (always do cleanups)
7524     local $SIG{INT} =
7525         local $SIG{TERM} =
7526         local $SIG{QUIT} =
7527         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7528
7529     eval {
7530         # enable interrupts
7531         local $SIG{INT} =
7532             local $SIG{TERM} =
7533             local $SIG{QUIT} =
7534             local $SIG{HUP} =
7535             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7536
7537         if ($archive eq '-') {
7538             print "extracting archive from STDIN\n";
7539             run_command($cmd, input => "<&STDIN");
7540         } else {
7541             print "extracting archive '$archive'\n";
7542             run_command($cmd);
7543         }
7544
7545         return if $opts->{info};
7546
7547         # read new mapping
7548         my $map = {};
7549         my $statfile = "$tmpdir/qmrestore.stat";
7550         if (my $fd = IO::File->new($statfile, "r")) {
7551             while (defined (my $line = <$fd>)) {
7552                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7553                     $map->{$1} = $2 if $1;
7554                 } else {
7555                     print STDERR "unable to parse line in statfile - $line\n";
7556                 }
7557             }
7558             $fd->close();
7559         }
7560
7561         my $confsrc = "$tmpdir/qemu-server.conf";
7562
7563         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7564
7565         my $cookie = { netcount => 0 };
7566         while (defined (my $line = <$srcfd>)) {
7567             $new_conf_raw .= restore_update_config_line(
7568                 $cookie,
7569                 $map,
7570                 $line,
7571                 $opts->{unique},
7572             );
7573         }
7574
7575         $srcfd->close();
7576     };
7577     if (my $err = $@) {
7578         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7579         die $err;
7580     }
7581
7582     rmtree $tmpdir;
7583
7584     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7585
7586     PVE::Cluster::cfs_update(); # make sure we read new file
7587
7588     eval { rescan($vmid, 1); };
7589     warn $@ if $@;
7590 };
7591
7592 sub foreach_storage_used_by_vm {
7593     my ($conf, $func) = @_;
7594
7595     my $sidhash = {};
7596
7597     PVE::QemuConfig->foreach_volume($conf, sub {
7598         my ($ds, $drive) = @_;
7599         return if drive_is_cdrom($drive);
7600
7601         my $volid = $drive->{file};
7602
7603         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7604         $sidhash->{$sid} = $sid if $sid;
7605     });
7606
7607     foreach my $sid (sort keys %$sidhash) {
7608         &$func($sid);
7609     }
7610 }
7611
7612 my $qemu_snap_storage = {
7613     rbd => 1,
7614 };
7615 sub do_snapshots_with_qemu {
7616     my ($storecfg, $volid, $deviceid) = @_;
7617
7618     return if $deviceid =~ m/tpmstate0/;
7619
7620     my $storage_name = PVE::Storage::parse_volume_id($volid);
7621     my $scfg = $storecfg->{ids}->{$storage_name};
7622     die "could not find storage '$storage_name'\n" if !defined($scfg);
7623
7624     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7625         return 1;
7626     }
7627
7628     if ($volid =~ m/\.(qcow2|qed)$/){
7629         return 1;
7630     }
7631
7632     return;
7633 }
7634
7635 sub qga_check_running {
7636     my ($vmid, $nowarn) = @_;
7637
7638     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7639     if ($@) {
7640         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7641         return 0;
7642     }
7643     return 1;
7644 }
7645
7646 sub template_create {
7647     my ($vmid, $conf, $disk) = @_;
7648
7649     my $storecfg = PVE::Storage::config();
7650
7651     PVE::QemuConfig->foreach_volume($conf, sub {
7652         my ($ds, $drive) = @_;
7653
7654         return if drive_is_cdrom($drive);
7655         return if $disk && $ds ne $disk;
7656
7657         my $volid = $drive->{file};
7658         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7659
7660         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7661         $drive->{file} = $voliddst;
7662         $conf->{$ds} = print_drive($drive);
7663         PVE::QemuConfig->write_config($vmid, $conf);
7664     });
7665 }
7666
7667 sub convert_iscsi_path {
7668     my ($path) = @_;
7669
7670     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7671         my $portal = $1;
7672         my $target = $2;
7673         my $lun = $3;
7674
7675         my $initiator_name = get_initiator_name();
7676
7677         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7678                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7679     }
7680
7681     die "cannot convert iscsi path '$path', unkown format\n";
7682 }
7683
7684 sub qemu_img_convert {
7685     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7686
7687     my $storecfg = PVE::Storage::config();
7688     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7689     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7690
7691     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7692
7693     my $cachemode;
7694     my $src_path;
7695     my $src_is_iscsi = 0;
7696     my $src_format;
7697
7698     if ($src_storeid) {
7699         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7700         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7701         $src_format = qemu_img_format($src_scfg, $src_volname);
7702         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7703         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7704         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7705     } elsif (-f $src_volid || -b $src_volid) {
7706         $src_path = $src_volid;
7707         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7708             $src_format = $1;
7709         }
7710     }
7711
7712     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7713
7714     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7715     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7716     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7717     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7718
7719     my $cmd = [];
7720     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7721     push @$cmd, '-l', "snapshot.name=$snapname"
7722         if $snapname && $src_format && $src_format eq "qcow2";
7723     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7724     push @$cmd, '-T', $cachemode if defined($cachemode);
7725     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7726
7727     if ($src_is_iscsi) {
7728         push @$cmd, '--image-opts';
7729         $src_path = convert_iscsi_path($src_path);
7730     } elsif ($src_format) {
7731         push @$cmd, '-f', $src_format;
7732     }
7733
7734     if ($dst_is_iscsi) {
7735         push @$cmd, '--target-image-opts';
7736         $dst_path = convert_iscsi_path($dst_path);
7737     } else {
7738         push @$cmd, '-O', $dst_format;
7739     }
7740
7741     push @$cmd, $src_path;
7742
7743     if (!$dst_is_iscsi && $is_zero_initialized) {
7744         push @$cmd, "zeroinit:$dst_path";
7745     } else {
7746         push @$cmd, $dst_path;
7747     }
7748
7749     my $parser = sub {
7750         my $line = shift;
7751         if($line =~ m/\((\S+)\/100\%\)/){
7752             my $percent = $1;
7753             my $transferred = int($size * $percent / 100);
7754             my $total_h = render_bytes($size, 1);
7755             my $transferred_h = render_bytes($transferred, 1);
7756
7757             print "transferred $transferred_h of $total_h ($percent%)\n";
7758         }
7759
7760     };
7761
7762     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7763     my $err = $@;
7764     die "copy failed: $err" if $err;
7765 }
7766
7767 sub qemu_img_format {
7768     my ($scfg, $volname) = @_;
7769
7770     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7771         return $1;
7772     } else {
7773         return "raw";
7774     }
7775 }
7776
7777 sub qemu_drive_mirror {
7778     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7779
7780     $jobs = {} if !$jobs;
7781
7782     my $qemu_target;
7783     my $format;
7784     $jobs->{"drive-$drive"} = {};
7785
7786     if ($dst_volid =~ /^nbd:/) {
7787         $qemu_target = $dst_volid;
7788         $format = "nbd";
7789     } else {
7790         my $storecfg = PVE::Storage::config();
7791         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7792
7793         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7794
7795         $format = qemu_img_format($dst_scfg, $dst_volname);
7796
7797         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7798
7799         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7800     }
7801
7802     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7803     $opts->{format} = $format if $format;
7804
7805     if (defined($src_bitmap)) {
7806         $opts->{sync} = 'incremental';
7807         $opts->{bitmap} = $src_bitmap;
7808         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7809     }
7810
7811     if (defined($bwlimit)) {
7812         $opts->{speed} = $bwlimit * 1024;
7813         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7814     } else {
7815         print "drive mirror is starting for drive-$drive\n";
7816     }
7817
7818     # if a job already runs for this device we get an error, catch it for cleanup
7819     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7820     if (my $err = $@) {
7821         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7822         warn "$@\n" if $@;
7823         die "mirroring error: $err\n";
7824     }
7825
7826     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7827 }
7828
7829 # $completion can be either
7830 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7831 # 'cancel': wait until all jobs are ready, block-job-cancel them
7832 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7833 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7834 sub qemu_drive_mirror_monitor {
7835     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7836
7837     $completion //= 'complete';
7838     $op //= "mirror";
7839
7840     eval {
7841         my $err_complete = 0;
7842
7843         my $starttime = time ();
7844         while (1) {
7845             die "block job ('$op') timed out\n" if $err_complete > 300;
7846
7847             my $stats = mon_cmd($vmid, "query-block-jobs");
7848             my $ctime = time();
7849
7850             my $running_jobs = {};
7851             for my $stat (@$stats) {
7852                 next if $stat->{type} ne $op;
7853                 $running_jobs->{$stat->{device}} = $stat;
7854             }
7855
7856             my $readycounter = 0;
7857
7858             for my $job_id (sort keys %$jobs) {
7859                 my $job = $running_jobs->{$job_id};
7860
7861                 my $vanished = !defined($job);
7862                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7863                 if($complete || ($vanished && $completion eq 'auto')) {
7864                     print "$job_id: $op-job finished\n";
7865                     delete $jobs->{$job_id};
7866                     next;
7867                 }
7868
7869                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7870
7871                 my $busy = $job->{busy};
7872                 my $ready = $job->{ready};
7873                 if (my $total = $job->{len}) {
7874                     my $transferred = $job->{offset} || 0;
7875                     my $remaining = $total - $transferred;
7876                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7877
7878                     my $duration = $ctime - $starttime;
7879                     my $total_h = render_bytes($total, 1);
7880                     my $transferred_h = render_bytes($transferred, 1);
7881
7882                     my $status = sprintf(
7883                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7884                         render_duration($duration),
7885                     );
7886
7887                     if ($ready) {
7888                         if ($busy) {
7889                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7890                         } else {
7891                             $status .= ", ready";
7892                         }
7893                     }
7894                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7895                     $jobs->{$job_id}->{ready} = $ready;
7896                 }
7897
7898                 $readycounter++ if $job->{ready};
7899             }
7900
7901             last if scalar(keys %$jobs) == 0;
7902
7903             if ($readycounter == scalar(keys %$jobs)) {
7904                 print "all '$op' jobs are ready\n";
7905
7906                 # do the complete later (or has already been done)
7907                 last if $completion eq 'skip' || $completion eq 'auto';
7908
7909                 if ($vmiddst && $vmiddst != $vmid) {
7910                     my $agent_running = $qga && qga_check_running($vmid);
7911                     if ($agent_running) {
7912                         print "freeze filesystem\n";
7913                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7914                         warn $@ if $@;
7915                     } else {
7916                         print "suspend vm\n";
7917                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7918                         warn $@ if $@;
7919                     }
7920
7921                     # if we clone a disk for a new target vm, we don't switch the disk
7922                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7923
7924                     if ($agent_running) {
7925                         print "unfreeze filesystem\n";
7926                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7927                         warn $@ if $@;
7928                     } else {
7929                         print "resume vm\n";
7930                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7931                         warn $@ if $@;
7932                     }
7933
7934                     last;
7935                 } else {
7936
7937                     for my $job_id (sort keys %$jobs) {
7938                         # try to switch the disk if source and destination are on the same guest
7939                         print "$job_id: Completing block job_id...\n";
7940
7941                         my $op;
7942                         if ($completion eq 'complete') {
7943                             $op = 'block-job-complete';
7944                         } elsif ($completion eq 'cancel') {
7945                             $op = 'block-job-cancel';
7946                         } else {
7947                             die "invalid completion value: $completion\n";
7948                         }
7949                         eval { mon_cmd($vmid, $op, device => $job_id) };
7950                         if ($@ =~ m/cannot be completed/) {
7951                             print "$job_id: block job cannot be completed, trying again.\n";
7952                             $err_complete++;
7953                         }else {
7954                             print "$job_id: Completed successfully.\n";
7955                             $jobs->{$job_id}->{complete} = 1;
7956                         }
7957                     }
7958                 }
7959             }
7960             sleep 1;
7961         }
7962     };
7963     my $err = $@;
7964
7965     if ($err) {
7966         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7967         die "block job ($op) error: $err";
7968     }
7969 }
7970
7971 sub qemu_blockjobs_cancel {
7972     my ($vmid, $jobs) = @_;
7973
7974     foreach my $job (keys %$jobs) {
7975         print "$job: Cancelling block job\n";
7976         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7977         $jobs->{$job}->{cancel} = 1;
7978     }
7979
7980     while (1) {
7981         my $stats = mon_cmd($vmid, "query-block-jobs");
7982
7983         my $running_jobs = {};
7984         foreach my $stat (@$stats) {
7985             $running_jobs->{$stat->{device}} = $stat;
7986         }
7987
7988         foreach my $job (keys %$jobs) {
7989
7990             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7991                 print "$job: Done.\n";
7992                 delete $jobs->{$job};
7993             }
7994         }
7995
7996         last if scalar(keys %$jobs) == 0;
7997
7998         sleep 1;
7999     }
8000 }
8001
8002 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8003 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
8004 my sub clone_disk_check_io_uring {
8005     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8006
8007     return if !$use_drive_mirror;
8008
8009     # Don't complain when not changing storage.
8010     # Assume if it works for the source, it'll work for the target too.
8011     return if $src_storeid eq $dst_storeid;
8012
8013     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8014     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8015
8016     my $cache_direct = drive_uses_cache_direct($src_drive);
8017
8018     my $src_uses_io_uring;
8019     if ($src_drive->{aio}) {
8020         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8021     } else {
8022         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8023     }
8024
8025     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8026         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8027 }
8028
8029 sub clone_disk {
8030     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8031
8032     my ($vmid, $running) = $source->@{qw(vmid running)};
8033     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8034
8035     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8036     my ($storage, $format) = $dest->@{qw(storage format)};
8037
8038     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8039
8040     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8041         die "cloning from/to EFI disk requires EFI disk\n"
8042             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8043         die "cloning from/to TPM state requires TPM state\n"
8044             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8045
8046         # This would lead to two device nodes in QEMU pointing to the same backing image!
8047         die "cannot change drive name when cloning disk from/to the same VM\n"
8048             if $use_drive_mirror && $vmid == $newvmid;
8049     }
8050
8051     die "cannot move TPM state while VM is running\n"
8052         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8053
8054     my $newvolid;
8055
8056     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8057     print "$src_drivename " if $src_drivename;
8058     print "($drive->{file})\n";
8059
8060     if (!$full) {
8061         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8062         push @$newvollist, $newvolid;
8063     } else {
8064         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8065         my $storeid = $storage || $src_storeid;
8066
8067         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8068
8069         my $name = undef;
8070         my $size = undef;
8071         if (drive_is_cloudinit($drive)) {
8072             $name = "vm-$newvmid-cloudinit";
8073             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8074             if ($scfg->{path}) {
8075                 $name .= ".$dst_format";
8076             }
8077             $snapname = undef;
8078             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8079         } elsif ($dst_drivename eq 'efidisk0') {
8080             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8081         } elsif ($dst_drivename eq 'tpmstate0') {
8082             $dst_format = 'raw';
8083             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8084         } else {
8085             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8086
8087             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8088         }
8089         $newvolid = PVE::Storage::vdisk_alloc(
8090             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8091         );
8092         push @$newvollist, $newvolid;
8093
8094         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8095
8096         if (drive_is_cloudinit($drive)) {
8097             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8098             # if this is the case, we have to complete any block-jobs still there from
8099             # previous drive-mirrors
8100             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8101                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8102             }
8103             goto no_data_clone;
8104         }
8105
8106         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8107         if ($use_drive_mirror) {
8108             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8109                 $completion, $qga, $bwlimit);
8110         } else {
8111             if ($dst_drivename eq 'efidisk0') {
8112                 # the relevant data on the efidisk may be smaller than the source
8113                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8114                 # that is given by the OVMF_VARS.fd
8115                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8116                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8117
8118                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8119
8120                 # better for Ceph if block size is not too small, see bug #3324
8121                 my $bs = 1024*1024;
8122
8123                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8124
8125                 if ($src_format eq 'qcow2' && $snapname) {
8126                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8127                         if !min_version(kvm_user_version(), 6, 2);
8128                     push $cmd->@*, '-l', $snapname;
8129                 }
8130                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8131                 run_command($cmd);
8132             } else {
8133                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8134             }
8135         }
8136     }
8137
8138 no_data_clone:
8139     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8140
8141     my $disk = dclone($drive);
8142     delete $disk->{format};
8143     $disk->{file} = $newvolid;
8144     $disk->{size} = $size if defined($size);
8145
8146     return $disk;
8147 }
8148
8149 sub get_running_qemu_version {
8150     my ($vmid) = @_;
8151     my $res = mon_cmd($vmid, "query-version");
8152     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8153 }
8154
8155 sub qemu_use_old_bios_files {
8156     my ($machine_type) = @_;
8157
8158     return if !$machine_type;
8159
8160     my $use_old_bios_files = undef;
8161
8162     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8163         $machine_type = $1;
8164         $use_old_bios_files = 1;
8165     } else {
8166         my $version = extract_version($machine_type, kvm_user_version());
8167         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8168         # load new efi bios files on migration. So this hack is required to allow
8169         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8170         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8171         $use_old_bios_files = !min_version($version, 2, 4);
8172     }
8173
8174     return ($use_old_bios_files, $machine_type);
8175 }
8176
8177 sub get_efivars_size {
8178     my ($conf, $efidisk) = @_;
8179
8180     my $arch = get_vm_arch($conf);
8181     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8182     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8183     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8184     return -s $ovmf_vars;
8185 }
8186
8187 sub update_efidisk_size {
8188     my ($conf) = @_;
8189
8190     return if !defined($conf->{efidisk0});
8191
8192     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8193     $disk->{size} = get_efivars_size($conf);
8194     $conf->{efidisk0} = print_drive($disk);
8195
8196     return;
8197 }
8198
8199 sub update_tpmstate_size {
8200     my ($conf) = @_;
8201
8202     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8203     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8204     $conf->{tpmstate0} = print_drive($disk);
8205 }
8206
8207 sub create_efidisk($$$$$$$) {
8208     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8209
8210     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8211
8212     my $vars_size_b = -s $ovmf_vars;
8213     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8214     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8215     PVE::Storage::activate_volumes($storecfg, [$volid]);
8216
8217     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8218     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8219
8220     return ($volid, $size/1024);
8221 }
8222
8223 sub vm_iothreads_list {
8224     my ($vmid) = @_;
8225
8226     my $res = mon_cmd($vmid, 'query-iothreads');
8227
8228     my $iothreads = {};
8229     foreach my $iothread (@$res) {
8230         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8231     }
8232
8233     return $iothreads;
8234 }
8235
8236 sub scsihw_infos {
8237     my ($conf, $drive) = @_;
8238
8239     my $maxdev = 0;
8240
8241     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8242         $maxdev = 7;
8243     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8244         $maxdev = 1;
8245     } else {
8246         $maxdev = 256;
8247     }
8248
8249     my $controller = int($drive->{index} / $maxdev);
8250     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8251         ? "virtioscsi"
8252         : "scsihw";
8253
8254     return ($maxdev, $controller, $controller_prefix);
8255 }
8256
8257 sub resolve_dst_disk_format {
8258         my ($storecfg, $storeid, $src_volname, $format) = @_;
8259         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8260
8261         if (!$format) {
8262             # if no target format is specified, use the source disk format as hint
8263             if ($src_volname) {
8264                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8265                 $format = qemu_img_format($scfg, $src_volname);
8266             } else {
8267                 return $defFormat;
8268             }
8269         }
8270
8271         # test if requested format is supported - else use default
8272         my $supported = grep { $_ eq $format } @$validFormats;
8273         $format = $defFormat if !$supported;
8274         return $format;
8275 }
8276
8277 # NOTE: if this logic changes, please update docs & possibly gui logic
8278 sub find_vmstate_storage {
8279     my ($conf, $storecfg) = @_;
8280
8281     # first, return storage from conf if set
8282     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8283
8284     my ($target, $shared, $local);
8285
8286     foreach_storage_used_by_vm($conf, sub {
8287         my ($sid) = @_;
8288         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8289         my $dst = $scfg->{shared} ? \$shared : \$local;
8290         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8291     });
8292
8293     # second, use shared storage where VM has at least one disk
8294     # third, use local storage where VM has at least one disk
8295     # fall back to local storage
8296     $target = $shared // $local // 'local';
8297
8298     return $target;
8299 }
8300
8301 sub generate_uuid {
8302     my ($uuid, $uuid_str);
8303     UUID::generate($uuid);
8304     UUID::unparse($uuid, $uuid_str);
8305     return $uuid_str;
8306 }
8307
8308 sub generate_smbios1_uuid {
8309     return "uuid=".generate_uuid();
8310 }
8311
8312 sub nbd_stop {
8313     my ($vmid) = @_;
8314
8315     mon_cmd($vmid, 'nbd-server-stop');
8316 }
8317
8318 sub create_reboot_request {
8319     my ($vmid) = @_;
8320     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8321         or die "failed to create reboot trigger file: $!\n";
8322     close($fh);
8323 }
8324
8325 sub clear_reboot_request {
8326     my ($vmid) = @_;
8327     my $path = "/run/qemu-server/$vmid.reboot";
8328     my $res = 0;
8329
8330     $res = unlink($path);
8331     die "could not remove reboot request for $vmid: $!"
8332         if !$res && $! != POSIX::ENOENT;
8333
8334     return $res;
8335 }
8336
8337 sub bootorder_from_legacy {
8338     my ($conf, $bootcfg) = @_;
8339
8340     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8341     my $bootindex_hash = {};
8342     my $i = 1;
8343     foreach my $o (split(//, $boot)) {
8344         $bootindex_hash->{$o} = $i*100;
8345         $i++;
8346     }
8347
8348     my $bootorder = {};
8349
8350     PVE::QemuConfig->foreach_volume($conf, sub {
8351         my ($ds, $drive) = @_;
8352
8353         if (drive_is_cdrom ($drive, 1)) {
8354             if ($bootindex_hash->{d}) {
8355                 $bootorder->{$ds} = $bootindex_hash->{d};
8356                 $bootindex_hash->{d} += 1;
8357             }
8358         } elsif ($bootindex_hash->{c}) {
8359             $bootorder->{$ds} = $bootindex_hash->{c}
8360                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8361             $bootindex_hash->{c} += 1;
8362         }
8363     });
8364
8365     if ($bootindex_hash->{n}) {
8366         for (my $i = 0; $i < $MAX_NETS; $i++) {
8367             my $netname = "net$i";
8368             next if !$conf->{$netname};
8369             $bootorder->{$netname} = $bootindex_hash->{n};
8370             $bootindex_hash->{n} += 1;
8371         }
8372     }
8373
8374     return $bootorder;
8375 }
8376
8377 # Generate default device list for 'boot: order=' property. Matches legacy
8378 # default boot order, but with explicit device names. This is important, since
8379 # the fallback for when neither 'order' nor the old format is specified relies
8380 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8381 sub get_default_bootdevices {
8382     my ($conf) = @_;
8383
8384     my @ret = ();
8385
8386     # harddisk
8387     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8388     push @ret, $first if $first;
8389
8390     # cdrom
8391     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8392     push @ret, $first if $first;
8393
8394     # network
8395     for (my $i = 0; $i < $MAX_NETS; $i++) {
8396         my $netname = "net$i";
8397         next if !$conf->{$netname};
8398         push @ret, $netname;
8399         last;
8400     }
8401
8402     return \@ret;
8403 }
8404
8405 sub device_bootorder {
8406     my ($conf) = @_;
8407
8408     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8409
8410     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8411
8412     my $bootorder = {};
8413     if (!defined($boot) || $boot->{legacy}) {
8414         $bootorder = bootorder_from_legacy($conf, $boot);
8415     } elsif ($boot->{order}) {
8416         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8417         for my $dev (PVE::Tools::split_list($boot->{order})) {
8418             $bootorder->{$dev} = $i++;
8419         }
8420     }
8421
8422     return $bootorder;
8423 }
8424
8425 sub register_qmeventd_handle {
8426     my ($vmid) = @_;
8427
8428     my $fh;
8429     my $peer = "/var/run/qmeventd.sock";
8430     my $count = 0;
8431
8432     for (;;) {
8433         $count++;
8434         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8435         last if $fh;
8436         if ($! != EINTR && $! != EAGAIN) {
8437             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8438         }
8439         if ($count > 4) {
8440             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8441               . "after $count retries\n";
8442         }
8443         usleep(25000);
8444     }
8445
8446     # send handshake to mark VM as backing up
8447     print $fh to_json({vzdump => {vmid => "$vmid"}});
8448
8449     # return handle to be closed later when inhibit is no longer required
8450     return $fh;
8451 }
8452
8453 # bash completion helper
8454
8455 sub complete_backup_archives {
8456     my ($cmdname, $pname, $cvalue) = @_;
8457
8458     my $cfg = PVE::Storage::config();
8459
8460     my $storeid;
8461
8462     if ($cvalue =~ m/^([^:]+):/) {
8463         $storeid = $1;
8464     }
8465
8466     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8467
8468     my $res = [];
8469     foreach my $id (keys %$data) {
8470         foreach my $item (@{$data->{$id}}) {
8471             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8472             push @$res, $item->{volid} if defined($item->{volid});
8473         }
8474     }
8475
8476     return $res;
8477 }
8478
8479 my $complete_vmid_full = sub {
8480     my ($running) = @_;
8481
8482     my $idlist = vmstatus();
8483
8484     my $res = [];
8485
8486     foreach my $id (keys %$idlist) {
8487         my $d = $idlist->{$id};
8488         if (defined($running)) {
8489             next if $d->{template};
8490             next if $running && $d->{status} ne 'running';
8491             next if !$running && $d->{status} eq 'running';
8492         }
8493         push @$res, $id;
8494
8495     }
8496     return $res;
8497 };
8498
8499 sub complete_vmid {
8500     return &$complete_vmid_full();
8501 }
8502
8503 sub complete_vmid_stopped {
8504     return &$complete_vmid_full(0);
8505 }
8506
8507 sub complete_vmid_running {
8508     return &$complete_vmid_full(1);
8509 }
8510
8511 sub complete_storage {
8512
8513     my $cfg = PVE::Storage::config();
8514     my $ids = $cfg->{ids};
8515
8516     my $res = [];
8517     foreach my $sid (keys %$ids) {
8518         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8519         next if !$ids->{$sid}->{content}->{images};
8520         push @$res, $sid;
8521     }
8522
8523     return $res;
8524 }
8525
8526 sub complete_migration_storage {
8527     my ($cmd, $param, $current_value, $all_args) = @_;
8528
8529     my $targetnode = @$all_args[1];
8530
8531     my $cfg = PVE::Storage::config();
8532     my $ids = $cfg->{ids};
8533
8534     my $res = [];
8535     foreach my $sid (keys %$ids) {
8536         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8537         next if !$ids->{$sid}->{content}->{images};
8538         push @$res, $sid;
8539     }
8540
8541     return $res;
8542 }
8543
8544 sub vm_is_paused {
8545     my ($vmid) = @_;
8546     my $qmpstatus = eval {
8547         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8548         mon_cmd($vmid, "query-status");
8549     };
8550     warn "$@\n" if $@;
8551     return $qmpstatus && $qmpstatus->{status} eq "paused";
8552 }
8553
8554 sub check_volume_storage_type {
8555     my ($storecfg, $vol) = @_;
8556
8557     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8558     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8559     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8560
8561     die "storage '$storeid' does not support content-type '$vtype'\n"
8562         if !$scfg->{content}->{$vtype};
8563
8564     return 1;
8565 }
8566
8567 sub add_nets_bridge_fdb {
8568     my ($conf, $vmid) = @_;
8569
8570     for my $opt (keys %$conf) {
8571         next if $opt !~ m/^net(\d+)$/;
8572         my $iface = "tap${vmid}i$1";
8573         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8574         my $net = parse_net($conf->{$opt}, 1) or next;
8575
8576         my $mac = $net->{macaddr};
8577         if (!$mac) {
8578             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8579                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8580             next;
8581         }
8582
8583         my $bridge = $net->{bridge};
8584         if (!$bridge) {
8585             log_warn("Interface '$iface' not attached to any bridge.");
8586             next;
8587         }
8588         if ($have_sdn) {
8589             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8590         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8591             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8592         }
8593     }
8594 }
8595
8596 sub del_nets_bridge_fdb {
8597     my ($conf, $vmid) = @_;
8598
8599     for my $opt (keys %$conf) {
8600         next if $opt !~ m/^net(\d+)$/;
8601         my $iface = "tap${vmid}i$1";
8602
8603         my $net = parse_net($conf->{$opt}) or next;
8604         my $mac = $net->{macaddr} or next;
8605
8606         my $bridge = $net->{bridge};
8607         if ($have_sdn) {
8608             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8609         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8610             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8611         }
8612     }
8613 }
8614
8615 1;