PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use List::Util qw(first);
  23 use MIME::Base64;
  24 use POSIX;
  25 use Storable qw(dclone);
  26 use Time::HiRes qw(gettimeofday usleep);
  27 use URI::Escape;
  28 use UUID;
  29
  30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  31 use PVE::CGroup;
  32 use PVE::CpuSet;
  33 use PVE::DataCenterConfig;
  34 use PVE::Exception qw(raise raise_param_exc);
  35 use PVE::Format qw(render_duration render_bytes);
  36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  37 use PVE::INotify;
  38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  39 use PVE::ProcFSTools;
  40 use PVE::PBSClient;
  41 use PVE::RESTEnvironment qw(log_warn);
  42 use PVE::RPCEnvironment;
  43 use PVE::Storage;
  44 use PVE::SysFSTools;
  45 use PVE::Systemd;
  46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  47
  48 use PVE::QMPClient;
  49 use PVE::QemuConfig;
  50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  51 use PVE::QemuServer::Cloudinit;
  52 use PVE::QemuServer::CGroup;
  53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  55 use PVE::QemuServer::Machine;
  56 use PVE::QemuServer::Memory;
  57 use PVE::QemuServer::Monitor qw(mon_cmd);
  58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  59 use PVE::QemuServer::USB qw(parse_usb_device);
  60
  61 my $have_sdn;
  62 eval {
  63     require PVE::Network::SDN::Zones;
  64     $have_sdn = 1;
  65 };
  66
  67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  68 my $OVMF = {
  69     x86_64 => {
  70         '4m-no-smm' => [
  71             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  72             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  73         ],
  74         '4m-no-smm-ms' => [
  75             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  76             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  77         ],
  78         '4m' => [
  79             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  80             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  81         ],
  82         '4m-ms' => [
  83             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  84             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  85         ],
  86         default => [
  87             "$EDK2_FW_BASE/OVMF_CODE.fd",
  88             "$EDK2_FW_BASE/OVMF_VARS.fd",
  89         ],
  90     },
  91     aarch64 => {
  92         default => [
  93             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  94             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  95         ],
  96     },
  97 };
  98
  99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
 100
 101 # Note about locking: we use flock on the config file protect against concurent actions.
 102 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 104 # But you can ignore this kind of lock with the --skiplock flag.
 105
 106 cfs_register_file(
 107     '/qemu-server/',
 108     \&parse_vm_config,
 109     \&write_vm_config
 110 );
 111
 112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 113     description => "Some command save/restore state from this location.",
 114     type => 'string',
 115     maxLength => 128,
 116     optional => 1,
 117 });
 118
 119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 120         description => "Specifies the QEMU machine type.",
 121         type => 'string',
 122         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 123         maxLength => 40,
 124         optional => 1,
 125 });
 126
 127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 128 my $nodename_cache;
 129 sub nodename {
 130     $nodename_cache //= PVE::INotify::nodename();
 131     return $nodename_cache;
 132 }
 133
 134 my $watchdog_fmt = {
 135     model => {
 136         default_key => 1,
 137         type => 'string',
 138         enum => [qw(i6300esb ib700)],
 139         description => "Watchdog type to emulate.",
 140         default => 'i6300esb',
 141         optional => 1,
 142     },
 143     action => {
 144         type => 'string',
 145         enum => [qw(reset shutdown poweroff pause debug none)],
 146         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 147         optional => 1,
 148     },
 149 };
 150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 151
 152 my $agent_fmt = {
 153     enabled => {
 154         description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
 155         type => 'boolean',
 156         default => 0,
 157         default_key => 1,
 158     },
 159     fstrim_cloned_disks => {
 160         description => "Run fstrim after moving a disk or migrating the VM.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0,
 164     },
 165     'freeze-fs-on-backup' => {
 166         description => "Freeze/thaw guest filesystems on backup for consistency.",
 167         type => 'boolean',
 168         optional => 1,
 169         default => 1,
 170     },
 171     type => {
 172         description => "Select the agent type",
 173         type => 'string',
 174         default => 'virtio',
 175         optional => 1,
 176         enum => [qw(virtio isa)],
 177     },
 178 };
 179
 180 my $vga_fmt = {
 181     type => {
 182         description => "Select the VGA type.",
 183         type => 'string',
 184         default => 'std',
 185         optional => 1,
 186         default_key => 1,
 187         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 188     },
 189     memory => {
 190         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 191         type => 'integer',
 192         optional => 1,
 193         minimum => 4,
 194         maximum => 512,
 195     },
 196 };
 197
 198 my $ivshmem_fmt = {
 199     size => {
 200         type => 'integer',
 201         minimum => 1,
 202         description => "The size of the file in MB.",
 203     },
 204     name => {
 205         type => 'string',
 206         pattern => '[a-zA-Z0-9\-]+',
 207         optional => 1,
 208         format_description => 'string',
 209         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 210     },
 211 };
 212
 213 my $audio_fmt = {
 214     device => {
 215         type => 'string',
 216         enum => [qw(ich9-intel-hda intel-hda AC97)],
 217         description =>  "Configure an audio device."
 218     },
 219     driver =>  {
 220         type => 'string',
 221         enum => ['spice', 'none'],
 222         default => 'spice',
 223         optional => 1,
 224         description => "Driver backend for the audio device."
 225     },
 226 };
 227
 228 my $spice_enhancements_fmt = {
 229     foldersharing => {
 230         type => 'boolean',
 231         optional => 1,
 232         default => '0',
 233         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 234     },
 235     videostreaming =>  {
 236         type => 'string',
 237         enum => ['off', 'all', 'filter'],
 238         default => 'off',
 239         optional => 1,
 240         description => "Enable video streaming. Uses compression for detected video streams."
 241     },
 242 };
 243
 244 my $rng_fmt = {
 245     source => {
 246         type => 'string',
 247         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 248         default_key => 1,
 249         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 250             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 251             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 252             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 253             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 254             ." a hardware RNG from the host.",
 255     },
 256     max_bytes => {
 257         type => 'integer',
 258         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 259             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 260             ." `0` to disable limiting (potentially dangerous!).",
 261         optional => 1,
 262
 263         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 264         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 265         # reading from /dev/urandom
 266         default => 1024,
 267     },
 268     period => {
 269         type => 'integer',
 270         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 271             ." the guest to retrieve another 'max_bytes' of entropy.",
 272         optional => 1,
 273         default => 1000,
 274     },
 275 };
 276
 277 my $meta_info_fmt = {
 278     'ctime' => {
 279         type => 'integer',
 280         description => "The guest creation timestamp as UNIX epoch time",
 281         minimum => 0,
 282         optional => 1,
 283     },
 284     'creation-qemu' => {
 285         type => 'string',
 286         description => "The QEMU (machine) version from the time this VM was created.",
 287         pattern => '\d+(\.\d+)+',
 288         optional => 1,
 289     },
 290 };
 291
 292 my $confdesc = {
 293     onboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Specifies whether a VM will be started during system bootup.",
 297         default => 0,
 298     },
 299     autostart => {
 300         optional => 1,
 301         type => 'boolean',
 302         description => "Automatic restart after crash (currently ignored).",
 303         default => 0,
 304     },
 305     hotplug => {
 306         optional => 1,
 307         type => 'string', format => 'pve-hotplug-features',
 308         description => "Selectively enable hotplug features. This is a comma separated list of"
 309             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 310             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 311             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 312             ." windows > 7.",
 313         default => 'network,disk,usb',
 314     },
 315     reboot => {
 316         optional => 1,
 317         type => 'boolean',
 318         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 319         default => 1,
 320     },
 321     lock => {
 322         optional => 1,
 323         type => 'string',
 324         description => "Lock/unlock the VM.",
 325         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 326     },
 327     cpulimit => {
 328         optional => 1,
 329         type => 'number',
 330         description => "Limit of CPU usage.",
 331         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 332             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 333         minimum => 0,
 334         maximum => 128,
 335         default => 0,
 336     },
 337     cpuunits => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 341         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 342             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 343             ." weights of all the other running VMs.",
 344         minimum => 1,
 345         maximum => 262144,
 346         default => 'cgroup v1: 1024, cgroup v2: 100',
 347     },
 348     memory => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
 352             ." you use the balloon device.",
 353         minimum => 16,
 354         default => 512,
 355     },
 356     balloon => {
 357         optional => 1,
 358         type => 'integer',
 359         description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
 360         minimum => 0,
 361     },
 362     shares => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 366             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 367             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 368         minimum => 0,
 369         maximum => 50000,
 370         default => 1000,
 371     },
 372     keyboard => {
 373         optional => 1,
 374         type => 'string',
 375         description => "Keyboard layout for VNC server. This option is generally not required and"
 376         ." is often better handled from within the guest OS.",
 377         enum => PVE::Tools::kvmkeymaplist(),
 378         default => undef,
 379     },
 380     name => {
 381         optional => 1,
 382         type => 'string', format => 'dns-name',
 383         description => "Set a name for the VM. Only used on the configuration web interface.",
 384     },
 385     scsihw => {
 386         optional => 1,
 387         type => 'string',
 388         description => "SCSI controller model",
 389         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 390         default => 'lsi',
 391     },
 392     description => {
 393         optional => 1,
 394         type => 'string',
 395         description => "Description for the VM. Shown in the web-interface VM's summary."
 396             ." This is saved as comment inside the configuration file.",
 397         maxLength => 1024 * 8,
 398     },
 399     ostype => {
 400         optional => 1,
 401         type => 'string',
 402         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 403         description => "Specify guest operating system.",
 404         verbose_description => <<EODESC,
 405 Specify guest operating system. This is used to enable special
 406 optimization/features for specific operating systems:
 407
 408 [horizontal]
 409 other;; unspecified OS
 410 wxp;; Microsoft Windows XP
 411 w2k;; Microsoft Windows 2000
 412 w2k3;; Microsoft Windows 2003
 413 w2k8;; Microsoft Windows 2008
 414 wvista;; Microsoft Windows Vista
 415 win7;; Microsoft Windows 7
 416 win8;; Microsoft Windows 8/2012/2012r2
 417 win10;; Microsoft Windows 10/2016/2019
 418 win11;; Microsoft Windows 11/2022
 419 l24;; Linux 2.4 Kernel
 420 l26;; Linux 2.6 - 6.X Kernel
 421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 422 EODESC
 423     },
 424     boot => {
 425         optional => 1,
 426         type => 'string', format => 'pve-qm-boot',
 427         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 428             ." key or 'legacy=' is deprecated.",
 429     },
 430     bootdisk => {
 431         optional => 1,
 432         type => 'string', format => 'pve-qm-bootdisk',
 433         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 434         pattern => '(ide|sata|scsi|virtio)\d+',
 435     },
 436     smp => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPUs. Please use option -sockets instead.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     sockets => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of CPU sockets.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     cores => {
 451         optional => 1,
 452         type => 'integer',
 453         description => "The number of cores per socket.",
 454         minimum => 1,
 455         default => 1,
 456     },
 457     numa => {
 458         optional => 1,
 459         type => 'boolean',
 460         description => "Enable/disable NUMA.",
 461         default => 0,
 462     },
 463     hugepages => {
 464         optional => 1,
 465         type => 'string',
 466         description => "Enable/disable hugepages memory.",
 467         enum => [qw(any 2 1024)],
 468     },
 469     keephugepages => {
 470         optional => 1,
 471         type => 'boolean',
 472         default => 0,
 473         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 474             ." after VM shutdown and can be used for subsequent starts.",
 475     },
 476     vcpus => {
 477         optional => 1,
 478         type => 'integer',
 479         description => "Number of hotplugged vcpus.",
 480         minimum => 1,
 481         default => 0,
 482     },
 483     acpi => {
 484         optional => 1,
 485         type => 'boolean',
 486         description => "Enable/disable ACPI.",
 487         default => 1,
 488     },
 489     agent => {
 490         optional => 1,
 491         description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
 492         type => 'string',
 493         format => $agent_fmt,
 494     },
 495     kvm => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Enable/disable KVM hardware virtualization.",
 499         default => 1,
 500     },
 501     tdf => {
 502         optional => 1,
 503         type => 'boolean',
 504         description => "Enable/disable time drift fix.",
 505         default => 0,
 506     },
 507     localtime => {
 508         optional => 1,
 509         type => 'boolean',
 510         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 511             ." the `ostype` indicates a Microsoft Windows OS.",
 512     },
 513     freeze => {
 514         optional => 1,
 515         type => 'boolean',
 516         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 517     },
 518     vga => {
 519         optional => 1,
 520         type => 'string', format => $vga_fmt,
 521         description => "Configure the VGA hardware.",
 522         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 523             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 524             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 525             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 526             ." display server. For win* OS you can select how many independent displays you want,"
 527             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 528             ." using a serial device as terminal.",
 529     },
 530     watchdog => {
 531         optional => 1,
 532         type => 'string', format => 'pve-qm-watchdog',
 533         description => "Create a virtual hardware watchdog device.",
 534         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 535             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 536             ." else the watchdog will reset the guest (or execute the respective action specified)",
 537     },
 538     startdate => {
 539         optional => 1,
 540         type => 'string',
 541         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 542         description => "Set the initial date of the real time clock. Valid format for date are:"
 543             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 544         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 545         default => 'now',
 546     },
 547     startup =>  get_standard_option('pve-startup-order'),
 548     template => {
 549         optional => 1,
 550         type => 'boolean',
 551         description => "Enable/disable Template.",
 552         default => 0,
 553     },
 554     args => {
 555         optional => 1,
 556         type => 'string',
 557         description => "Arbitrary arguments passed to kvm.",
 558         verbose_description => <<EODESCR,
 559 Arbitrary arguments passed to kvm, for example:
 560
 561 args: -no-reboot -no-hpet
 562
 563 NOTE: this option is for experts only.
 564 EODESCR
 565     },
 566     tablet => {
 567         optional => 1,
 568         type => 'boolean',
 569         default => 1,
 570         description => "Enable/disable the USB tablet device.",
 571         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 572             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 573             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 574             ." may consider disabling this to save some context switches. This is turned off by"
 575             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 576     },
 577     migrate_speed => {
 578         optional => 1,
 579         type => 'integer',
 580         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 581         minimum => 0,
 582         default => 0,
 583     },
 584     migrate_downtime => {
 585         optional => 1,
 586         type => 'number',
 587         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 588         minimum => 0,
 589         default => 0.1,
 590     },
 591     cdrom => {
 592         optional => 1,
 593         type => 'string', format => 'pve-qm-ide',
 594         typetext => '<volume>',
 595         description => "This is an alias for option -ide2",
 596     },
 597     cpu => {
 598         optional => 1,
 599         description => "Emulated CPU type.",
 600         type => 'string',
 601         format => 'pve-vm-cpu-conf',
 602     },
 603     parent => get_standard_option('pve-snapshot-name', {
 604         optional => 1,
 605         description => "Parent snapshot name. This is used internally, and should not be modified.",
 606     }),
 607     snaptime => {
 608         optional => 1,
 609         description => "Timestamp for snapshots.",
 610         type => 'integer',
 611         minimum => 0,
 612     },
 613     vmstate => {
 614         optional => 1,
 615         type => 'string', format => 'pve-volume-id',
 616         description => "Reference to a volume which stores the VM state. This is used internally"
 617             ." for snapshots.",
 618     },
 619     vmstatestorage => get_standard_option('pve-storage-id', {
 620         description => "Default storage for VM state volumes/files.",
 621         optional => 1,
 622     }),
 623     runningmachine => get_standard_option('pve-qemu-machine', {
 624         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 625             ." for snapshots.",
 626     }),
 627     runningcpu => {
 628         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 629             ." internally for snapshots.",
 630         optional => 1,
 631         type => 'string',
 632         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 633         format_description => 'QEMU -cpu parameter'
 634     },
 635     machine => get_standard_option('pve-qemu-machine'),
 636     arch => {
 637         description => "Virtual processor architecture. Defaults to the host.",
 638         optional => 1,
 639         type => 'string',
 640         enum => [qw(x86_64 aarch64)],
 641     },
 642     smbios1 => {
 643         description => "Specify SMBIOS type 1 fields.",
 644         type => 'string', format => 'pve-qm-smbios1',
 645         maxLength => 512,
 646         optional => 1,
 647     },
 648     protection => {
 649         optional => 1,
 650         type => 'boolean',
 651         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 652             ." remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 667             ." to disable explicitly.",
 668         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 669             ." value identifier to the guest OS. This allows to notify the guest operating system"
 670             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 671             ." execution or creation from a template). The guest operating system notices the"
 672             ." change, and is then able to react as appropriate by marking its copies of"
 673             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 674             ."Note that auto-creation only works when done through API/CLI create or update methods"
 675             .", but not when manually editing the config file.",
 676         default => "1 (autogenerated)",
 677         optional => 1,
 678     },
 679     hookscript => {
 680         type => 'string',
 681         format => 'pve-volume-id',
 682         optional => 1,
 683         description => "Script that will be executed during various steps in the vms lifetime.",
 684     },
 685     ivshmem => {
 686         type => 'string',
 687         format => $ivshmem_fmt,
 688         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 689             ." the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704     tags => {
 705         type => 'string', format => 'pve-tag-list',
 706         description => 'Tags of the VM. This is only meta information.',
 707         optional => 1,
 708     },
 709     rng0 => {
 710         type => 'string',
 711         format => $rng_fmt,
 712         description => "Configure a VirtIO-based Random Number Generator.",
 713         optional => 1,
 714     },
 715     meta => {
 716         type => 'string',
 717         format => $meta_info_fmt,
 718         description => "Some (read-only) meta-information about this guest.",
 719         optional => 1,
 720     },
 721     affinity => {
 722         type => 'string', format => 'pve-cpuset',
 723         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 724         optional => 1,
 725     },
 726 };
 727
 728 my $cicustom_fmt = {
 729     meta => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all meta data passed to the VM via"
 733             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     network => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'To pass a custom file containing all network data to the VM via cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     user => {
 745         type => 'string',
 746         optional => 1,
 747         description => 'To pass a custom file containing all user data to the VM via cloud-init.',
 748         format => 'pve-volume-id',
 749         format_description => 'volume',
 750     },
 751     vendor => {
 752         type => 'string',
 753         optional => 1,
 754         description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758 };
 759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 760
 761 my $confdesc_cloudinit = {
 762     citype => {
 763         optional => 1,
 764         type => 'string',
 765         description => 'Specifies the cloud-init configuration format. The default depends on the'
 766             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 767             .' and `configdrive2` for windows.',
 768         enum => ['configdrive2', 'nocloud', 'opennebula'],
 769     },
 770     ciuser => {
 771         optional => 1,
 772         type => 'string',
 773         description => "cloud-init: User name to change ssh keys and password for instead of the"
 774             ." image's configured default user.",
 775     },
 776     cipassword => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Password to assign the user. Using this is generally not'
 780             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 781             .' support hashed passwords.',
 782     },
 783     cicustom => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Specify custom files to replace the automatically generated'
 787             .' ones at start.',
 788         format => 'pve-qm-cicustom',
 789     },
 790     searchdomain => {
 791         optional => 1,
 792         type => 'string',
 793         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     nameserver => {
 798         optional => 1,
 799         type => 'string', format => 'address-list',
 800         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 801             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 802             .' are set.',
 803     },
 804     sshkeys => {
 805         optional => 1,
 806         type => 'string',
 807         format => 'urlencoded',
 808         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 809     },
 810 };
 811
 812 # what about other qemu settings ?
 813 #cpu => 'string',
 814 #machine => 'string',
 815 #fda => 'file',
 816 #fdb => 'file',
 817 #mtdblock => 'file',
 818 #sd => 'file',
 819 #pflash => 'file',
 820 #snapshot => 'bool',
 821 #bootp => 'file',
 822 ##tftp => 'dir',
 823 ##smb => 'dir',
 824 #kernel => 'file',
 825 #append => 'string',
 826 #initrd => 'file',
 827 ##soundhw => 'string',
 828
 829 while (my ($k, $v) = each %$confdesc) {
 830     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 831 }
 832
 833 my $MAX_USB_DEVICES = 14;
 834 my $MAX_NETS = 32;
 835 my $MAX_SERIAL_PORTS = 4;
 836 my $MAX_PARALLEL_PORTS = 3;
 837 my $MAX_NUMA = 8;
 838
 839 my $numa_fmt = {
 840     cpus => {
 841         type => "string",
 842         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 843         description => "CPUs accessing this NUMA node.",
 844         format_description => "id[-id];...",
 845     },
 846     memory => {
 847         type => "number",
 848         description => "Amount of memory this NUMA node provides.",
 849         optional => 1,
 850     },
 851     hostnodes => {
 852         type => "string",
 853         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 854         description => "Host NUMA nodes to use.",
 855         format_description => "id[-id];...",
 856         optional => 1,
 857     },
 858     policy => {
 859         type => 'string',
 860         enum => [qw(preferred bind interleave)],
 861         description => "NUMA allocation policy.",
 862         optional => 1,
 863     },
 864 };
 865 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 866 my $numadesc = {
 867     optional => 1,
 868     type => 'string', format => $numa_fmt,
 869     description => "NUMA topology.",
 870 };
 871 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 872
 873 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 874     $confdesc->{"numa$i"} = $numadesc;
 875 }
 876
 877 my $nic_model_list = [
 878     'e1000',
 879     'e1000-82540em',
 880     'e1000-82544gc',
 881     'e1000-82545em',
 882     'e1000e',
 883     'i82551',
 884     'i82557b',
 885     'i82559er',
 886     'ne2k_isa',
 887     'ne2k_pci',
 888     'pcnet',
 889     'rtl8139',
 890     'virtio',
 891     'vmxnet3',
 892 ];
 893 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 894
 895 my $net_fmt_bridge_descr = <<__EOD__;
 896 Bridge to attach the network device to. The Proxmox VE standard bridge
 897 is called 'vmbr0'.
 898
 899 If you do not specify a bridge, we create a kvm user (NATed) network
 900 device, which provides DHCP and DNS services. The following addresses
 901 are used:
 902
 903  10.0.2.2   Gateway
 904  10.0.2.3   DNS Server
 905  10.0.2.4   SMB Server
 906
 907 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 908 __EOD__
 909
 910 my $net_fmt = {
 911     macaddr  => get_standard_option('mac-addr', {
 912         description => "MAC address. That address must be unique withing your network. This is"
 913             ." automatically generated if not specified.",
 914     }),
 915     model => {
 916         type => 'string',
 917         description => "Network Card Model. The 'virtio' model provides the best performance with"
 918             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 919             ." best to use 'e1000'.",
 920         enum => $nic_model_list,
 921         default_key => 1,
 922     },
 923     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 924     bridge => get_standard_option('pve-bridge-id', {
 925         description => $net_fmt_bridge_descr,
 926         optional => 1,
 927     }),
 928     queues => {
 929         type => 'integer',
 930         minimum => 0, maximum => 64,
 931         description => 'Number of packet queues to be used on the device.',
 932         optional => 1,
 933     },
 934     rate => {
 935         type => 'number',
 936         minimum => 0,
 937         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 938         optional => 1,
 939     },
 940     tag => {
 941         type => 'integer',
 942         minimum => 1, maximum => 4094,
 943         description => 'VLAN tag to apply to packets on this interface.',
 944         optional => 1,
 945     },
 946     trunks => {
 947         type => 'string',
 948         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 949         description => 'VLAN trunks to pass through this interface.',
 950         format_description => 'vlanid[;vlanid...]',
 951         optional => 1,
 952     },
 953     firewall => {
 954         type => 'boolean',
 955         description => 'Whether this interface should be protected by the firewall.',
 956         optional => 1,
 957     },
 958     link_down => {
 959         type => 'boolean',
 960         description => 'Whether this interface should be disconnected (like pulling the plug).',
 961         optional => 1,
 962     },
 963     mtu => {
 964         type => 'integer',
 965         minimum => 1, maximum => 65520,
 966         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 967         optional => 1,
 968     },
 969 };
 970
 971 my $netdesc = {
 972     optional => 1,
 973     type => 'string', format => $net_fmt,
 974     description => "Specify network devices.",
 975 };
 976
 977 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 978
 979 my $ipconfig_fmt = {
 980     ip => {
 981         type => 'string',
 982         format => 'pve-ipv4-config',
 983         format_description => 'IPv4Format/CIDR',
 984         description => 'IPv4 address in CIDR format.',
 985         optional => 1,
 986         default => 'dhcp',
 987     },
 988     gw => {
 989         type => 'string',
 990         format => 'ipv4',
 991         format_description => 'GatewayIPv4',
 992         description => 'Default gateway for IPv4 traffic.',
 993         optional => 1,
 994         requires => 'ip',
 995     },
 996     ip6 => {
 997         type => 'string',
 998         format => 'pve-ipv6-config',
 999         format_description => 'IPv6Format/CIDR',
1000         description => 'IPv6 address in CIDR format.',
1001         optional => 1,
1002         default => 'dhcp',
1003     },
1004     gw6 => {
1005         type => 'string',
1006         format => 'ipv6',
1007         format_description => 'GatewayIPv6',
1008         description => 'Default gateway for IPv6 traffic.',
1009         optional => 1,
1010         requires => 'ip6',
1011     },
1012 };
1013 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1014 my $ipconfigdesc = {
1015     optional => 1,
1016     type => 'string', format => 'pve-qm-ipconfig',
1017     description => <<'EODESCR',
1018 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1019
1020 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1021
1022 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1023 gateway should be provided.
1024 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1025 cloud-init 19.4 or newer.
1026
1027 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1028 dhcp on IPv4.
1029 EODESCR
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1032
1033 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1034     $confdesc->{"net$i"} = $netdesc;
1035     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1036 }
1037
1038 foreach my $key (keys %$confdesc_cloudinit) {
1039     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1040 }
1041
1042 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1043 sub pve_verify_cpuset {
1044     my ($set_text, $noerr) = @_;
1045
1046     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1047
1048     if ($@) {
1049         return if $noerr;
1050         die "unable to parse cpuset option\n";
1051     }
1052
1053     return PVE::CpuSet->new($members)->short_string();
1054 }
1055
1056 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1057 sub verify_volume_id_or_qm_path {
1058     my ($volid, $noerr) = @_;
1059
1060     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1061
1062     return verify_volume_id_or_absolute_path($volid, $noerr);
1063 }
1064
1065 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1066 sub verify_volume_id_or_absolute_path {
1067     my ($volid, $noerr) = @_;
1068
1069     return $volid if $volid =~ m|^/|;
1070
1071     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1072     if ($@) {
1073         return if $noerr;
1074         die $@;
1075     }
1076     return $volid;
1077 }
1078
1079 my $usb_fmt = {
1080     host => {
1081         default_key => 1,
1082         type => 'string', format => 'pve-qm-usb-device',
1083         format_description => 'HOSTUSBDEVICE|spice',
1084         description => <<EODESCR,
1085 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1086
1087  'bus-port(.port)*' (decimal numbers) or
1088  'vendor_id:product_id' (hexadeciaml numbers) or
1089  'spice'
1090
1091 You can use the 'lsusb -t' command to list existing usb devices.
1092
1093 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1094 machines - use with special care.
1095
1096 The value 'spice' can be used to add a usb redirection devices for spice.
1097 EODESCR
1098     },
1099     usb3 => {
1100         optional => 1,
1101         type => 'boolean',
1102         description => "Specifies whether if given host option is a USB3 device or port."
1103             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1104             ." is irrelevant (all devices are plugged into a xhci controller).",
1105         default => 0,
1106     },
1107 };
1108
1109 my $usbdesc = {
1110     optional => 1,
1111     type => 'string', format => $usb_fmt,
1112     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1113         ." l26 or windows > 7, n can be up to 14).",
1114 };
1115 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1116
1117 my $serialdesc = {
1118         optional => 1,
1119         type => 'string',
1120         pattern => '(/dev/.+|socket)',
1121         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1122         verbose_description =>  <<EODESCR,
1123 Create a serial device inside the VM (n is 0 to 3), and pass through a
1124 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1125 host side (use 'qm terminal' to open a terminal connection).
1126
1127 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1128 use with special care.
1129
1130 CAUTION: Experimental! User reported problems with this option.
1131 EODESCR
1132 };
1133
1134 my $paralleldesc= {
1135         optional => 1,
1136         type => 'string',
1137         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1138         description =>  "Map host parallel devices (n is 0 to 2).",
1139         verbose_description =>  <<EODESCR,
1140 Map host parallel devices (n is 0 to 2).
1141
1142 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1143 machines - use with special care.
1144
1145 CAUTION: Experimental! User reported problems with this option.
1146 EODESCR
1147 };
1148
1149 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1150     $confdesc->{"parallel$i"} = $paralleldesc;
1151 }
1152
1153 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1154     $confdesc->{"serial$i"} = $serialdesc;
1155 }
1156
1157 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1158     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1159 }
1160
1161 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1162     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1163 }
1164
1165 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1166     $confdesc->{"usb$i"} = $usbdesc;
1167 }
1168
1169 my $boot_fmt = {
1170     legacy => {
1171         optional => 1,
1172         default_key => 1,
1173         type => 'string',
1174         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1175                      . " Deprecated, use 'order=' instead.",
1176         pattern => '[acdn]{1,4}',
1177         format_description => "[acdn]{1,4}",
1178
1179         # note: this is also the fallback if boot: is not given at all
1180         default => 'cdn',
1181     },
1182     order => {
1183         optional => 1,
1184         type => 'string',
1185         format => 'pve-qm-bootdev-list',
1186         format_description => "device[;device...]",
1187         description => <<EODESC,
1188 The guest will attempt to boot from devices in the order they appear here.
1189
1190 Disks, optical drives and passed-through storage USB devices will be directly
1191 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1192 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1193
1194 Note that only devices in this list will be marked as bootable and thus loaded
1195 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1196 (e.g. software-raid), you need to specify all of them here.
1197
1198 Overrides the deprecated 'legacy=[acdn]*' value when given.
1199 EODESC
1200     },
1201 };
1202 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1203
1204 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1205 sub verify_bootdev {
1206     my ($dev, $noerr) = @_;
1207
1208     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1209     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1210
1211     my $check = sub {
1212         my ($base) = @_;
1213         return 0 if $dev !~ m/^$base\d+$/;
1214         return 0 if !$confdesc->{$dev};
1215         return 1;
1216     };
1217
1218     return $dev if $check->("net");
1219     return $dev if $check->("usb");
1220     return $dev if $check->("hostpci");
1221
1222     return if $noerr;
1223     die "invalid boot device '$dev'\n";
1224 }
1225
1226 sub print_bootorder {
1227     my ($devs) = @_;
1228     return "" if !@$devs;
1229     my $data = { order => join(';', @$devs) };
1230     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1231 }
1232
1233 my $kvm_api_version = 0;
1234
1235 sub kvm_version {
1236     return $kvm_api_version if $kvm_api_version;
1237
1238     open my $fh, '<', '/dev/kvm' or return;
1239
1240     # 0xae00 => KVM_GET_API_VERSION
1241     $kvm_api_version = ioctl($fh, 0xae00, 0);
1242     close($fh);
1243
1244     return $kvm_api_version;
1245 }
1246
1247 my $kvm_user_version = {};
1248 my $kvm_mtime = {};
1249
1250 sub kvm_user_version {
1251     my ($binary) = @_;
1252
1253     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1254     my $st = stat($binary);
1255
1256     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1257     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1258         $cachedmtime == $st->mtime;
1259
1260     $kvm_user_version->{$binary} = 'unknown';
1261     $kvm_mtime->{$binary} = $st->mtime;
1262
1263     my $code = sub {
1264         my $line = shift;
1265         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1266             $kvm_user_version->{$binary} = $2;
1267         }
1268     };
1269
1270     eval { run_command([$binary, '--version'], outfunc => $code); };
1271     warn $@ if $@;
1272
1273     return $kvm_user_version->{$binary};
1274
1275 }
1276 my sub extract_version {
1277     my ($machine_type, $version) = @_;
1278     $version = kvm_user_version() if !defined($version);
1279     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1280 }
1281
1282 sub kernel_has_vhost_net {
1283     return -c '/dev/vhost-net';
1284 }
1285
1286 sub option_exists {
1287     my $key = shift;
1288     return defined($confdesc->{$key});
1289 }
1290
1291 my $cdrom_path;
1292 sub get_cdrom_path {
1293
1294     return $cdrom_path if defined($cdrom_path);
1295
1296     $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1297
1298     if (!defined($cdrom_path)) {
1299         log_warn("no physical CD-ROM available, ignoring");
1300         $cdrom_path = '';
1301     }
1302
1303     return $cdrom_path;
1304 }
1305
1306 sub get_iso_path {
1307     my ($storecfg, $vmid, $cdrom) = @_;
1308
1309     if ($cdrom eq 'cdrom') {
1310         return get_cdrom_path();
1311     } elsif ($cdrom eq 'none') {
1312         return '';
1313     } elsif ($cdrom =~ m|^/|) {
1314         return $cdrom;
1315     } else {
1316         return PVE::Storage::path($storecfg, $cdrom);
1317     }
1318 }
1319
1320 # try to convert old style file names to volume IDs
1321 sub filename_to_volume_id {
1322     my ($vmid, $file, $media) = @_;
1323
1324      if (!($file eq 'none' || $file eq 'cdrom' ||
1325           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1326
1327         return if $file =~ m|/|;
1328
1329         if ($media && $media eq 'cdrom') {
1330             $file = "local:iso/$file";
1331         } else {
1332             $file = "local:$vmid/$file";
1333         }
1334     }
1335
1336     return $file;
1337 }
1338
1339 sub verify_media_type {
1340     my ($opt, $vtype, $media) = @_;
1341
1342     return if !$media;
1343
1344     my $etype;
1345     if ($media eq 'disk') {
1346         $etype = 'images';
1347     } elsif ($media eq 'cdrom') {
1348         $etype = 'iso';
1349     } else {
1350         die "internal error";
1351     }
1352
1353     return if ($vtype eq $etype);
1354
1355     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1356 }
1357
1358 sub cleanup_drive_path {
1359     my ($opt, $storecfg, $drive) = @_;
1360
1361     # try to convert filesystem paths to volume IDs
1362
1363     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1364         ($drive->{file} !~ m|^/dev/.+|) &&
1365         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1366         ($drive->{file} !~ m/^\d+$/)) {
1367         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1368         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1369             if !$vtype;
1370         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1371         verify_media_type($opt, $vtype, $drive->{media});
1372         $drive->{file} = $volid;
1373     }
1374
1375     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1376 }
1377
1378 sub parse_hotplug_features {
1379     my ($data) = @_;
1380
1381     my $res = {};
1382
1383     return $res if $data eq '0';
1384
1385     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1386
1387     foreach my $feature (PVE::Tools::split_list($data)) {
1388         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1389             $res->{$1} = 1;
1390         } else {
1391             die "invalid hotplug feature '$feature'\n";
1392         }
1393     }
1394     return $res;
1395 }
1396
1397 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1398 sub pve_verify_hotplug_features {
1399     my ($value, $noerr) = @_;
1400
1401     return $value if parse_hotplug_features($value);
1402
1403     return if $noerr;
1404
1405     die "unable to parse hotplug option\n";
1406 }
1407
1408 sub scsi_inquiry {
1409     my($fh, $noerr) = @_;
1410
1411     my $SG_IO = 0x2285;
1412     my $SG_GET_VERSION_NUM = 0x2282;
1413
1414     my $versionbuf = "\x00" x 8;
1415     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1416     if (!$ret) {
1417         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1418         return;
1419     }
1420     my $version = unpack("I", $versionbuf);
1421     if ($version < 30000) {
1422         die "scsi generic interface too old\n"  if !$noerr;
1423         return;
1424     }
1425
1426     my $buf = "\x00" x 36;
1427     my $sensebuf = "\x00" x 8;
1428     my $cmd = pack("C x3 C x1", 0x12, 36);
1429
1430     # see /usr/include/scsi/sg.h
1431     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1432
1433     my $packet = pack(
1434         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1435     );
1436
1437     $ret = ioctl($fh, $SG_IO, $packet);
1438     if (!$ret) {
1439         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1440         return;
1441     }
1442
1443     my @res = unpack($sg_io_hdr_t, $packet);
1444     if ($res[17] || $res[18]) {
1445         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1446         return;
1447     }
1448
1449     my $res = {};
1450     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1451
1452     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1453     $res->{type} &= 0x1F;
1454
1455     return $res;
1456 }
1457
1458 sub path_is_scsi {
1459     my ($path) = @_;
1460
1461     my $fh = IO::File->new("+<$path") || return;
1462     my $res = scsi_inquiry($fh, 1);
1463     close($fh);
1464
1465     return $res;
1466 }
1467
1468 sub print_tabletdevice_full {
1469     my ($conf, $arch) = @_;
1470
1471     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1472
1473     # we use uhci for old VMs because tablet driver was buggy in older qemu
1474     my $usbbus;
1475     if ($q35 || $arch eq 'aarch64') {
1476         $usbbus = 'ehci';
1477     } else {
1478         $usbbus = 'uhci';
1479     }
1480
1481     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1482 }
1483
1484 sub print_keyboarddevice_full {
1485     my ($conf, $arch) = @_;
1486
1487     return if $arch ne 'aarch64';
1488
1489     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1490 }
1491
1492 my sub get_drive_id {
1493     my ($drive) = @_;
1494     return "$drive->{interface}$drive->{index}";
1495 }
1496
1497 sub print_drivedevice_full {
1498     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1499
1500     my $device = '';
1501     my $maxdev = 0;
1502
1503     my $drive_id = get_drive_id($drive);
1504     if ($drive->{interface} eq 'virtio') {
1505         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1506         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1507         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1508     } elsif ($drive->{interface} eq 'scsi') {
1509
1510         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1511         my $unit = $drive->{index} % $maxdev;
1512         my $devicetype = 'hd';
1513         my $path = '';
1514         if (drive_is_cdrom($drive)) {
1515             $devicetype = 'cd';
1516         } else {
1517             if ($drive->{file} =~ m|^/|) {
1518                 $path = $drive->{file};
1519                 if (my $info = path_is_scsi($path)) {
1520                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1521                         $devicetype = 'block';
1522                     } elsif ($info->{type} == 1) { # tape
1523                         $devicetype = 'generic';
1524                     }
1525                 }
1526             } else {
1527                  $path = PVE::Storage::path($storecfg, $drive->{file});
1528             }
1529
1530             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1531             my $version = extract_version($machine_type, kvm_user_version());
1532             if ($path =~ m/^iscsi\:\/\// &&
1533                !min_version($version, 4, 1)) {
1534                 $devicetype = 'generic';
1535             }
1536         }
1537
1538         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1539             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1540         } else {
1541             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1542                 .",lun=$drive->{index}";
1543         }
1544         $device .= ",drive=drive-$drive_id,id=$drive_id";
1545
1546         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1547             $device .= ",rotation_rate=1";
1548         }
1549         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1550
1551     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1552         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1553         my $controller = int($drive->{index} / $maxdev);
1554         my $unit = $drive->{index} % $maxdev;
1555         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1556
1557         $device = "ide-$devicetype";
1558         if ($drive->{interface} eq 'ide') {
1559             $device .= ",bus=ide.$controller,unit=$unit";
1560         } else {
1561             $device .= ",bus=ahci$controller.$unit";
1562         }
1563         $device .= ",drive=drive-$drive_id,id=$drive_id";
1564
1565         if ($devicetype eq 'hd') {
1566             if (my $model = $drive->{model}) {
1567                 $model = URI::Escape::uri_unescape($model);
1568                 $device .= ",model=$model";
1569             }
1570             if ($drive->{ssd}) {
1571                 $device .= ",rotation_rate=1";
1572             }
1573         }
1574         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1575     } elsif ($drive->{interface} eq 'usb') {
1576         die "implement me";
1577         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1578     } else {
1579         die "unsupported interface type";
1580     }
1581
1582     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1583
1584     if (my $serial = $drive->{serial}) {
1585         $serial = URI::Escape::uri_unescape($serial);
1586         $device .= ",serial=$serial";
1587     }
1588
1589
1590     return $device;
1591 }
1592
1593 sub get_initiator_name {
1594     my $initiator;
1595
1596     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1597     while (defined(my $line = <$fh>)) {
1598         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1599         $initiator = $1;
1600         last;
1601     }
1602     $fh->close();
1603
1604     return $initiator;
1605 }
1606
1607 my sub storage_allows_io_uring_default {
1608     my ($scfg, $cache_direct) = @_;
1609
1610     # io_uring with cache mode writeback or writethrough on krbd will hang...
1611     return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1612
1613     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1614     # sometimes, just plain disable...
1615     return if $scfg && $scfg->{type} eq 'lvm';
1616
1617     # io_uring causes problems when used with CIFS since kernel 5.15
1618     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1619     return if $scfg && $scfg->{type} eq 'cifs';
1620
1621     return 1;
1622 }
1623
1624 my sub drive_uses_cache_direct {
1625     my ($drive, $scfg) = @_;
1626
1627     my $cache_direct = 0;
1628
1629     if (my $cache = $drive->{cache}) {
1630         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1631     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1632         $cache_direct = 1;
1633     }
1634
1635     return $cache_direct;
1636 }
1637
1638 sub print_drive_commandline_full {
1639     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1640
1641     my $path;
1642     my $volid = $drive->{file};
1643     my $format = $drive->{format};
1644     my $drive_id = get_drive_id($drive);
1645
1646     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1647     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1648
1649     if (drive_is_cdrom($drive)) {
1650         $path = get_iso_path($storecfg, $vmid, $volid);
1651         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1652     } else {
1653         if ($storeid) {
1654             $path = PVE::Storage::path($storecfg, $volid);
1655             $format //= qemu_img_format($scfg, $volname);
1656         } else {
1657             $path = $volid;
1658             $format //= "raw";
1659         }
1660    }
1661
1662    my $is_rbd = $path =~ m/^rbd:/;
1663
1664     my $opts = '';
1665     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1666     foreach my $o (@qemu_drive_options) {
1667         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1668     }
1669
1670     # snapshot only accepts on|off
1671     if (defined($drive->{snapshot})) {
1672         my $v = $drive->{snapshot} ? 'on' : 'off';
1673         $opts .= ",snapshot=$v";
1674     }
1675
1676     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1677         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1678     }
1679
1680     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1681         my ($dir, $qmpname) = @$type;
1682         if (my $v = $drive->{"mbps$dir"}) {
1683             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1684         }
1685         if (my $v = $drive->{"mbps${dir}_max"}) {
1686             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1687         }
1688         if (my $v = $drive->{"bps${dir}_max_length"}) {
1689             $opts .= ",throttling.bps$qmpname-max-length=$v";
1690         }
1691         if (my $v = $drive->{"iops${dir}"}) {
1692             $opts .= ",throttling.iops$qmpname=$v";
1693         }
1694         if (my $v = $drive->{"iops${dir}_max"}) {
1695             $opts .= ",throttling.iops$qmpname-max=$v";
1696         }
1697         if (my $v = $drive->{"iops${dir}_max_length"}) {
1698             $opts .= ",throttling.iops$qmpname-max-length=$v";
1699         }
1700     }
1701
1702     if ($pbs_name) {
1703         $format = "rbd" if $is_rbd;
1704         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1705             if !$format;
1706         $opts .= ",format=alloc-track,file.driver=$format";
1707     } elsif ($format) {
1708         $opts .= ",format=$format";
1709     }
1710
1711     my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1712
1713     $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1714
1715     if (!$drive->{aio}) {
1716         if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1717             # io_uring supports all cache modes
1718             $opts .= ",aio=io_uring";
1719         } else {
1720             # aio native works only with O_DIRECT
1721             if($cache_direct) {
1722                 $opts .= ",aio=native";
1723             } else {
1724                 $opts .= ",aio=threads";
1725             }
1726         }
1727     }
1728
1729     if (!drive_is_cdrom($drive)) {
1730         my $detectzeroes;
1731         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1732             $detectzeroes = 'off';
1733         } elsif ($drive->{discard}) {
1734             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1735         } else {
1736             # This used to be our default with discard not being specified:
1737             $detectzeroes = 'on';
1738         }
1739
1740         # note: 'detect-zeroes' works per blockdev and we want it to persist
1741         # after the alloc-track is removed, so put it on 'file' directly
1742         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1743         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1744     }
1745
1746     if ($pbs_name) {
1747         $opts .= ",backing=$pbs_name";
1748         $opts .= ",auto-remove=on";
1749     }
1750
1751     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1752     my $file_param = "file";
1753     if ($pbs_name) {
1754         # non-rbd drivers require the underlying file to be a seperate block
1755         # node, so add a second .file indirection
1756         $file_param .= ".file" if !$is_rbd;
1757         $file_param .= ".filename";
1758     }
1759     my $pathinfo = $path ? "$file_param=$path," : '';
1760
1761     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1762 }
1763
1764 sub print_pbs_blockdev {
1765     my ($pbs_conf, $pbs_name) = @_;
1766     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1767     $blockdev .= ",repository=$pbs_conf->{repository}";
1768     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1769     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1770     $blockdev .= ",archive=$pbs_conf->{archive}";
1771     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1772     return $blockdev;
1773 }
1774
1775 sub print_netdevice_full {
1776     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1777
1778     my $device = $net->{model};
1779     if ($net->{model} eq 'virtio') {
1780          $device = 'virtio-net-pci';
1781      };
1782
1783     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1784     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1785     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1786         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1787         # and out of each queue plus one config interrupt and control vector queue
1788         my $vectors = $net->{queues} * 2 + 2;
1789         $tmpstr .= ",vectors=$vectors,mq=on";
1790         if (min_version($machine_version, 7, 1)) {
1791             $tmpstr .= ",packed=on";
1792         }
1793     }
1794
1795     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1796         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1797     }
1798
1799     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1800
1801     if (my $mtu = $net->{mtu}) {
1802         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1803             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1804             if ($mtu == 1) {
1805                  $mtu = $bridge_mtu;
1806             } elsif ($mtu < 576) {
1807                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1808             } elsif ($mtu > $bridge_mtu) {
1809                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1810             }
1811             $tmpstr .= ",host_mtu=$mtu";
1812         } else {
1813             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1814         }
1815     }
1816
1817     if ($use_old_bios_files) {
1818         my $romfile;
1819         if ($device eq 'virtio-net-pci') {
1820             $romfile = 'pxe-virtio.rom';
1821         } elsif ($device eq 'e1000') {
1822             $romfile = 'pxe-e1000.rom';
1823         } elsif ($device eq 'e1000e') {
1824             $romfile = 'pxe-e1000e.rom';
1825         } elsif ($device eq 'ne2k') {
1826             $romfile = 'pxe-ne2k_pci.rom';
1827         } elsif ($device eq 'pcnet') {
1828             $romfile = 'pxe-pcnet.rom';
1829         } elsif ($device eq 'rtl8139') {
1830             $romfile = 'pxe-rtl8139.rom';
1831         }
1832         $tmpstr .= ",romfile=$romfile" if $romfile;
1833     }
1834
1835     return $tmpstr;
1836 }
1837
1838 sub print_netdev_full {
1839     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1840
1841     my $i = '';
1842     if ($netid =~ m/^net(\d+)$/) {
1843         $i = int($1);
1844     }
1845
1846     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1847
1848     my $ifname = "tap${vmid}i$i";
1849
1850     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1851     die "interface name '$ifname' is too long (max 15 character)\n"
1852         if length($ifname) >= 16;
1853
1854     my $vhostparam = '';
1855     if (is_native($arch)) {
1856         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1857     }
1858
1859     my $vmname = $conf->{name} || "vm$vmid";
1860
1861     my $netdev = "";
1862     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1863
1864     if ($net->{bridge}) {
1865         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1866             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1867     } else {
1868         $netdev = "type=user,id=$netid,hostname=$vmname";
1869     }
1870
1871     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1872
1873     return $netdev;
1874 }
1875
1876 my $vga_map = {
1877     'cirrus' => 'cirrus-vga',
1878     'std' => 'VGA',
1879     'vmware' => 'vmware-svga',
1880     'virtio' => 'virtio-vga',
1881     'virtio-gl' => 'virtio-vga-gl',
1882 };
1883
1884 sub print_vga_device {
1885     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1886
1887     my $type = $vga_map->{$vga->{type}};
1888     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1889         $type = 'virtio-gpu';
1890     }
1891     my $vgamem_mb = $vga->{memory};
1892
1893     my $max_outputs = '';
1894     if ($qxlnum) {
1895         $type = $id ? 'qxl' : 'qxl-vga';
1896
1897         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1898             # set max outputs so linux can have up to 4 qxl displays with one device
1899             if (min_version($machine_version, 4, 1)) {
1900                 $max_outputs = ",max_outputs=4";
1901             }
1902         }
1903     }
1904
1905     die "no devicetype for $vga->{type}\n" if !$type;
1906
1907     my $memory = "";
1908     if ($vgamem_mb) {
1909         if ($vga->{type} =~ /^virtio/) {
1910             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1911             $memory = ",max_hostmem=$bytes";
1912         } elsif ($qxlnum) {
1913             # from https://www.spice-space.org/multiple-monitors.html
1914             $memory = ",vgamem_mb=$vga->{memory}";
1915             my $ram = $vgamem_mb * 4;
1916             my $vram = $vgamem_mb * 2;
1917             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1918         } else {
1919             $memory = ",vgamem_mb=$vga->{memory}";
1920         }
1921     } elsif ($qxlnum && $id) {
1922         $memory = ",ram_size=67108864,vram_size=33554432";
1923     }
1924
1925     my $edidoff = "";
1926     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1927         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1928     }
1929
1930     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1931     my $vgaid = "vga" . ($id // '');
1932     my $pciaddr;
1933     if ($q35 && $vgaid eq 'vga') {
1934         # the first display uses pcie.0 bus on q35 machines
1935         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1936     } else {
1937         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1938     }
1939
1940     if ($vga->{type} eq 'virtio-gl') {
1941         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1942         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1943             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1944
1945         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1946             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1947     }
1948
1949     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1950 }
1951
1952 sub parse_number_sets {
1953     my ($set) = @_;
1954     my $res = [];
1955     foreach my $part (split(/;/, $set)) {
1956         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1957             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1958             push @$res, [ $1, $2 ];
1959         } else {
1960             die "invalid range: $part\n";
1961         }
1962     }
1963     return $res;
1964 }
1965
1966 sub parse_numa {
1967     my ($data) = @_;
1968
1969     my $res = parse_property_string($numa_fmt, $data);
1970     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1971     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1972     return $res;
1973 }
1974
1975 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1976 sub parse_net {
1977     my ($data, $disable_mac_autogen) = @_;
1978
1979     my $res = eval { parse_property_string($net_fmt, $data) };
1980     if ($@) {
1981         warn $@;
1982         return;
1983     }
1984     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1985         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1986         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1987     }
1988     return $res;
1989 }
1990
1991 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1992 sub parse_ipconfig {
1993     my ($data) = @_;
1994
1995     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1996     if ($@) {
1997         warn $@;
1998         return;
1999     }
2000
2001     if ($res->{gw} && !$res->{ip}) {
2002         warn 'gateway specified without specifying an IP address';
2003         return;
2004     }
2005     if ($res->{gw6} && !$res->{ip6}) {
2006         warn 'IPv6 gateway specified without specifying an IPv6 address';
2007         return;
2008     }
2009     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2010         warn 'gateway specified together with DHCP';
2011         return;
2012     }
2013     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2014         # gw6 + auto/dhcp
2015         warn "IPv6 gateway specified together with $res->{ip6} address";
2016         return;
2017     }
2018
2019     if (!$res->{ip} && !$res->{ip6}) {
2020         return { ip => 'dhcp', ip6 => 'dhcp' };
2021     }
2022
2023     return $res;
2024 }
2025
2026 sub print_net {
2027     my $net = shift;
2028
2029     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2030 }
2031
2032 sub add_random_macs {
2033     my ($settings) = @_;
2034
2035     foreach my $opt (keys %$settings) {
2036         next if $opt !~ m/^net(\d+)$/;
2037         my $net = parse_net($settings->{$opt});
2038         next if !$net;
2039         $settings->{$opt} = print_net($net);
2040     }
2041 }
2042
2043 sub vm_is_volid_owner {
2044     my ($storecfg, $vmid, $volid) = @_;
2045
2046     if ($volid !~  m|^/|) {
2047         my ($path, $owner);
2048         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2049         if ($owner && ($owner == $vmid)) {
2050             return 1;
2051         }
2052     }
2053
2054     return;
2055 }
2056
2057 sub vmconfig_register_unused_drive {
2058     my ($storecfg, $vmid, $conf, $drive) = @_;
2059
2060     if (drive_is_cloudinit($drive)) {
2061         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2062         warn $@ if $@;
2063         delete $conf->{cloudinit};
2064     } elsif (!drive_is_cdrom($drive)) {
2065         my $volid = $drive->{file};
2066         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2067             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2068         }
2069     }
2070 }
2071
2072 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2073 my $smbios1_fmt = {
2074     uuid => {
2075         type => 'string',
2076         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2077         format_description => 'UUID',
2078         description => "Set SMBIOS1 UUID.",
2079         optional => 1,
2080     },
2081     version => {
2082         type => 'string',
2083         pattern => '[A-Za-z0-9+\/]+={0,2}',
2084         format_description => 'Base64 encoded string',
2085         description => "Set SMBIOS1 version.",
2086         optional => 1,
2087     },
2088     serial => {
2089         type => 'string',
2090         pattern => '[A-Za-z0-9+\/]+={0,2}',
2091         format_description => 'Base64 encoded string',
2092         description => "Set SMBIOS1 serial number.",
2093         optional => 1,
2094     },
2095     manufacturer => {
2096         type => 'string',
2097         pattern => '[A-Za-z0-9+\/]+={0,2}',
2098         format_description => 'Base64 encoded string',
2099         description => "Set SMBIOS1 manufacturer.",
2100         optional => 1,
2101     },
2102     product => {
2103         type => 'string',
2104         pattern => '[A-Za-z0-9+\/]+={0,2}',
2105         format_description => 'Base64 encoded string',
2106         description => "Set SMBIOS1 product ID.",
2107         optional => 1,
2108     },
2109     sku => {
2110         type => 'string',
2111         pattern => '[A-Za-z0-9+\/]+={0,2}',
2112         format_description => 'Base64 encoded string',
2113         description => "Set SMBIOS1 SKU string.",
2114         optional => 1,
2115     },
2116     family => {
2117         type => 'string',
2118         pattern => '[A-Za-z0-9+\/]+={0,2}',
2119         format_description => 'Base64 encoded string',
2120         description => "Set SMBIOS1 family string.",
2121         optional => 1,
2122     },
2123     base64 => {
2124         type => 'boolean',
2125         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2126         optional => 1,
2127     },
2128 };
2129
2130 sub parse_smbios1 {
2131     my ($data) = @_;
2132
2133     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2134     warn $@ if $@;
2135     return $res;
2136 }
2137
2138 sub print_smbios1 {
2139     my ($smbios1) = @_;
2140     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2141 }
2142
2143 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2144
2145 sub parse_watchdog {
2146     my ($value) = @_;
2147
2148     return if !$value;
2149
2150     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2151     warn $@ if $@;
2152     return $res;
2153 }
2154
2155 sub parse_guest_agent {
2156     my ($conf) = @_;
2157
2158     return {} if !defined($conf->{agent});
2159
2160     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2161     warn $@ if $@;
2162
2163     # if the agent is disabled ignore the other potentially set properties
2164     return {} if !$res->{enabled};
2165     return $res;
2166 }
2167
2168 sub get_qga_key {
2169     my ($conf, $key) = @_;
2170     return undef if !defined($conf->{agent});
2171
2172     my $agent = parse_guest_agent($conf);
2173     return $agent->{$key};
2174 }
2175
2176 sub parse_vga {
2177     my ($value) = @_;
2178
2179     return {} if !$value;
2180     my $res = eval { parse_property_string($vga_fmt, $value) };
2181     warn $@ if $@;
2182     return $res;
2183 }
2184
2185 sub parse_rng {
2186     my ($value) = @_;
2187
2188     return if !$value;
2189
2190     my $res = eval { parse_property_string($rng_fmt, $value) };
2191     warn $@ if $@;
2192     return $res;
2193 }
2194
2195 sub parse_meta_info {
2196     my ($value) = @_;
2197
2198     return if !$value;
2199
2200     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2201     warn $@ if $@;
2202     return $res;
2203 }
2204
2205 sub new_meta_info_string {
2206     my () = @_; # for now do not allow to override any value
2207
2208     return PVE::JSONSchema::print_property_string(
2209         {
2210             'creation-qemu' => kvm_user_version(),
2211             ctime => "". int(time()),
2212         },
2213         $meta_info_fmt
2214     );
2215 }
2216
2217 sub qemu_created_version_fixups {
2218     my ($conf, $forcemachine, $kvmver) = @_;
2219
2220     my $meta = parse_meta_info($conf->{meta}) // {};
2221     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2222
2223     # check if we need to apply some handling for VMs that always use the latest machine version but
2224     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2225     # would be required (we do not want to pin machine version for non-windows OS type)
2226     if (
2227         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2228         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2229         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2230         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2231     ) {
2232         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2233         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2234             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2235             # and thus with the predictable interface naming of systemd
2236             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2237         }
2238     }
2239     return;
2240 }
2241
2242 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2243 sub verify_usb_device {
2244     my ($value, $noerr) = @_;
2245
2246     return $value if parse_usb_device($value);
2247
2248     return if $noerr;
2249
2250     die "unable to parse usb device\n";
2251 }
2252
2253 # add JSON properties for create and set function
2254 sub json_config_properties {
2255     my ($prop, $with_disk_alloc) = @_;
2256
2257     my $skip_json_config_opts = {
2258         parent => 1,
2259         snaptime => 1,
2260         vmstate => 1,
2261         runningmachine => 1,
2262         runningcpu => 1,
2263         meta => 1,
2264     };
2265
2266     foreach my $opt (keys %$confdesc) {
2267         next if $skip_json_config_opts->{$opt};
2268
2269         if ($with_disk_alloc && is_valid_drivename($opt)) {
2270             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2271         } else {
2272             $prop->{$opt} = $confdesc->{$opt};
2273         }
2274     }
2275
2276     return $prop;
2277 }
2278
2279 # Properties that we can read from an OVF file
2280 sub json_ovf_properties {
2281     my $prop = {};
2282
2283     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2284         $prop->{$device} = {
2285             type => 'string',
2286             format => 'pve-volume-id-or-absolute-path',
2287             description => "Disk image that gets imported to $device",
2288             optional => 1,
2289         };
2290     }
2291
2292     $prop->{cores} = {
2293         type => 'integer',
2294         description => "The number of CPU cores.",
2295         optional => 1,
2296     };
2297     $prop->{memory} = {
2298         type => 'integer',
2299         description => "Amount of RAM for the VM in MB.",
2300         optional => 1,
2301     };
2302     $prop->{name} = {
2303         type => 'string',
2304         description => "Name of the VM.",
2305         optional => 1,
2306     };
2307
2308     return $prop;
2309 }
2310
2311 # return copy of $confdesc_cloudinit to generate documentation
2312 sub cloudinit_config_properties {
2313
2314     return dclone($confdesc_cloudinit);
2315 }
2316
2317 sub cloudinit_pending_properties {
2318     my $p = {
2319         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2320         name => 1,
2321     };
2322     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2323     return $p;
2324 }
2325
2326 sub check_type {
2327     my ($key, $value) = @_;
2328
2329     die "unknown setting '$key'\n" if !$confdesc->{$key};
2330
2331     my $type = $confdesc->{$key}->{type};
2332
2333     if (!defined($value)) {
2334         die "got undefined value\n";
2335     }
2336
2337     if ($value =~ m/[\n\r]/) {
2338         die "property contains a line feed\n";
2339     }
2340
2341     if ($type eq 'boolean') {
2342         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2343         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2344         die "type check ('boolean') failed - got '$value'\n";
2345     } elsif ($type eq 'integer') {
2346         return int($1) if $value =~ m/^(\d+)$/;
2347         die "type check ('integer') failed - got '$value'\n";
2348     } elsif ($type eq 'number') {
2349         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2350         die "type check ('number') failed - got '$value'\n";
2351     } elsif ($type eq 'string') {
2352         if (my $fmt = $confdesc->{$key}->{format}) {
2353             PVE::JSONSchema::check_format($fmt, $value);
2354             return $value;
2355         }
2356         $value =~ s/^\"(.*)\"$/$1/;
2357         return $value;
2358     } else {
2359         die "internal error"
2360     }
2361 }
2362
2363 sub destroy_vm {
2364     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2365
2366     my $conf = PVE::QemuConfig->load_config($vmid);
2367
2368     if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2369         PVE::QemuConfig->check_lock($conf);
2370     }
2371
2372     if ($conf->{template}) {
2373         # check if any base image is still used by a linked clone
2374         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2375                 my ($ds, $drive) = @_;
2376                 return if drive_is_cdrom($drive);
2377
2378                 my $volid = $drive->{file};
2379                 return if !$volid || $volid =~ m|^/|;
2380
2381                 die "base volume '$volid' is still in use by linked cloned\n"
2382                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2383
2384         });
2385     }
2386
2387     my $volids = {};
2388     my $remove_owned_drive = sub {
2389         my ($ds, $drive) = @_;
2390         return if drive_is_cdrom($drive, 1);
2391
2392         my $volid = $drive->{file};
2393         return if !$volid || $volid =~ m|^/|;
2394         return if $volids->{$volid};
2395
2396         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2397         return if !$path || !$owner || ($owner != $vmid);
2398
2399         $volids->{$volid} = 1;
2400         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2401         warn "Could not remove disk '$volid', check manually: $@" if $@;
2402     };
2403
2404     # only remove disks owned by this VM (referenced in the config)
2405     my $include_opts = {
2406         include_unused => 1,
2407         extra_keys => ['vmstate'],
2408     };
2409     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2410
2411     for my $snap (values %{$conf->{snapshots}}) {
2412         next if !defined($snap->{vmstate});
2413         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2414         next if !defined($drive);
2415         $remove_owned_drive->('vmstate', $drive);
2416     }
2417
2418     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2419
2420     if ($purge_unreferenced) { # also remove unreferenced disk
2421         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2422         PVE::Storage::foreach_volid($vmdisks, sub {
2423             my ($volid, $sid, $volname, $d) = @_;
2424             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2425             warn $@ if $@;
2426         });
2427     }
2428
2429     if (defined $replacement_conf) {
2430         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2431     } else {
2432         PVE::QemuConfig->destroy_config($vmid);
2433     }
2434 }
2435
2436 sub parse_vm_config {
2437     my ($filename, $raw, $strict) = @_;
2438
2439     return if !defined($raw);
2440
2441     my $res = {
2442         digest => Digest::SHA::sha1_hex($raw),
2443         snapshots => {},
2444         pending => {},
2445         cloudinit => {},
2446     };
2447
2448     my $handle_error = sub {
2449         my ($msg) = @_;
2450
2451         if ($strict) {
2452             die $msg;
2453         } else {
2454             warn $msg;
2455         }
2456     };
2457
2458     $filename =~ m|/qemu-server/(\d+)\.conf$|
2459         || die "got strange filename '$filename'";
2460
2461     my $vmid = $1;
2462
2463     my $conf = $res;
2464     my $descr;
2465     my $finish_description = sub {
2466         if (defined($descr)) {
2467             $descr =~ s/\s+$//;
2468             $conf->{description} = $descr;
2469         }
2470         $descr = undef;
2471     };
2472     my $section = '';
2473
2474     my @lines = split(/\n/, $raw);
2475     foreach my $line (@lines) {
2476         next if $line =~ m/^\s*$/;
2477
2478         if ($line =~ m/^\[PENDING\]\s*$/i) {
2479             $section = 'pending';
2480             $finish_description->();
2481             $conf = $res->{$section} = {};
2482             next;
2483         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2484             $section = 'cloudinit';
2485             $finish_description->();
2486             $conf = $res->{$section} = {};
2487             next;
2488
2489         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2490             $section = $1;
2491             $finish_description->();
2492             $conf = $res->{snapshots}->{$section} = {};
2493             next;
2494         }
2495
2496         if ($line =~ m/^\#(.*)$/) {
2497             $descr = '' if !defined($descr);
2498             $descr .= PVE::Tools::decode_text($1) . "\n";
2499             next;
2500         }
2501
2502         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2503             $descr = '' if !defined($descr);
2504             $descr .= PVE::Tools::decode_text($2);
2505         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2506             $conf->{snapstate} = $1;
2507         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2508             my $key = $1;
2509             my $value = $2;
2510             $conf->{$key} = $value;
2511         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2512             my $value = $1;
2513             if ($section eq 'pending') {
2514                 $conf->{delete} = $value; # we parse this later
2515             } else {
2516                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2517             }
2518         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2519             my $key = $1;
2520             my $value = $2;
2521             if ($section eq 'cloudinit') {
2522                 # ignore validation only used for informative purpose
2523                 $conf->{$key} = $value;
2524                 next;
2525             }
2526             eval { $value = check_type($key, $value); };
2527             if ($@) {
2528                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2529             } else {
2530                 $key = 'ide2' if $key eq 'cdrom';
2531                 my $fmt = $confdesc->{$key}->{format};
2532                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2533                     my $v = parse_drive($key, $value);
2534                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2535                         $v->{file} = $volid;
2536                         $value = print_drive($v);
2537                     } else {
2538                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2539                         next;
2540                     }
2541                 }
2542
2543                 $conf->{$key} = $value;
2544             }
2545         } else {
2546             $handle_error->("vm $vmid - unable to parse config: $line\n");
2547         }
2548     }
2549
2550     $finish_description->();
2551     delete $res->{snapstate}; # just to be sure
2552
2553     return $res;
2554 }
2555
2556 sub write_vm_config {
2557     my ($filename, $conf) = @_;
2558
2559     delete $conf->{snapstate}; # just to be sure
2560
2561     if ($conf->{cdrom}) {
2562         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2563         $conf->{ide2} = $conf->{cdrom};
2564         delete $conf->{cdrom};
2565     }
2566
2567     # we do not use 'smp' any longer
2568     if ($conf->{sockets}) {
2569         delete $conf->{smp};
2570     } elsif ($conf->{smp}) {
2571         $conf->{sockets} = $conf->{smp};
2572         delete $conf->{cores};
2573         delete $conf->{smp};
2574     }
2575
2576     my $used_volids = {};
2577
2578     my $cleanup_config = sub {
2579         my ($cref, $pending, $snapname) = @_;
2580
2581         foreach my $key (keys %$cref) {
2582             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2583                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2584             my $value = $cref->{$key};
2585             if ($key eq 'delete') {
2586                 die "propertry 'delete' is only allowed in [PENDING]\n"
2587                     if !$pending;
2588                 # fixme: check syntax?
2589                 next;
2590             }
2591             eval { $value = check_type($key, $value); };
2592             die "unable to parse value of '$key' - $@" if $@;
2593
2594             $cref->{$key} = $value;
2595
2596             if (!$snapname && is_valid_drivename($key)) {
2597                 my $drive = parse_drive($key, $value);
2598                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2599             }
2600         }
2601     };
2602
2603     &$cleanup_config($conf);
2604
2605     &$cleanup_config($conf->{pending}, 1);
2606
2607     foreach my $snapname (keys %{$conf->{snapshots}}) {
2608         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2609         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2610     }
2611
2612     # remove 'unusedX' settings if we re-add a volume
2613     foreach my $key (keys %$conf) {
2614         my $value = $conf->{$key};
2615         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2616             delete $conf->{$key};
2617         }
2618     }
2619
2620     my $generate_raw_config = sub {
2621         my ($conf, $pending) = @_;
2622
2623         my $raw = '';
2624
2625         # add description as comment to top of file
2626         if (defined(my $descr = $conf->{description})) {
2627             if ($descr) {
2628                 foreach my $cl (split(/\n/, $descr)) {
2629                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2630                 }
2631             } else {
2632                 $raw .= "#\n" if $pending;
2633             }
2634         }
2635
2636         foreach my $key (sort keys %$conf) {
2637             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2638             $raw .= "$key: $conf->{$key}\n";
2639         }
2640         return $raw;
2641     };
2642
2643     my $raw = &$generate_raw_config($conf);
2644
2645     if (scalar(keys %{$conf->{pending}})){
2646         $raw .= "\n[PENDING]\n";
2647         $raw .= &$generate_raw_config($conf->{pending}, 1);
2648     }
2649
2650     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2651         $raw .= "\n[special:cloudinit]\n";
2652         $raw .= &$generate_raw_config($conf->{cloudinit});
2653     }
2654
2655     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2656         $raw .= "\n[$snapname]\n";
2657         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2658     }
2659
2660     return $raw;
2661 }
2662
2663 sub load_defaults {
2664
2665     my $res = {};
2666
2667     # we use static defaults from our JSON schema configuration
2668     foreach my $key (keys %$confdesc) {
2669         if (defined(my $default = $confdesc->{$key}->{default})) {
2670             $res->{$key} = $default;
2671         }
2672     }
2673
2674     return $res;
2675 }
2676
2677 sub config_list {
2678     my $vmlist = PVE::Cluster::get_vmlist();
2679     my $res = {};
2680     return $res if !$vmlist || !$vmlist->{ids};
2681     my $ids = $vmlist->{ids};
2682     my $nodename = nodename();
2683
2684     foreach my $vmid (keys %$ids) {
2685         my $d = $ids->{$vmid};
2686         next if !$d->{node} || $d->{node} ne $nodename;
2687         next if !$d->{type} || $d->{type} ne 'qemu';
2688         $res->{$vmid}->{exists} = 1;
2689     }
2690     return $res;
2691 }
2692
2693 # test if VM uses local resources (to prevent migration)
2694 sub check_local_resources {
2695     my ($conf, $noerr) = @_;
2696
2697     my @loc_res = ();
2698
2699     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2700     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2701
2702     push @loc_res, "ivshmem" if $conf->{ivshmem};
2703
2704     foreach my $k (keys %$conf) {
2705         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2706         # sockets are safe: they will recreated be on the target side post-migrate
2707         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2708         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2709     }
2710
2711     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2712
2713     return \@loc_res;
2714 }
2715
2716 # check if used storages are available on all nodes (use by migrate)
2717 sub check_storage_availability {
2718     my ($storecfg, $conf, $node) = @_;
2719
2720     PVE::QemuConfig->foreach_volume($conf, sub {
2721         my ($ds, $drive) = @_;
2722
2723         my $volid = $drive->{file};
2724         return if !$volid;
2725
2726         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2727         return if !$sid;
2728
2729         # check if storage is available on both nodes
2730         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2731         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2732
2733         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2734
2735         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2736             if !$scfg->{content}->{$vtype};
2737    });
2738 }
2739
2740 # list nodes where all VM images are available (used by has_feature API)
2741 sub shared_nodes {
2742     my ($conf, $storecfg) = @_;
2743
2744     my $nodelist = PVE::Cluster::get_nodelist();
2745     my $nodehash = { map { $_ => 1 } @$nodelist };
2746     my $nodename = nodename();
2747
2748     PVE::QemuConfig->foreach_volume($conf, sub {
2749         my ($ds, $drive) = @_;
2750
2751         my $volid = $drive->{file};
2752         return if !$volid;
2753
2754         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2755         if ($storeid) {
2756             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2757             if ($scfg->{disable}) {
2758                 $nodehash = {};
2759             } elsif (my $avail = $scfg->{nodes}) {
2760                 foreach my $node (keys %$nodehash) {
2761                     delete $nodehash->{$node} if !$avail->{$node};
2762                 }
2763             } elsif (!$scfg->{shared}) {
2764                 foreach my $node (keys %$nodehash) {
2765                     delete $nodehash->{$node} if $node ne $nodename
2766                 }
2767             }
2768         }
2769     });
2770
2771     return $nodehash
2772 }
2773
2774 sub check_local_storage_availability {
2775     my ($conf, $storecfg) = @_;
2776
2777     my $nodelist = PVE::Cluster::get_nodelist();
2778     my $nodehash = { map { $_ => {} } @$nodelist };
2779
2780     PVE::QemuConfig->foreach_volume($conf, sub {
2781         my ($ds, $drive) = @_;
2782
2783         my $volid = $drive->{file};
2784         return if !$volid;
2785
2786         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2787         if ($storeid) {
2788             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2789
2790             if ($scfg->{disable}) {
2791                 foreach my $node (keys %$nodehash) {
2792                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2793                 }
2794             } elsif (my $avail = $scfg->{nodes}) {
2795                 foreach my $node (keys %$nodehash) {
2796                     if (!$avail->{$node}) {
2797                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2798                     }
2799                 }
2800             }
2801         }
2802     });
2803
2804     foreach my $node (values %$nodehash) {
2805         if (my $unavail = $node->{unavailable_storages}) {
2806             $node->{unavailable_storages} = [ sort keys %$unavail ];
2807         }
2808     }
2809
2810     return $nodehash
2811 }
2812
2813 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2814 sub check_running {
2815     my ($vmid, $nocheck, $node) = @_;
2816
2817     # $nocheck is set when called during a migration, in which case the config
2818     # file might still or already reside on the *other* node
2819     # - because rename has already happened, and current node is source
2820     # - because rename hasn't happened yet, and current node is target
2821     # - because rename has happened, current node is target, but hasn't yet
2822     # processed it yet
2823     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2824     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2825 }
2826
2827 sub vzlist {
2828
2829     my $vzlist = config_list();
2830
2831     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2832
2833     while (defined(my $de = $fd->read)) {
2834         next if $de !~ m/^(\d+)\.pid$/;
2835         my $vmid = $1;
2836         next if !defined($vzlist->{$vmid});
2837         if (my $pid = check_running($vmid)) {
2838             $vzlist->{$vmid}->{pid} = $pid;
2839         }
2840     }
2841
2842     return $vzlist;
2843 }
2844
2845 our $vmstatus_return_properties = {
2846     vmid => get_standard_option('pve-vmid'),
2847     status => {
2848         description => "QEMU process status.",
2849         type => 'string',
2850         enum => ['stopped', 'running'],
2851     },
2852     maxmem => {
2853         description => "Maximum memory in bytes.",
2854         type => 'integer',
2855         optional => 1,
2856         renderer => 'bytes',
2857     },
2858     maxdisk => {
2859         description => "Root disk size in bytes.",
2860         type => 'integer',
2861         optional => 1,
2862         renderer => 'bytes',
2863     },
2864     name => {
2865         description => "VM name.",
2866         type => 'string',
2867         optional => 1,
2868     },
2869     qmpstatus => {
2870         description => "QEMU QMP agent status.",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     pid => {
2875         description => "PID of running qemu process.",
2876         type => 'integer',
2877         optional => 1,
2878     },
2879     uptime => {
2880         description => "Uptime.",
2881         type => 'integer',
2882         optional => 1,
2883         renderer => 'duration',
2884     },
2885     cpus => {
2886         description => "Maximum usable CPUs.",
2887         type => 'number',
2888         optional => 1,
2889     },
2890     lock => {
2891         description => "The current config lock, if any.",
2892         type => 'string',
2893         optional => 1,
2894     },
2895     tags => {
2896         description  => "The current configured tags, if any",
2897         type => 'string',
2898         optional => 1,
2899     },
2900     'running-machine' => {
2901         description => "The currently running machine type (if running).",
2902         type => 'string',
2903         optional => 1,
2904     },
2905     'running-qemu' => {
2906         description => "The currently running QEMU version (if running).",
2907         type => 'string',
2908         optional => 1,
2909     },
2910 };
2911
2912 my $last_proc_pid_stat;
2913
2914 # get VM status information
2915 # This must be fast and should not block ($full == false)
2916 # We only query KVM using QMP if $full == true (this can be slow)
2917 sub vmstatus {
2918     my ($opt_vmid, $full) = @_;
2919
2920     my $res = {};
2921
2922     my $storecfg = PVE::Storage::config();
2923
2924     my $list = vzlist();
2925     my $defaults = load_defaults();
2926
2927     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2928
2929     my $cpucount = $cpuinfo->{cpus} || 1;
2930
2931     foreach my $vmid (keys %$list) {
2932         next if $opt_vmid && ($vmid ne $opt_vmid);
2933
2934         my $conf = PVE::QemuConfig->load_config($vmid);
2935
2936         my $d = { vmid => int($vmid) };
2937         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2938
2939         # fixme: better status?
2940         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2941
2942         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2943         if (defined($size)) {
2944             $d->{disk} = 0; # no info available
2945             $d->{maxdisk} = $size;
2946         } else {
2947             $d->{disk} = 0;
2948             $d->{maxdisk} = 0;
2949         }
2950
2951         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2952             * ($conf->{cores} || $defaults->{cores});
2953         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2954         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2955
2956         $d->{name} = $conf->{name} || "VM $vmid";
2957         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2958             : $defaults->{memory}*(1024*1024);
2959
2960         if ($conf->{balloon}) {
2961             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2962             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2963                 : $defaults->{shares};
2964         }
2965
2966         $d->{uptime} = 0;
2967         $d->{cpu} = 0;
2968         $d->{mem} = 0;
2969
2970         $d->{netout} = 0;
2971         $d->{netin} = 0;
2972
2973         $d->{diskread} = 0;
2974         $d->{diskwrite} = 0;
2975
2976         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2977
2978         $d->{serial} = 1 if conf_has_serial($conf);
2979         $d->{lock} = $conf->{lock} if $conf->{lock};
2980         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2981
2982         $res->{$vmid} = $d;
2983     }
2984
2985     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2986     foreach my $dev (keys %$netdev) {
2987         next if $dev !~ m/^tap([1-9]\d*)i/;
2988         my $vmid = $1;
2989         my $d = $res->{$vmid};
2990         next if !$d;
2991
2992         $d->{netout} += $netdev->{$dev}->{receive};
2993         $d->{netin} += $netdev->{$dev}->{transmit};
2994
2995         if ($full) {
2996             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2997             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2998         }
2999
3000     }
3001
3002     my $ctime = gettimeofday;
3003
3004     foreach my $vmid (keys %$list) {
3005
3006         my $d = $res->{$vmid};
3007         my $pid = $d->{pid};
3008         next if !$pid;
3009
3010         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3011         next if !$pstat; # not running
3012
3013         my $used = $pstat->{utime} + $pstat->{stime};
3014
3015         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3016
3017         if ($pstat->{vsize}) {
3018             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3019         }
3020
3021         my $old = $last_proc_pid_stat->{$pid};
3022         if (!$old) {
3023             $last_proc_pid_stat->{$pid} = {
3024                 time => $ctime,
3025                 used => $used,
3026                 cpu => 0,
3027             };
3028             next;
3029         }
3030
3031         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3032
3033         if ($dtime > 1000) {
3034             my $dutime = $used -  $old->{used};
3035
3036             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3037             $last_proc_pid_stat->{$pid} = {
3038                 time => $ctime,
3039                 used => $used,
3040                 cpu => $d->{cpu},
3041             };
3042         } else {
3043             $d->{cpu} = $old->{cpu};
3044         }
3045     }
3046
3047     return $res if !$full;
3048
3049     my $qmpclient = PVE::QMPClient->new();
3050
3051     my $ballooncb = sub {
3052         my ($vmid, $resp) = @_;
3053
3054         my $info = $resp->{'return'};
3055         return if !$info->{max_mem};
3056
3057         my $d = $res->{$vmid};
3058
3059         # use memory assigned to VM
3060         $d->{maxmem} = $info->{max_mem};
3061         $d->{balloon} = $info->{actual};
3062
3063         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3064             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3065             $d->{freemem} = $info->{free_mem};
3066         }
3067
3068         $d->{ballooninfo} = $info;
3069     };
3070
3071     my $blockstatscb = sub {
3072         my ($vmid, $resp) = @_;
3073         my $data = $resp->{'return'} || [];
3074         my $totalrdbytes = 0;
3075         my $totalwrbytes = 0;
3076
3077         for my $blockstat (@$data) {
3078             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3079             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3080
3081             $blockstat->{device} =~ s/drive-//;
3082             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3083         }
3084         $res->{$vmid}->{diskread} = $totalrdbytes;
3085         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3086     };
3087
3088     my $machinecb = sub {
3089         my ($vmid, $resp) = @_;
3090         my $data = $resp->{'return'} || [];
3091
3092         $res->{$vmid}->{'running-machine'} =
3093             PVE::QemuServer::Machine::current_from_query_machines($data);
3094     };
3095
3096     my $versioncb = sub {
3097         my ($vmid, $resp) = @_;
3098         my $data = $resp->{'return'} // {};
3099         my $version = 'unknown';
3100
3101         if (my $v = $data->{qemu}) {
3102             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3103         }
3104
3105         $res->{$vmid}->{'running-qemu'} = $version;
3106     };
3107
3108     my $statuscb = sub {
3109         my ($vmid, $resp) = @_;
3110
3111         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3112         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3113         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3114         # this fails if ballon driver is not loaded, so this must be
3115         # the last commnand (following command are aborted if this fails).
3116         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3117
3118         my $status = 'unknown';
3119         if (!defined($status = $resp->{'return'}->{status})) {
3120             warn "unable to get VM status\n";
3121             return;
3122         }
3123
3124         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3125     };
3126
3127     foreach my $vmid (keys %$list) {
3128         next if $opt_vmid && ($vmid ne $opt_vmid);
3129         next if !$res->{$vmid}->{pid}; # not running
3130         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3131     }
3132
3133     $qmpclient->queue_execute(undef, 2);
3134
3135     foreach my $vmid (keys %$list) {
3136         next if $opt_vmid && ($vmid ne $opt_vmid);
3137         next if !$res->{$vmid}->{pid}; #not running
3138
3139         # we can't use the $qmpclient since it might have already aborted on
3140         # 'query-balloon', but this might also fail for older versions...
3141         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3142         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3143     }
3144
3145     foreach my $vmid (keys %$list) {
3146         next if $opt_vmid && ($vmid ne $opt_vmid);
3147         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3148     }
3149
3150     return $res;
3151 }
3152
3153 sub conf_has_serial {
3154     my ($conf) = @_;
3155
3156     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3157         if ($conf->{"serial$i"}) {
3158             return 1;
3159         }
3160     }
3161
3162     return 0;
3163 }
3164
3165 sub conf_has_audio {
3166     my ($conf, $id) = @_;
3167
3168     $id //= 0;
3169     my $audio = $conf->{"audio$id"};
3170     return if !defined($audio);
3171
3172     my $audioproperties = parse_property_string($audio_fmt, $audio);
3173     my $audiodriver = $audioproperties->{driver} // 'spice';
3174
3175     return {
3176         dev => $audioproperties->{device},
3177         dev_id => "audiodev$id",
3178         backend => $audiodriver,
3179         backend_id => "$audiodriver-backend${id}",
3180     };
3181 }
3182
3183 sub audio_devs {
3184     my ($audio, $audiopciaddr, $machine_version) = @_;
3185
3186     my $devs = [];
3187
3188     my $id = $audio->{dev_id};
3189     my $audiodev = "";
3190     if (min_version($machine_version, 4, 2)) {
3191         $audiodev = ",audiodev=$audio->{backend_id}";
3192     }
3193
3194     if ($audio->{dev} eq 'AC97') {
3195         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3196     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3197         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3198         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3199         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3200     } else {
3201         die "unkown audio device '$audio->{dev}', implement me!";
3202     }
3203
3204     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3205
3206     return $devs;
3207 }
3208
3209 sub get_tpm_paths {
3210     my ($vmid) = @_;
3211     return {
3212         socket => "/var/run/qemu-server/$vmid.swtpm",
3213         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3214     };
3215 }
3216
3217 sub add_tpm_device {
3218     my ($vmid, $devices, $conf) = @_;
3219
3220     return if !$conf->{tpmstate0};
3221
3222     my $paths = get_tpm_paths($vmid);
3223
3224     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3225     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3226     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3227 }
3228
3229 sub start_swtpm {
3230     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3231
3232     return if !$tpmdrive;
3233
3234     my $state;
3235     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3236     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3237     if ($storeid) {
3238         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3239     } else {
3240         $state = $tpm->{file};
3241     }
3242
3243     my $paths = get_tpm_paths($vmid);
3244
3245     # during migration, we will get state from remote
3246     #
3247     if (!$migration) {
3248         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3249         my $setup_cmd = [
3250             "swtpm_setup",
3251             "--tpmstate",
3252             "file://$state",
3253             "--createek",
3254             "--create-ek-cert",
3255             "--create-platform-cert",
3256             "--lock-nvram",
3257             "--config",
3258             "/etc/swtpm_setup.conf", # do not use XDG configs
3259             "--runas",
3260             "0", # force creation as root, error if not possible
3261             "--not-overwrite", # ignore existing state, do not modify
3262         ];
3263
3264         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3265         # TPM 2.0 supports ECC crypto, use if possible
3266         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3267
3268         run_command($setup_cmd, outfunc => sub {
3269             print "swtpm_setup: $1\n";
3270         });
3271     }
3272
3273     # Used to distinguish different invocations in the log.
3274     my $log_prefix = "[id=" . int(time()) . "] ";
3275
3276     my $emulator_cmd = [
3277         "swtpm",
3278         "socket",
3279         "--tpmstate",
3280         "backend-uri=file://$state,mode=0600",
3281         "--ctrl",
3282         "type=unixio,path=$paths->{socket},mode=0600",
3283         "--pid",
3284         "file=$paths->{pid}",
3285         "--terminate", # terminate on QEMU disconnect
3286         "--daemon",
3287         "--log",
3288         "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3289     ];
3290     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3291     run_command($emulator_cmd, outfunc => sub { print $1; });
3292
3293     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3294     while (! -e $paths->{pid}) {
3295         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3296         usleep(50_000);
3297     }
3298
3299     # return untainted PID of swtpm daemon so it can be killed on error
3300     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3301     return $1;
3302 }
3303
3304 sub vga_conf_has_spice {
3305     my ($vga) = @_;
3306
3307     my $vgaconf = parse_vga($vga);
3308     my $vgatype = $vgaconf->{type};
3309     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3310
3311     return $1 || 1;
3312 }
3313
3314 sub is_native($) {
3315     my ($arch) = @_;
3316     return get_host_arch() eq $arch;
3317 }
3318
3319 sub get_vm_arch {
3320     my ($conf) = @_;
3321     return $conf->{arch} // get_host_arch();
3322 }
3323
3324 my $default_machines = {
3325     x86_64 => 'pc',
3326     aarch64 => 'virt',
3327 };
3328
3329 sub get_installed_machine_version {
3330     my ($kvmversion) = @_;
3331     $kvmversion = kvm_user_version() if !defined($kvmversion);
3332     $kvmversion =~ m/^(\d+\.\d+)/;
3333     return $1;
3334 }
3335
3336 sub windows_get_pinned_machine_version {
3337     my ($machine, $base_version, $kvmversion) = @_;
3338
3339     my $pin_version = $base_version;
3340     if (!defined($base_version) ||
3341         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3342     ) {
3343         $pin_version = get_installed_machine_version($kvmversion);
3344     }
3345     if (!$machine || $machine eq 'pc') {
3346         $machine = "pc-i440fx-$pin_version";
3347     } elsif ($machine eq 'q35') {
3348         $machine = "pc-q35-$pin_version";
3349     } elsif ($machine eq 'virt') {
3350         $machine = "virt-$pin_version";
3351     } else {
3352         warn "unknown machine type '$machine', not touching that!\n";
3353     }
3354
3355     return $machine;
3356 }
3357
3358 sub get_vm_machine {
3359     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3360
3361     my $machine = $forcemachine || $conf->{machine};
3362
3363     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3364         $kvmversion //= kvm_user_version();
3365         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3366         # layout which confuses windows quite a bit and may result in various regressions..
3367         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3368         if (windows_version($conf->{ostype})) {
3369             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3370         }
3371         $arch //= 'x86_64';
3372         $machine ||= $default_machines->{$arch};
3373         if ($add_pve_version) {
3374             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3375             $machine .= "+pve$pvever";
3376         }
3377     }
3378
3379     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3380         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3381         $machine = $1 if $is_pxe;
3382
3383         # for version-pinned machines that do not include a pve-version (e.g.
3384         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3385         $machine .= '+pve0';
3386
3387         $machine .= '.pxe' if $is_pxe;
3388     }
3389
3390     return $machine;
3391 }
3392
3393 sub get_ovmf_files($$$) {
3394     my ($arch, $efidisk, $smm) = @_;
3395
3396     my $types = $OVMF->{$arch}
3397         or die "no OVMF images known for architecture '$arch'\n";
3398
3399     my $type = 'default';
3400     if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3401         $type = $smm ? "4m" : "4m-no-smm";
3402         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3403     }
3404
3405     my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3406     die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3407     die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3408
3409     return ($ovmf_code, $ovmf_vars);
3410 }
3411
3412 my $Arch2Qemu = {
3413     aarch64 => '/usr/bin/qemu-system-aarch64',
3414     x86_64 => '/usr/bin/qemu-system-x86_64',
3415 };
3416 sub get_command_for_arch($) {
3417     my ($arch) = @_;
3418     return '/usr/bin/kvm' if is_native($arch);
3419
3420     my $cmd = $Arch2Qemu->{$arch}
3421         or die "don't know how to emulate architecture '$arch'\n";
3422     return $cmd;
3423 }
3424
3425 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3426 # to use in a QEMU command line (-cpu element), first array_intersect the result
3427 # of query_supported_ with query_understood_. This is necessary because:
3428 #
3429 # a) query_understood_ returns flags the host cannot use and
3430 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3431 #    flags, but CPU settings - with most of them being flags. Those settings
3432 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3433 #
3434 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3435 # expensive. If you need the value returned from this, you can get it much
3436 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3437 # $accel being 'kvm' or 'tcg'.
3438 #
3439 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3440 # changes, automatically populating pmxcfs.
3441 #
3442 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3443 # since kvm and tcg machines support different flags
3444 #
3445 sub query_supported_cpu_flags {
3446     my ($arch) = @_;
3447
3448     $arch //= get_host_arch();
3449     my $default_machine = $default_machines->{$arch};
3450
3451     my $flags = {};
3452
3453     # FIXME: Once this is merged, the code below should work for ARM as well:
3454     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3455     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3456         $arch eq "aarch64";
3457
3458     my $kvm_supported = defined(kvm_version());
3459     my $qemu_cmd = get_command_for_arch($arch);
3460     my $fakevmid = -1;
3461     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3462
3463     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3464     my $query_supported_run_qemu = sub {
3465         my ($kvm) = @_;
3466
3467         my $flags = {};
3468         my $cmd = [
3469             $qemu_cmd,
3470             '-machine', $default_machine,
3471             '-display', 'none',
3472             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3473             '-mon', 'chardev=qmp,mode=control',
3474             '-pidfile', $pidfile,
3475             '-S', '-daemonize'
3476         ];
3477
3478         if (!$kvm) {
3479             push @$cmd, '-accel', 'tcg';
3480         }
3481
3482         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3483         die "QEMU flag querying VM exited with code " . $rc if $rc;
3484
3485         eval {
3486             my $cmd_result = mon_cmd(
3487                 $fakevmid,
3488                 'query-cpu-model-expansion',
3489                 type => 'full',
3490                 model => { name => 'host' }
3491             );
3492
3493             my $props = $cmd_result->{model}->{props};
3494             foreach my $prop (keys %$props) {
3495                 next if $props->{$prop} ne '1';
3496                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3497                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3498                 # We only keep those with underscores, to match /proc/cpuinfo
3499                 $prop =~ s/\.|-/_/g;
3500                 $flags->{$prop} = 1;
3501             }
3502         };
3503         my $err = $@;
3504
3505         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3506         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3507
3508         die $err if $err;
3509
3510         return [ sort keys %$flags ];
3511     };
3512
3513     # We need to query QEMU twice, since KVM and TCG have different supported flags
3514     PVE::QemuConfig->lock_config($fakevmid, sub {
3515         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3516         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3517
3518         if ($kvm_supported) {
3519             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3520             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3521         }
3522     });
3523
3524     return $flags;
3525 }
3526
3527 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3528 my $understood_cpu_flag_dir = "/usr/share/kvm";
3529 sub query_understood_cpu_flags {
3530     my $arch = get_host_arch();
3531     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3532
3533     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3534         if ! -e $filepath;
3535
3536     my $raw = file_get_contents($filepath);
3537     $raw =~ s/^\s+|\s+$//g;
3538     my @flags = split(/\s+/, $raw);
3539
3540     return \@flags;
3541 }
3542
3543 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3544 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3545 my sub should_disable_smm {
3546     my ($conf, $vga, $machine) = @_;
3547
3548     return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3549
3550     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3551         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3552 }
3553
3554 my sub print_ovmf_drive_commandlines {
3555     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3556
3557     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3558
3559     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3560
3561     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3562     if ($d) {
3563         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3564         my ($path, $format) = $d->@{'file', 'format'};
3565         if ($storeid) {
3566             $path = PVE::Storage::path($storecfg, $d->{file});
3567             if (!defined($format)) {
3568                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3569                 $format = qemu_img_format($scfg, $volname);
3570             }
3571         } elsif (!defined($format)) {
3572             die "efidisk format must be specified\n";
3573         }
3574         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3575         if ($path =~ m/^rbd:/) {
3576             $var_drive_str .= ',cache=writeback';
3577             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3578         }
3579         $var_drive_str .= ",format=$format,file=$path";
3580
3581         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3582         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3583     } else {
3584         log_warn("no efidisk configured! Using temporary efivars disk.");
3585         my $path = "/tmp/$vmid-ovmf.fd";
3586         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3587         $var_drive_str .= ",format=raw,file=$path";
3588         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3589     }
3590
3591     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3592 }
3593
3594 sub config_to_command {
3595     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3596         $pbs_backing) = @_;
3597
3598     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3599     my $devices = [];
3600     my $bridges = {};
3601     my $ostype = $conf->{ostype};
3602     my $winversion = windows_version($ostype);
3603     my $kvm = $conf->{kvm};
3604     my $nodename = nodename();
3605
3606     my $arch = get_vm_arch($conf);
3607     my $kvm_binary = get_command_for_arch($arch);
3608     my $kvmver = kvm_user_version($kvm_binary);
3609
3610     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3611         $kvmver //= "undefined";
3612         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3613     }
3614
3615     my $add_pve_version = min_version($kvmver, 4, 1);
3616
3617     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3618     my $machine_version = extract_version($machine_type, $kvmver);
3619     $kvm //= 1 if is_native($arch);
3620
3621     $machine_version =~ m/(\d+)\.(\d+)/;
3622     my ($machine_major, $machine_minor) = ($1, $2);
3623
3624     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3625         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3626     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3627         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3628             ." please upgrade node '$nodename'\n"
3629     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3630         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3631         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3632             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3633             ." node '$nodename'\n";
3634     }
3635
3636     # if a specific +pve version is required for a feature, use $version_guard
3637     # instead of min_version to allow machines to be run with the minimum
3638     # required version
3639     my $required_pve_version = 0;
3640     my $version_guard = sub {
3641         my ($major, $minor, $pve) = @_;
3642         return 0 if !min_version($machine_version, $major, $minor, $pve);
3643         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3644         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3645         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3646         return 1;
3647     };
3648
3649     if ($kvm && !defined kvm_version()) {
3650         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3651             ." or enable in BIOS.\n";
3652     }
3653
3654     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3655     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3656     my $use_old_bios_files = undef;
3657     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3658
3659     my $cmd = [];
3660     if ($conf->{affinity}) {
3661         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3662     }
3663
3664     push @$cmd, $kvm_binary;
3665
3666     push @$cmd, '-id', $vmid;
3667
3668     my $vmname = $conf->{name} || "vm$vmid";
3669
3670     push @$cmd, '-name', "$vmname,debug-threads=on";
3671
3672     push @$cmd, '-no-shutdown';
3673
3674     my $use_virtio = 0;
3675
3676     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3677     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3678     push @$cmd, '-mon', "chardev=qmp,mode=control";
3679
3680     if (min_version($machine_version, 2, 12)) {
3681         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3682         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3683     }
3684
3685     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3686
3687     push @$cmd, '-daemonize';
3688
3689     if ($conf->{smbios1}) {
3690         my $smbios_conf = parse_smbios1($conf->{smbios1});
3691         if ($smbios_conf->{base64}) {
3692             # Do not pass base64 flag to qemu
3693             delete $smbios_conf->{base64};
3694             my $smbios_string = "";
3695             foreach my $key (keys %$smbios_conf) {
3696                 my $value;
3697                 if ($key eq "uuid") {
3698                     $value = $smbios_conf->{uuid}
3699                 } else {
3700                     $value = decode_base64($smbios_conf->{$key});
3701                 }
3702                 # qemu accepts any binary data, only commas need escaping by double comma
3703                 $value =~ s/,/,,/g;
3704                 $smbios_string .= "," . $key . "=" . $value if $value;
3705             }
3706             push @$cmd, '-smbios', "type=1" . $smbios_string;
3707         } else {
3708             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3709         }
3710     }
3711
3712     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3713         my ($code_drive_str, $var_drive_str) =
3714             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3715         push $cmd->@*, '-drive', $code_drive_str;
3716         push $cmd->@*, '-drive', $var_drive_str;
3717     }
3718
3719     if ($q35) { # tell QEMU to load q35 config early
3720         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3721         if (min_version($machine_version, 4, 0)) {
3722             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3723         } else {
3724             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3725         }
3726     }
3727
3728     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3729         push @$cmd, $fixups->@*;
3730     }
3731
3732     if ($conf->{vmgenid}) {
3733         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3734     }
3735
3736     # add usb controllers
3737     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3738         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3739     push @$devices, @usbcontrollers if @usbcontrollers;
3740     my $vga = parse_vga($conf->{vga});
3741
3742     my $qxlnum = vga_conf_has_spice($conf->{vga});
3743     $vga->{type} = 'qxl' if $qxlnum;
3744
3745     if (!$vga->{type}) {
3746         if ($arch eq 'aarch64') {
3747             $vga->{type} = 'virtio';
3748         } elsif (min_version($machine_version, 2, 9)) {
3749             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3750         } else {
3751             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3752         }
3753     }
3754
3755     # enable absolute mouse coordinates (needed by vnc)
3756     my $tablet = $conf->{tablet};
3757     if (!defined($tablet)) {
3758         $tablet = $defaults->{tablet};
3759         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3760         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3761     }
3762
3763     if ($tablet) {
3764         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3765         my $kbd = print_keyboarddevice_full($conf, $arch);
3766         push @$devices, '-device', $kbd if defined($kbd);
3767     }
3768
3769     my $bootorder = device_bootorder($conf);
3770
3771     # host pci device passthrough
3772     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3773         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3774
3775     # usb devices
3776     my $usb_dev_features = {};
3777     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3778
3779     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3780         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3781     push @$devices, @usbdevices if @usbdevices;
3782
3783     # serial devices
3784     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3785         my $path = $conf->{"serial$i"} or next;
3786         if ($path eq 'socket') {
3787             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3788             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3789             # On aarch64, serial0 is the UART device. QEMU only allows
3790             # connecting UART devices via the '-serial' command line, as
3791             # the device has a fixed slot on the hardware...
3792             if ($arch eq 'aarch64' && $i == 0) {
3793                 push @$devices, '-serial', "chardev:serial$i";
3794             } else {
3795                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3796             }
3797         } else {
3798             die "no such serial device\n" if ! -c $path;
3799             push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3800             push @$devices, '-device', "isa-serial,chardev=serial$i";
3801         }
3802     }
3803
3804     # parallel devices
3805     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3806         if (my $path = $conf->{"parallel$i"}) {
3807             die "no such parallel device\n" if ! -c $path;
3808             my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3809             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3810             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3811         }
3812     }
3813
3814     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3815         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3816         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3817         push @$devices, @$audio_devs;
3818     }
3819
3820     add_tpm_device($vmid, $devices, $conf);
3821
3822     my $sockets = 1;
3823     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3824     $sockets = $conf->{sockets} if  $conf->{sockets};
3825
3826     my $cores = $conf->{cores} || 1;
3827
3828     my $maxcpus = $sockets * $cores;
3829
3830     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3831
3832     my $allowed_vcpus = $cpuinfo->{cpus};
3833
3834     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3835
3836     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3837         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3838         for (my $i = 2; $i <= $vcpus; $i++)  {
3839             my $cpustr = print_cpu_device($conf,$i);
3840             push @$cmd, '-device', $cpustr;
3841         }
3842
3843     } else {
3844
3845         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3846     }
3847     push @$cmd, '-nodefaults';
3848
3849     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3850
3851     push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3852
3853     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3854
3855     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3856         push @$devices, '-device', print_vga_device(
3857             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3858
3859         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3860
3861         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3862         push @$cmd,  '-vnc', "unix:$socket,password=on";
3863     } else {
3864         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3865         push @$cmd, '-nographic';
3866     }
3867
3868     # time drift fix
3869     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3870     my $useLocaltime = $conf->{localtime};
3871
3872     if ($winversion >= 5) { # windows
3873         $useLocaltime = 1 if !defined($conf->{localtime});
3874
3875         # use time drift fix when acpi is enabled
3876         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3877             $tdf = 1 if !defined($conf->{tdf});
3878         }
3879     }
3880
3881     if ($winversion >= 6) {
3882         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3883         push @$cmd, '-no-hpet';
3884     }
3885
3886     push @$rtcFlags, 'driftfix=slew' if $tdf;
3887
3888     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3889         push @$rtcFlags, "base=$conf->{startdate}";
3890     } elsif ($useLocaltime) {
3891         push @$rtcFlags, 'base=localtime';
3892     }
3893
3894     if ($forcecpu) {
3895         push @$cmd, '-cpu', $forcecpu;
3896     } else {
3897         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3898     }
3899
3900     PVE::QemuServer::Memory::config(
3901         $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3902
3903     push @$cmd, '-S' if $conf->{freeze};
3904
3905     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3906
3907     my $guest_agent = parse_guest_agent($conf);
3908
3909     if ($guest_agent->{enabled}) {
3910         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3911         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3912
3913         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3914             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3915             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3916             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3917         } elsif ($guest_agent->{type} eq 'isa') {
3918             push @$devices, '-device', "isa-serial,chardev=qga0";
3919         }
3920     }
3921
3922     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3923     if ($rng && $version_guard->(4, 1, 2)) {
3924         check_rng_source($rng->{source});
3925
3926         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3927         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3928         my $limiter_str = "";
3929         if ($max_bytes) {
3930             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3931         }
3932
3933         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3934         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3935         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3936     }
3937
3938     my $spice_port;
3939
3940     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3941         if ($qxlnum > 1) {
3942             if ($winversion){
3943                 for (my $i = 1; $i < $qxlnum; $i++){
3944                     push @$devices, '-device', print_vga_device(
3945                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3946                 }
3947             } else {
3948                 # assume other OS works like Linux
3949                 my ($ram, $vram) = ("134217728", "67108864");
3950                 if ($vga->{memory}) {
3951                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3952                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3953                 }
3954                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3955                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3956             }
3957         }
3958
3959         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3960
3961         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3962         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3963         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3964
3965         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3966         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3967         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3968
3969         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3970         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3971
3972         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3973         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3974         if ($spice_enhancement->{foldersharing}) {
3975             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3976             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3977         }
3978
3979         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3980         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3981             if $spice_enhancement->{videostreaming};
3982
3983         push @$devices, '-spice', "$spice_opts";
3984     }
3985
3986     # enable balloon by default, unless explicitly disabled
3987     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3988         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3989         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3990         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3991         push @$devices, '-device', $ballooncmd;
3992     }
3993
3994     if ($conf->{watchdog}) {
3995         my $wdopts = parse_watchdog($conf->{watchdog});
3996         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3997         my $watchdog = $wdopts->{model} || 'i6300esb';
3998         push @$devices, '-device', "$watchdog$pciaddr";
3999         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4000     }
4001
4002     my $vollist = [];
4003     my $scsicontroller = {};
4004     my $ahcicontroller = {};
4005     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4006
4007     # Add iscsi initiator name if available
4008     if (my $initiator = get_initiator_name()) {
4009         push @$devices, '-iscsi', "initiator-name=$initiator";
4010     }
4011
4012     PVE::QemuConfig->foreach_volume($conf, sub {
4013         my ($ds, $drive) = @_;
4014
4015         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4016             check_volume_storage_type($storecfg, $drive->{file});
4017             push @$vollist, $drive->{file};
4018         }
4019
4020         # ignore efidisk here, already added in bios/fw handling code above
4021         return if $drive->{interface} eq 'efidisk';
4022         # similar for TPM
4023         return if $drive->{interface} eq 'tpmstate';
4024
4025         $use_virtio = 1 if $ds =~ m/^virtio/;
4026
4027         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
4028
4029         if ($drive->{interface} eq 'virtio'){
4030            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4031         }
4032
4033         if ($drive->{interface} eq 'scsi') {
4034
4035             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4036
4037             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4038                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4039
4040             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4041             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4042
4043             my $iothread = '';
4044             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4045                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4046                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4047             } elsif ($drive->{iothread}) {
4048                 log_warn(
4049                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4050                 );
4051             }
4052
4053             my $queues = '';
4054             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4055                 $queues = ",num_queues=$drive->{queues}";
4056             }
4057
4058             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4059                 if !$scsicontroller->{$controller};
4060             $scsicontroller->{$controller}=1;
4061         }
4062
4063         if ($drive->{interface} eq 'sata') {
4064             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4065             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4066             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4067                 if !$ahcicontroller->{$controller};
4068             $ahcicontroller->{$controller}=1;
4069         }
4070
4071         my $pbs_conf = $pbs_backing->{$ds};
4072         my $pbs_name = undef;
4073         if ($pbs_conf) {
4074             $pbs_name = "drive-$ds-pbs";
4075             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4076         }
4077
4078         my $drive_cmd = print_drive_commandline_full(
4079             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4080
4081         # extra protection for templates, but SATA and IDE don't support it..
4082         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4083
4084         push @$devices, '-drive',$drive_cmd;
4085         push @$devices, '-device', print_drivedevice_full(
4086             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4087     });
4088
4089     for (my $i = 0; $i < $MAX_NETS; $i++) {
4090         my $netname = "net$i";
4091
4092         next if !$conf->{$netname};
4093         my $d = parse_net($conf->{$netname});
4094         next if !$d;
4095         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4096
4097         $use_virtio = 1 if $d->{model} eq 'virtio';
4098
4099         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4100
4101         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4102         push @$devices, '-netdev', $netdevfull;
4103
4104         my $netdevicefull = print_netdevice_full(
4105             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4106
4107         push @$devices, '-device', $netdevicefull;
4108     }
4109
4110     if ($conf->{ivshmem}) {
4111         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4112
4113         my $bus;
4114         if ($q35) {
4115             $bus = print_pcie_addr("ivshmem");
4116         } else {
4117             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4118         }
4119
4120         my $ivshmem_name = $ivshmem->{name} // $vmid;
4121         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4122
4123         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4124         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4125             .",size=$ivshmem->{size}M";
4126     }
4127
4128     # pci.4 is nested in pci.1
4129     $bridges->{1} = 1 if $bridges->{4};
4130
4131     if (!$q35) { # add pci bridges
4132         if (min_version($machine_version, 2, 3)) {
4133            $bridges->{1} = 1;
4134            $bridges->{2} = 1;
4135         }
4136         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4137     }
4138
4139     for my $k (sort {$b cmp $a} keys %$bridges) {
4140         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4141
4142         my $k_name = $k;
4143         if ($k == 2 && $legacy_igd) {
4144             $k_name = "$k-igd";
4145         }
4146         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4147         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4148
4149         if ($q35) { # add after -readconfig pve-q35.cfg
4150             splice @$devices, 2, 0, '-device', $devstr;
4151         } else {
4152             unshift @$devices, '-device', $devstr if $k > 0;
4153         }
4154     }
4155
4156     if (!$kvm) {
4157         push @$machineFlags, 'accel=tcg';
4158     }
4159
4160     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4161
4162     my $machine_type_min = $machine_type;
4163     if ($add_pve_version) {
4164         $machine_type_min =~ s/\+pve\d+$//;
4165         $machine_type_min .= "+pve$required_pve_version";
4166     }
4167     push @$machineFlags, "type=${machine_type_min}";
4168
4169     push @$cmd, @$devices;
4170     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4171     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4172     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4173
4174     if (my $vmstate = $conf->{vmstate}) {
4175         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4176         push @$vollist, $vmstate;
4177         push @$cmd, '-loadstate', $statepath;
4178         print "activating and using '$vmstate' as vmstate\n";
4179     }
4180
4181     if (PVE::QemuConfig->is_template($conf)) {
4182         # needed to workaround base volumes being read-only
4183         push @$cmd, '-snapshot';
4184     }
4185
4186     # add custom args
4187     if ($conf->{args}) {
4188         my $aa = PVE::Tools::split_args($conf->{args});
4189         push @$cmd, @$aa;
4190     }
4191
4192     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4193 }
4194
4195 sub check_rng_source {
4196     my ($source) = @_;
4197
4198     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4199     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4200         if ! -e $source;
4201
4202     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4203     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4204         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4205         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4206         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4207             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4208             ." to the host.\n";
4209     }
4210 }
4211
4212 sub spice_port {
4213     my ($vmid) = @_;
4214
4215     my $res = mon_cmd($vmid, 'query-spice');
4216
4217     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4218 }
4219
4220 sub vm_devices_list {
4221     my ($vmid) = @_;
4222
4223     my $res = mon_cmd($vmid, 'query-pci');
4224     my $devices_to_check = [];
4225     my $devices = {};
4226     foreach my $pcibus (@$res) {
4227         push @$devices_to_check, @{$pcibus->{devices}},
4228     }
4229
4230     while (@$devices_to_check) {
4231         my $to_check = [];
4232         for my $d (@$devices_to_check) {
4233             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4234             next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4235
4236             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4237             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4238         }
4239         $devices_to_check = $to_check;
4240     }
4241
4242     my $resblock = mon_cmd($vmid, 'query-block');
4243     foreach my $block (@$resblock) {
4244         if($block->{device} =~ m/^drive-(\S+)/){
4245                 $devices->{$1} = 1;
4246         }
4247     }
4248
4249     my $resmice = mon_cmd($vmid, 'query-mice');
4250     foreach my $mice (@$resmice) {
4251         if ($mice->{name} eq 'QEMU HID Tablet') {
4252             $devices->{tablet} = 1;
4253             last;
4254         }
4255     }
4256
4257     # for usb devices there is no query-usb
4258     # but we can iterate over the entries in
4259     # qom-list path=/machine/peripheral
4260     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4261     foreach my $per (@$resperipheral) {
4262         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4263             $devices->{$per->{name}} = 1;
4264         }
4265     }
4266
4267     return $devices;
4268 }
4269
4270 sub vm_deviceplug {
4271     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4272
4273     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4274
4275     my $devices_list = vm_devices_list($vmid);
4276     return 1 if defined($devices_list->{$deviceid});
4277
4278     # add PCI bridge if we need it for the device
4279     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4280
4281     if ($deviceid eq 'tablet') {
4282         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4283     } elsif ($deviceid eq 'keyboard') {
4284         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4285     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4286         my $id = $1;
4287         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4288         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4289     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4290         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4291     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4292         qemu_iothread_add($vmid, $deviceid, $device);
4293
4294         qemu_driveadd($storecfg, $vmid, $device);
4295         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4296
4297         qemu_deviceadd($vmid, $devicefull);
4298         eval { qemu_deviceaddverify($vmid, $deviceid); };
4299         if (my $err = $@) {
4300             eval { qemu_drivedel($vmid, $deviceid); };
4301             warn $@ if $@;
4302             die $err;
4303         }
4304     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4305         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4306         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4307         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4308
4309         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4310
4311         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4312             qemu_iothread_add($vmid, $deviceid, $device);
4313             $devicefull .= ",iothread=iothread-$deviceid";
4314         }
4315
4316         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4317             $devicefull .= ",num_queues=$device->{queues}";
4318         }
4319
4320         qemu_deviceadd($vmid, $devicefull);
4321         qemu_deviceaddverify($vmid, $deviceid);
4322     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4323         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4324         qemu_driveadd($storecfg, $vmid, $device);
4325
4326         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4327         eval { qemu_deviceadd($vmid, $devicefull); };
4328         if (my $err = $@) {
4329             eval { qemu_drivedel($vmid, $deviceid); };
4330             warn $@ if $@;
4331             die $err;
4332         }
4333     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4334         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4335
4336         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4337         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4338         my $use_old_bios_files = undef;
4339         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4340
4341         my $netdevicefull = print_netdevice_full(
4342             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4343         qemu_deviceadd($vmid, $netdevicefull);
4344         eval {
4345             qemu_deviceaddverify($vmid, $deviceid);
4346             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4347         };
4348         if (my $err = $@) {
4349             eval { qemu_netdevdel($vmid, $deviceid); };
4350             warn $@ if $@;
4351             die $err;
4352         }
4353     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4354         my $bridgeid = $2;
4355         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4356         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4357
4358         qemu_deviceadd($vmid, $devicefull);
4359         qemu_deviceaddverify($vmid, $deviceid);
4360     } else {
4361         die "can't hotplug device '$deviceid'\n";
4362     }
4363
4364     return 1;
4365 }
4366
4367 # fixme: this should raise exceptions on error!
4368 sub vm_deviceunplug {
4369     my ($vmid, $conf, $deviceid) = @_;
4370
4371     my $devices_list = vm_devices_list($vmid);
4372     return 1 if !defined($devices_list->{$deviceid});
4373
4374     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4375     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4376
4377     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4378         qemu_devicedel($vmid, $deviceid);
4379     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4380         qemu_devicedel($vmid, $deviceid);
4381         qemu_devicedelverify($vmid, $deviceid);
4382     } elsif ($deviceid =~ m/^usb\d+$/) {
4383         qemu_devicedel($vmid, $deviceid);
4384         qemu_devicedelverify($vmid, $deviceid);
4385     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4386         my $device = parse_drive($deviceid, $conf->{$deviceid});
4387
4388         qemu_devicedel($vmid, $deviceid);
4389         qemu_devicedelverify($vmid, $deviceid);
4390         qemu_drivedel($vmid, $deviceid);
4391         qemu_iothread_del($vmid, $deviceid, $device);
4392     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4393         qemu_devicedel($vmid, $deviceid);
4394         qemu_devicedelverify($vmid, $deviceid);
4395     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4396         my $device = parse_drive($deviceid, $conf->{$deviceid});
4397
4398         qemu_devicedel($vmid, $deviceid);
4399         qemu_devicedelverify($vmid, $deviceid);
4400         qemu_drivedel($vmid, $deviceid);
4401         qemu_deletescsihw($conf, $vmid, $deviceid);
4402
4403         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4404             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4405     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4406         qemu_devicedel($vmid, $deviceid);
4407         qemu_devicedelverify($vmid, $deviceid);
4408         qemu_netdevdel($vmid, $deviceid);
4409     } else {
4410         die "can't unplug device '$deviceid'\n";
4411     }
4412
4413     return 1;
4414 }
4415
4416 sub qemu_spice_usbredir_chardev_add {
4417     my ($vmid, $id) = @_;
4418
4419     mon_cmd($vmid, "chardev-add" , (
4420         id => $id,
4421         backend => {
4422             type => 'spicevmc',
4423             data => {
4424                 type => "usbredir",
4425             },
4426         },
4427     ));
4428 }
4429
4430 sub qemu_deviceadd {
4431     my ($vmid, $devicefull) = @_;
4432
4433     $devicefull = "driver=".$devicefull;
4434     my %options =  split(/[=,]/, $devicefull);
4435
4436     mon_cmd($vmid, "device_add" , %options);
4437 }
4438
4439 sub qemu_devicedel {
4440     my ($vmid, $deviceid) = @_;
4441
4442     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4443 }
4444
4445 sub qemu_iothread_add {
4446     my ($vmid, $deviceid, $device) = @_;
4447
4448     if ($device->{iothread}) {
4449         my $iothreads = vm_iothreads_list($vmid);
4450         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4451     }
4452 }
4453
4454 sub qemu_iothread_del {
4455     my ($vmid, $deviceid, $device) = @_;
4456
4457     if ($device->{iothread}) {
4458         my $iothreads = vm_iothreads_list($vmid);
4459         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4460     }
4461 }
4462
4463 sub qemu_objectadd {
4464     my ($vmid, $objectid, $qomtype) = @_;
4465
4466     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4467
4468     return 1;
4469 }
4470
4471 sub qemu_objectdel {
4472     my ($vmid, $objectid) = @_;
4473
4474     mon_cmd($vmid, "object-del", id => $objectid);
4475
4476     return 1;
4477 }
4478
4479 sub qemu_driveadd {
4480     my ($storecfg, $vmid, $device) = @_;
4481
4482     my $kvmver = get_running_qemu_version($vmid);
4483     my $io_uring = min_version($kvmver, 6, 0);
4484     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4485     $drive =~ s/\\/\\\\/g;
4486     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4487
4488     # If the command succeeds qemu prints: "OK"
4489     return 1 if $ret =~ m/OK/s;
4490
4491     die "adding drive failed: $ret\n";
4492 }
4493
4494 sub qemu_drivedel {
4495     my ($vmid, $deviceid) = @_;
4496
4497     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4498     $ret =~ s/^\s+//;
4499
4500     return 1 if $ret eq "";
4501
4502     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4503     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4504
4505     die "deleting drive $deviceid failed : $ret\n";
4506 }
4507
4508 sub qemu_deviceaddverify {
4509     my ($vmid, $deviceid) = @_;
4510
4511     for (my $i = 0; $i <= 5; $i++) {
4512          my $devices_list = vm_devices_list($vmid);
4513          return 1 if defined($devices_list->{$deviceid});
4514          sleep 1;
4515     }
4516
4517     die "error on hotplug device '$deviceid'\n";
4518 }
4519
4520
4521 sub qemu_devicedelverify {
4522     my ($vmid, $deviceid) = @_;
4523
4524     # need to verify that the device is correctly removed as device_del
4525     # is async and empty return is not reliable
4526
4527     for (my $i = 0; $i <= 5; $i++) {
4528          my $devices_list = vm_devices_list($vmid);
4529          return 1 if !defined($devices_list->{$deviceid});
4530          sleep 1;
4531     }
4532
4533     die "error on hot-unplugging device '$deviceid'\n";
4534 }
4535
4536 sub qemu_findorcreatescsihw {
4537     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4538
4539     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4540
4541     my $scsihwid="$controller_prefix$controller";
4542     my $devices_list = vm_devices_list($vmid);
4543
4544     if (!defined($devices_list->{$scsihwid})) {
4545         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4546     }
4547
4548     return 1;
4549 }
4550
4551 sub qemu_deletescsihw {
4552     my ($conf, $vmid, $opt) = @_;
4553
4554     my $device = parse_drive($opt, $conf->{$opt});
4555
4556     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4557         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4558         return 1;
4559     }
4560
4561     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4562
4563     my $devices_list = vm_devices_list($vmid);
4564     foreach my $opt (keys %{$devices_list}) {
4565         if (is_valid_drivename($opt)) {
4566             my $drive = parse_drive($opt, $conf->{$opt});
4567             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4568                 return 1;
4569             }
4570         }
4571     }
4572
4573     my $scsihwid="scsihw$controller";
4574
4575     vm_deviceunplug($vmid, $conf, $scsihwid);
4576
4577     return 1;
4578 }
4579
4580 sub qemu_add_pci_bridge {
4581     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4582
4583     my $bridges = {};
4584
4585     my $bridgeid;
4586
4587     print_pci_addr($device, $bridges, $arch, $machine_type);
4588
4589     while (my ($k, $v) = each %$bridges) {
4590         $bridgeid = $k;
4591     }
4592     return 1 if !defined($bridgeid) || $bridgeid < 1;
4593
4594     my $bridge = "pci.$bridgeid";
4595     my $devices_list = vm_devices_list($vmid);
4596
4597     if (!defined($devices_list->{$bridge})) {
4598         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4599     }
4600
4601     return 1;
4602 }
4603
4604 sub qemu_set_link_status {
4605     my ($vmid, $device, $up) = @_;
4606
4607     mon_cmd($vmid, "set_link", name => $device,
4608                up => $up ? JSON::true : JSON::false);
4609 }
4610
4611 sub qemu_netdevadd {
4612     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4613
4614     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4615     my %options =  split(/[=,]/, $netdev);
4616
4617     if (defined(my $vhost = $options{vhost})) {
4618         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4619     }
4620
4621     if (defined(my $queues = $options{queues})) {
4622         $options{queues} = $queues + 0;
4623     }
4624
4625     mon_cmd($vmid, "netdev_add",  %options);
4626     return 1;
4627 }
4628
4629 sub qemu_netdevdel {
4630     my ($vmid, $deviceid) = @_;
4631
4632     mon_cmd($vmid, "netdev_del", id => $deviceid);
4633 }
4634
4635 sub qemu_usb_hotplug {
4636     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4637
4638     return if !$device;
4639
4640     # remove the old one first
4641     vm_deviceunplug($vmid, $conf, $deviceid);
4642
4643     # check if xhci controller is necessary and available
4644     my $devicelist = vm_devices_list($vmid);
4645
4646     if (!$devicelist->{xhci}) {
4647         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4648         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4649     }
4650
4651     # print_usbdevice_full expects the parsed device
4652     my $d = parse_usb_device($device->{host});
4653     $d->{usb3} = $device->{usb3};
4654
4655     # add the new one
4656     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4657 }
4658
4659 sub qemu_cpu_hotplug {
4660     my ($vmid, $conf, $vcpus) = @_;
4661
4662     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4663
4664     my $sockets = 1;
4665     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4666     $sockets = $conf->{sockets} if  $conf->{sockets};
4667     my $cores = $conf->{cores} || 1;
4668     my $maxcpus = $sockets * $cores;
4669
4670     $vcpus = $maxcpus if !$vcpus;
4671
4672     die "you can't add more vcpus than maxcpus\n"
4673         if $vcpus > $maxcpus;
4674
4675     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4676
4677     if ($vcpus < $currentvcpus) {
4678
4679         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4680
4681             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4682                 qemu_devicedel($vmid, "cpu$i");
4683                 my $retry = 0;
4684                 my $currentrunningvcpus = undef;
4685                 while (1) {
4686                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4687                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4688                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4689                     $retry++;
4690                     sleep 1;
4691                 }
4692                 #update conf after each succesfull cpu unplug
4693                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4694                 PVE::QemuConfig->write_config($vmid, $conf);
4695             }
4696         } else {
4697             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4698         }
4699
4700         return;
4701     }
4702
4703     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4704     die "vcpus in running vm does not match its configuration\n"
4705         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4706
4707     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4708
4709         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4710             my $cpustr = print_cpu_device($conf, $i);
4711             qemu_deviceadd($vmid, $cpustr);
4712
4713             my $retry = 0;
4714             my $currentrunningvcpus = undef;
4715             while (1) {
4716                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4717                 last if scalar(@{$currentrunningvcpus}) == $i;
4718                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4719                 sleep 1;
4720                 $retry++;
4721             }
4722             #update conf after each succesfull cpu hotplug
4723             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4724             PVE::QemuConfig->write_config($vmid, $conf);
4725         }
4726     } else {
4727
4728         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4729             mon_cmd($vmid, "cpu-add", id => int($i));
4730         }
4731     }
4732 }
4733
4734 sub qemu_block_set_io_throttle {
4735     my ($vmid, $deviceid,
4736         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4737         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4738         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4739         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4740
4741     return if !check_running($vmid) ;
4742
4743     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4744         bps => int($bps),
4745         bps_rd => int($bps_rd),
4746         bps_wr => int($bps_wr),
4747         iops => int($iops),
4748         iops_rd => int($iops_rd),
4749         iops_wr => int($iops_wr),
4750         bps_max => int($bps_max),
4751         bps_rd_max => int($bps_rd_max),
4752         bps_wr_max => int($bps_wr_max),
4753         iops_max => int($iops_max),
4754         iops_rd_max => int($iops_rd_max),
4755         iops_wr_max => int($iops_wr_max),
4756         bps_max_length => int($bps_max_length),
4757         bps_rd_max_length => int($bps_rd_max_length),
4758         bps_wr_max_length => int($bps_wr_max_length),
4759         iops_max_length => int($iops_max_length),
4760         iops_rd_max_length => int($iops_rd_max_length),
4761         iops_wr_max_length => int($iops_wr_max_length),
4762     );
4763
4764 }
4765
4766 sub qemu_block_resize {
4767     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4768
4769     my $running = check_running($vmid);
4770
4771     PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4772
4773     return if !$running;
4774
4775     my $padding = (1024 - $size % 1024) % 1024;
4776     $size = $size + $padding;
4777
4778     mon_cmd(
4779         $vmid,
4780         "block_resize",
4781         device => $deviceid,
4782         size => int($size),
4783         timeout => 60,
4784     );
4785 }
4786
4787 sub qemu_volume_snapshot {
4788     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4789
4790     my $running = check_running($vmid);
4791
4792     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4793         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4794     } else {
4795         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4796     }
4797 }
4798
4799 sub qemu_volume_snapshot_delete {
4800     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4801
4802     my $running = check_running($vmid);
4803
4804     if($running) {
4805
4806         $running = undef;
4807         my $conf = PVE::QemuConfig->load_config($vmid);
4808         PVE::QemuConfig->foreach_volume($conf, sub {
4809             my ($ds, $drive) = @_;
4810             $running = 1 if $drive->{file} eq $volid;
4811         });
4812     }
4813
4814     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4815         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4816     } else {
4817         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4818     }
4819 }
4820
4821 sub set_migration_caps {
4822     my ($vmid, $savevm) = @_;
4823
4824     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4825
4826     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4827     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4828
4829     my $cap_ref = [];
4830
4831     my $enabled_cap = {
4832         "auto-converge" => 1,
4833         "xbzrle" => 1,
4834         "x-rdma-pin-all" => 0,
4835         "zero-blocks" => 0,
4836         "compress" => 0,
4837         "dirty-bitmaps" => $dirty_bitmaps,
4838     };
4839
4840     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4841
4842     for my $supported_capability (@$supported_capabilities) {
4843         push @$cap_ref, {
4844             capability => $supported_capability->{capability},
4845             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4846         };
4847     }
4848
4849     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4850 }
4851
4852 sub foreach_volid {
4853     my ($conf, $func, @param) = @_;
4854
4855     my $volhash = {};
4856
4857     my $test_volid = sub {
4858         my ($key, $drive, $snapname) = @_;
4859
4860         my $volid = $drive->{file};
4861         return if !$volid;
4862
4863         $volhash->{$volid}->{cdrom} //= 1;
4864         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4865
4866         my $replicate = $drive->{replicate} // 1;
4867         $volhash->{$volid}->{replicate} //= 0;
4868         $volhash->{$volid}->{replicate} = 1 if $replicate;
4869
4870         $volhash->{$volid}->{shared} //= 0;
4871         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4872
4873         $volhash->{$volid}->{referenced_in_config} //= 0;
4874         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4875
4876         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4877             if defined($snapname);
4878
4879         my $size = $drive->{size};
4880         $volhash->{$volid}->{size} //= $size if $size;
4881
4882         $volhash->{$volid}->{is_vmstate} //= 0;
4883         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4884
4885         $volhash->{$volid}->{is_tpmstate} //= 0;
4886         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4887
4888         $volhash->{$volid}->{is_unused} //= 0;
4889         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4890
4891         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4892     };
4893
4894     my $include_opts = {
4895         extra_keys => ['vmstate'],
4896         include_unused => 1,
4897     };
4898
4899     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4900     foreach my $snapname (keys %{$conf->{snapshots}}) {
4901         my $snap = $conf->{snapshots}->{$snapname};
4902         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4903     }
4904
4905     foreach my $volid (keys %$volhash) {
4906         &$func($volid, $volhash->{$volid}, @param);
4907     }
4908 }
4909
4910 my $fast_plug_option = {
4911     'lock' => 1,
4912     'name' => 1,
4913     'onboot' => 1,
4914     'shares' => 1,
4915     'startup' => 1,
4916     'description' => 1,
4917     'protection' => 1,
4918     'vmstatestorage' => 1,
4919     'hookscript' => 1,
4920     'tags' => 1,
4921 };
4922
4923 for my $opt (keys %$confdesc_cloudinit) {
4924     $fast_plug_option->{$opt} = 1;
4925 };
4926
4927 # hotplug changes in [PENDING]
4928 # $selection hash can be used to only apply specified options, for
4929 # example: { cores => 1 } (only apply changed 'cores')
4930 # $errors ref is used to return error messages
4931 sub vmconfig_hotplug_pending {
4932     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4933
4934     my $defaults = load_defaults();
4935     my $arch = get_vm_arch($conf);
4936     my $machine_type = get_vm_machine($conf, undef, $arch);
4937
4938     # commit values which do not have any impact on running VM first
4939     # Note: those option cannot raise errors, we we do not care about
4940     # $selection and always apply them.
4941
4942     my $add_error = sub {
4943         my ($opt, $msg) = @_;
4944         $errors->{$opt} = "hotplug problem - $msg";
4945     };
4946
4947     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4948
4949     my $cloudinit_record_changed = sub {
4950         my ($conf, $opt, $old, $new) = @_;
4951         return if !$cloudinit_pending_properties->{$opt};
4952
4953         my $ci = ($conf->{cloudinit} //= {});
4954
4955         my $recorded = $ci->{$opt};
4956         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4957
4958         if (defined($new)) {
4959             if (defined($old)) {
4960                 # an existing value is being modified
4961                 if (defined($recorded)) {
4962                     # the value was already not in sync
4963                     if ($new eq $recorded) {
4964                         # a value is being reverted to the cloud-init state:
4965                         delete $ci->{$opt};
4966                         delete $added{$opt};
4967                     } else {
4968                         # the value was changed multiple times, do nothing
4969                     }
4970                 } elsif ($added{$opt}) {
4971                     # the value had been marked as added and is being changed, do nothing
4972                 } else {
4973                     # the value is new, record it:
4974                     $ci->{$opt} = $old;
4975                 }
4976             } else {
4977                 # a new value is being added
4978                 if (defined($recorded)) {
4979                     # it was already not in sync
4980                     if ($new eq $recorded) {
4981                         # a value is being reverted to the cloud-init state:
4982                         delete $ci->{$opt};
4983                         delete $added{$opt};
4984                     } else {
4985                         # the value had temporarily been removed, do nothing
4986                     }
4987                 } elsif ($added{$opt}) {
4988                     # the value had been marked as added already, do nothing
4989                 } else {
4990                     # the value is new, add it
4991                     $added{$opt} = 1;
4992                 }
4993             }
4994         } elsif (!defined($old)) {
4995             # a non-existent value is being removed? ignore...
4996         } else {
4997             # a value is being deleted
4998             if (defined($recorded)) {
4999                 # a value was already recorded, just keep it
5000             } elsif ($added{$opt}) {
5001                 # the value was marked as added, remove it
5002                 delete $added{$opt};
5003             } else {
5004                 # a previously unrecorded value is being removed, record the old value:
5005                 $ci->{$opt} = $old;
5006             }
5007         }
5008
5009         my $added = join(',', sort keys %added);
5010         $ci->{added} = $added if length($added);
5011     };
5012
5013     my $changes = 0;
5014     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5015         if ($fast_plug_option->{$opt}) {
5016             my $new = delete $conf->{pending}->{$opt};
5017             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
5018             $conf->{$opt} = $new;
5019             $changes = 1;
5020         }
5021     }
5022
5023     if ($changes) {
5024         PVE::QemuConfig->write_config($vmid, $conf);
5025     }
5026
5027     my $ostype = $conf->{ostype};
5028     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
5029     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5030     my $usb_hotplug = $hotplug_features->{usb}
5031         && min_version($version, 7, 1)
5032         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
5033
5034     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
5035     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5036
5037     foreach my $opt (sort keys %$pending_delete_hash) {
5038         next if $selection && !$selection->{$opt};
5039         my $force = $pending_delete_hash->{$opt}->{force};
5040         eval {
5041             if ($opt eq 'hotplug') {
5042                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5043             } elsif ($opt eq 'tablet') {
5044                 die "skip\n" if !$hotplug_features->{usb};
5045                 if ($defaults->{tablet}) {
5046                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5047                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5048                         if $arch eq 'aarch64';
5049                 } else {
5050                     vm_deviceunplug($vmid, $conf, 'tablet');
5051                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5052                 }
5053             } elsif ($opt =~ m/^usb(\d+)$/) {
5054                 my $index = $1;
5055                 die "skip\n" if !$usb_hotplug;
5056                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5057                 vm_deviceunplug($vmid, $conf, $opt);
5058             } elsif ($opt eq 'vcpus') {
5059                 die "skip\n" if !$hotplug_features->{cpu};
5060                 qemu_cpu_hotplug($vmid, $conf, undef);
5061             } elsif ($opt eq 'balloon') {
5062                 # enable balloon device is not hotpluggable
5063                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5064                 # here we reset the ballooning value to memory
5065                 my $balloon = $conf->{memory} || $defaults->{memory};
5066                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5067             } elsif ($fast_plug_option->{$opt}) {
5068                 # do nothing
5069             } elsif ($opt =~ m/^net(\d+)$/) {
5070                 die "skip\n" if !$hotplug_features->{network};
5071                 vm_deviceunplug($vmid, $conf, $opt);
5072             } elsif (is_valid_drivename($opt)) {
5073                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5074                 vm_deviceunplug($vmid, $conf, $opt);
5075                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5076             } elsif ($opt =~ m/^memory$/) {
5077                 die "skip\n" if !$hotplug_features->{memory};
5078                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5079             } elsif ($opt eq 'cpuunits') {
5080                 $cgroup->change_cpu_shares(undef);
5081             } elsif ($opt eq 'cpulimit') {
5082                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5083             } else {
5084                 die "skip\n";
5085             }
5086         };
5087         if (my $err = $@) {
5088             &$add_error($opt, $err) if $err ne "skip\n";
5089         } else {
5090             my $old = delete $conf->{$opt};
5091             $cloudinit_record_changed->($conf, $opt, $old, undef);
5092             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5093         }
5094     }
5095
5096     my $cloudinit_opt;
5097     foreach my $opt (keys %{$conf->{pending}}) {
5098         next if $selection && !$selection->{$opt};
5099         my $value = $conf->{pending}->{$opt};
5100         eval {
5101             if ($opt eq 'hotplug') {
5102                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5103             } elsif ($opt eq 'tablet') {
5104                 die "skip\n" if !$hotplug_features->{usb};
5105                 if ($value == 1) {
5106                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5107                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5108                         if $arch eq 'aarch64';
5109                 } elsif ($value == 0) {
5110                     vm_deviceunplug($vmid, $conf, 'tablet');
5111                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5112                 }
5113             } elsif ($opt =~ m/^usb(\d+)$/) {
5114                 my $index = $1;
5115                 die "skip\n" if !$usb_hotplug;
5116                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5117                 my $id = $opt;
5118                 if ($d->{host} eq 'spice')  {
5119                     $id = "usbredirdev$index";
5120                 }
5121                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5122             } elsif ($opt eq 'vcpus') {
5123                 die "skip\n" if !$hotplug_features->{cpu};
5124                 qemu_cpu_hotplug($vmid, $conf, $value);
5125             } elsif ($opt eq 'balloon') {
5126                 # enable/disable balloning device is not hotpluggable
5127                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5128                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5129                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5130
5131                 # allow manual ballooning if shares is set to zero
5132                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5133                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5134                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5135                 }
5136             } elsif ($opt =~ m/^net(\d+)$/) {
5137                 # some changes can be done without hotplug
5138                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5139                                     $vmid, $opt, $value, $arch, $machine_type);
5140             } elsif (is_valid_drivename($opt)) {
5141                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5142                 # some changes can be done without hotplug
5143                 my $drive = parse_drive($opt, $value);
5144                 if (drive_is_cloudinit($drive)) {
5145                     $cloudinit_opt = [$opt, $drive];
5146                     # apply all the other changes first, then generate the cloudinit disk
5147                     die "skip\n";
5148                 }
5149                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5150                                      $vmid, $opt, $value, $arch, $machine_type);
5151             } elsif ($opt =~ m/^memory$/) { #dimms
5152                 die "skip\n" if !$hotplug_features->{memory};
5153                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5154             } elsif ($opt eq 'cpuunits') {
5155                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5156                 $cgroup->change_cpu_shares($new_cpuunits);
5157             } elsif ($opt eq 'cpulimit') {
5158                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5159                 $cgroup->change_cpu_quota($cpulimit, 100000);
5160             } elsif ($opt eq 'agent') {
5161                 vmconfig_update_agent($conf, $opt, $value);
5162             } else {
5163                 die "skip\n";  # skip non-hot-pluggable options
5164             }
5165         };
5166         if (my $err = $@) {
5167             &$add_error($opt, $err) if $err ne "skip\n";
5168         } else {
5169             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5170             $conf->{$opt} = $value;
5171             delete $conf->{pending}->{$opt};
5172         }
5173     }
5174
5175     if (defined($cloudinit_opt)) {
5176         my ($opt, $drive) = @$cloudinit_opt;
5177         my $value = $conf->{pending}->{$opt};
5178         eval {
5179             my $temp = {%$conf, $opt => $value};
5180             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5181             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5182                                  $vmid, $opt, $value, $arch, $machine_type);
5183         };
5184         if (my $err = $@) {
5185             &$add_error($opt, $err) if $err ne "skip\n";
5186         } else {
5187             $conf->{$opt} = $value;
5188             delete $conf->{pending}->{$opt};
5189         }
5190     }
5191
5192     # unplug xhci controller if no usb device is left
5193     if ($usb_hotplug) {
5194         my $has_usb = 0;
5195         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5196             next if !defined($conf->{"usb$i"});
5197             $has_usb = 1;
5198             last;
5199         }
5200         if (!$has_usb) {
5201             vm_deviceunplug($vmid, $conf, 'xhci');
5202         }
5203     }
5204
5205     PVE::QemuConfig->write_config($vmid, $conf);
5206
5207     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5208         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5209     }
5210 }
5211
5212 sub try_deallocate_drive {
5213     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5214
5215     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5216         my $volid = $drive->{file};
5217         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5218             my $sid = PVE::Storage::parse_volume_id($volid);
5219             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5220
5221             # check if the disk is really unused
5222             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5223                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5224             PVE::Storage::vdisk_free($storecfg, $volid);
5225             return 1;
5226         } else {
5227             # If vm is not owner of this disk remove from config
5228             return 1;
5229         }
5230     }
5231
5232     return;
5233 }
5234
5235 sub vmconfig_delete_or_detach_drive {
5236     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5237
5238     my $drive = parse_drive($opt, $conf->{$opt});
5239
5240     my $rpcenv = PVE::RPCEnvironment::get();
5241     my $authuser = $rpcenv->get_user();
5242
5243     if ($force) {
5244         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5245         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5246     } else {
5247         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5248     }
5249 }
5250
5251
5252
5253 sub vmconfig_apply_pending {
5254     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5255
5256     return if !scalar(keys %{$conf->{pending}});
5257
5258     my $add_apply_error = sub {
5259         my ($opt, $msg) = @_;
5260         my $err_msg = "unable to apply pending change $opt : $msg";
5261         $errors->{$opt} = $err_msg;
5262         warn $err_msg;
5263     };
5264
5265     # cold plug
5266
5267     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5268     foreach my $opt (sort keys %$pending_delete_hash) {
5269         my $force = $pending_delete_hash->{$opt}->{force};
5270         eval {
5271             if ($opt =~ m/^unused/) {
5272                 die "internal error";
5273             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5274                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5275             }
5276         };
5277         if (my $err = $@) {
5278             $add_apply_error->($opt, $err);
5279         } else {
5280             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5281             delete $conf->{$opt};
5282         }
5283     }
5284
5285     PVE::QemuConfig->cleanup_pending($conf);
5286
5287     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5288
5289     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5290         next if $opt eq 'delete'; # just to be sure
5291         eval {
5292             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5293                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5294             }
5295         };
5296         if (my $err = $@) {
5297             $add_apply_error->($opt, $err);
5298         } else {
5299
5300             if (is_valid_drivename($opt)) {
5301                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5302                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5303             }
5304
5305             $conf->{$opt} = delete $conf->{pending}->{$opt};
5306         }
5307     }
5308
5309     # write all changes at once to avoid unnecessary i/o
5310     PVE::QemuConfig->write_config($vmid, $conf);
5311     if ($generate_cloudinit) {
5312         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5313             # After successful generation and if there were changes to be applied, update the
5314             # config to drop the {cloudinit} entry.
5315             PVE::QemuConfig->write_config($vmid, $conf);
5316         }
5317     }
5318 }
5319
5320 sub vmconfig_update_net {
5321     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5322
5323     my $newnet = parse_net($value);
5324
5325     if ($conf->{$opt}) {
5326         my $oldnet = parse_net($conf->{$opt});
5327
5328         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5329             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5330             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5331             safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5332             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5333
5334             # for non online change, we try to hot-unplug
5335             die "skip\n" if !$hotplug;
5336             vm_deviceunplug($vmid, $conf, $opt);
5337         } else {
5338
5339             die "internal error" if $opt !~ m/net(\d+)/;
5340             my $iface = "tap${vmid}i$1";
5341
5342             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5343                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5344                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5345                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5346                 PVE::Network::tap_unplug($iface);
5347
5348                 if ($have_sdn) {
5349                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5350                 } else {
5351                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5352                 }
5353             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5354                 # Rate can be applied on its own but any change above needs to
5355                 # include the rate in tap_plug since OVS resets everything.
5356                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5357             }
5358
5359             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5360                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5361             }
5362
5363             return 1;
5364         }
5365     }
5366
5367     if ($hotplug) {
5368         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5369     } else {
5370         die "skip\n";
5371     }
5372 }
5373
5374 sub vmconfig_update_agent {
5375     my ($conf, $opt, $value) = @_;
5376
5377     die "skip\n" if !$conf->{$opt};
5378
5379     my $hotplug_options = { fstrim_cloned_disks => 1 };
5380
5381     my $old_agent = parse_guest_agent($conf);
5382     my $agent = parse_guest_agent({$opt => $value});
5383
5384     for my $option (keys %$agent) { # added/changed options
5385         next if defined($hotplug_options->{$option});
5386         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5387     }
5388
5389     for my $option (keys %$old_agent) { # removed options
5390         next if defined($hotplug_options->{$option});
5391         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5392     }
5393
5394     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5395 }
5396
5397 sub vmconfig_update_disk {
5398     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5399
5400     my $drive = parse_drive($opt, $value);
5401
5402     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5403         my $media = $drive->{media} || 'disk';
5404         my $oldmedia = $old_drive->{media} || 'disk';
5405         die "unable to change media type\n" if $media ne $oldmedia;
5406
5407         if (!drive_is_cdrom($old_drive)) {
5408
5409             if ($drive->{file} ne $old_drive->{file}) {
5410
5411                 die "skip\n" if !$hotplug;
5412
5413                 # unplug and register as unused
5414                 vm_deviceunplug($vmid, $conf, $opt);
5415                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5416
5417             } else {
5418                 # update existing disk
5419
5420                 # skip non hotpluggable value
5421                 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5422                     safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5423                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5424                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5425                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5426                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5427                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5428                     die "skip\n";
5429                 }
5430
5431                 # apply throttle
5432                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5433                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5434                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5435                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5436                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5437                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5438                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5439                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5440                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5441                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5442                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5443                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5444                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5445                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5446                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5447                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5448                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5449                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5450
5451                     qemu_block_set_io_throttle(
5452                         $vmid,"drive-$opt",
5453                         ($drive->{mbps} || 0)*1024*1024,
5454                         ($drive->{mbps_rd} || 0)*1024*1024,
5455                         ($drive->{mbps_wr} || 0)*1024*1024,
5456                          $drive->{iops} || 0,
5457                          $drive->{iops_rd} || 0,
5458                          $drive->{iops_wr} || 0,
5459                         ($drive->{mbps_max} || 0)*1024*1024,
5460                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5461                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5462                          $drive->{iops_max} || 0,
5463                          $drive->{iops_rd_max} || 0,
5464                          $drive->{iops_wr_max} || 0,
5465                          $drive->{bps_max_length} || 1,
5466                          $drive->{bps_rd_max_length} || 1,
5467                          $drive->{bps_wr_max_length} || 1,
5468                          $drive->{iops_max_length} || 1,
5469                          $drive->{iops_rd_max_length} || 1,
5470                          $drive->{iops_wr_max_length} || 1,
5471                     );
5472
5473                 }
5474
5475                 return 1;
5476             }
5477
5478         } else { # cdrom
5479
5480             if ($drive->{file} eq 'none') {
5481                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5482                 if (drive_is_cloudinit($old_drive)) {
5483                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5484                 }
5485             } else {
5486                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5487
5488                 # force eject if locked
5489                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5490
5491                 if ($path) {
5492                     mon_cmd($vmid, "blockdev-change-medium",
5493                         id => "$opt", filename => "$path");
5494                 }
5495             }
5496
5497             return 1;
5498         }
5499     }
5500
5501     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5502     # hotplug new disks
5503     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5504     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5505 }
5506
5507 sub vmconfig_update_cloudinit_drive {
5508     my ($storecfg, $conf, $vmid) = @_;
5509
5510     my $cloudinit_ds = undef;
5511     my $cloudinit_drive = undef;
5512
5513     PVE::QemuConfig->foreach_volume($conf, sub {
5514         my ($ds, $drive) = @_;
5515         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5516             $cloudinit_ds = $ds;
5517             $cloudinit_drive = $drive;
5518         }
5519     });
5520
5521     return if !$cloudinit_drive;
5522
5523     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5524         PVE::QemuConfig->write_config($vmid, $conf);
5525     }
5526
5527     my $running = PVE::QemuServer::check_running($vmid);
5528
5529     if ($running) {
5530         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5531         if ($path) {
5532             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5533             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5534         }
5535     }
5536 }
5537
5538 # called in locked context by incoming migration
5539 sub vm_migrate_get_nbd_disks {
5540     my ($storecfg, $conf, $replicated_volumes) = @_;
5541
5542     my $local_volumes = {};
5543     PVE::QemuConfig->foreach_volume($conf, sub {
5544         my ($ds, $drive) = @_;
5545
5546         return if drive_is_cdrom($drive);
5547         return if $ds eq 'tpmstate0';
5548
5549         my $volid = $drive->{file};
5550
5551         return if !$volid;
5552
5553         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5554
5555         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5556         return if $scfg->{shared};
5557
5558         # replicated disks re-use existing state via bitmap
5559         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5560         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5561     });
5562     return $local_volumes;
5563 }
5564
5565 # called in locked context by incoming migration
5566 sub vm_migrate_alloc_nbd_disks {
5567     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5568
5569     my $nbd = {};
5570     foreach my $opt (sort keys %$source_volumes) {
5571         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5572
5573         if ($use_existing) {
5574             $nbd->{$opt}->{drivestr} = print_drive($drive);
5575             $nbd->{$opt}->{volid} = $volid;
5576             $nbd->{$opt}->{replicated} = 1;
5577             next;
5578         }
5579
5580         # storage mapping + volname = regular migration
5581         # storage mapping + format = remote migration
5582         # order of precedence, filtered by whether storage supports it:
5583         # 1. explicit requested format
5584         # 2. format of current volume
5585         # 3. default format of storage
5586         if (!$storagemap->{identity}) {
5587             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5588             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5589             if (!$format || !grep { $format eq $_ } @$validFormats) {
5590                 if ($volname) {
5591                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5592                     my $fileFormat = qemu_img_format($scfg, $volname);
5593                     $format = $fileFormat
5594                         if grep { $fileFormat eq $_ } @$validFormats;
5595                 }
5596                 $format //= $defFormat;
5597             }
5598         } else {
5599             # can't happen for remote migration, so $volname is always defined
5600             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5601             $format = qemu_img_format($scfg, $volname);
5602         }
5603
5604         my $size = $drive->{size} / 1024;
5605         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5606         my $newdrive = $drive;
5607         $newdrive->{format} = $format;
5608         $newdrive->{file} = $newvolid;
5609         my $drivestr = print_drive($newdrive);
5610         $nbd->{$opt}->{drivestr} = $drivestr;
5611         $nbd->{$opt}->{volid} = $newvolid;
5612     }
5613
5614     return $nbd;
5615 }
5616
5617 # see vm_start_nolock for parameters, additionally:
5618 # migrate_opts:
5619 #   storagemap = parsed storage map for allocating NBD disks
5620 sub vm_start {
5621     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5622
5623     return PVE::QemuConfig->lock_config($vmid, sub {
5624         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5625
5626         die "you can't start a vm if it's a template\n"
5627             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5628
5629         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5630         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5631
5632         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5633
5634         if ($has_backup_lock && $running) {
5635             # a backup is currently running, attempt to start the guest in the
5636             # existing QEMU instance
5637             return vm_resume($vmid);
5638         }
5639
5640         PVE::QemuConfig->check_lock($conf)
5641             if !($params->{skiplock} || $has_suspended_lock);
5642
5643         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5644
5645         die "VM $vmid already running\n" if $running;
5646
5647         if (my $storagemap = $migrate_opts->{storagemap}) {
5648             my $replicated = $migrate_opts->{replicated_volumes};
5649             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5650             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5651
5652             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5653                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5654             }
5655         }
5656
5657         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5658     });
5659 }
5660
5661
5662 # params:
5663 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5664 #   skiplock => 0/1, skip checking for config lock
5665 #   skiptemplate => 0/1, skip checking whether VM is template
5666 #   forcemachine => to force QEMU machine (rollback/migration)
5667 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5668 #   timeout => in seconds
5669 #   paused => start VM in paused state (backup)
5670 #   resume => resume from hibernation
5671 #   pbs-backing => {
5672 #      sata0 => {
5673 #         repository
5674 #         snapshot
5675 #         keyfile
5676 #         archive
5677 #      },
5678 #      virtio2 => ...
5679 #   }
5680 # migrate_opts:
5681 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5682 #   migratedfrom => source node
5683 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5684 #   network => CIDR of migration network
5685 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5686 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5687 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5688 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5689 #       contained in config
5690 sub vm_start_nolock {
5691     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5692
5693     my $statefile = $params->{statefile};
5694     my $resume = $params->{resume};
5695
5696     my $migratedfrom = $migrate_opts->{migratedfrom};
5697     my $migration_type = $migrate_opts->{type};
5698
5699     my $res = {};
5700
5701     # clean up leftover reboot request files
5702     eval { clear_reboot_request($vmid); };
5703     warn $@ if $@;
5704
5705     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5706         vmconfig_apply_pending($vmid, $conf, $storecfg);
5707         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5708     }
5709
5710     # don't regenerate the ISO if the VM is started as part of a live migration
5711     # this way we can reuse the old ISO with the correct config
5712     if (!$migratedfrom) {
5713         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5714             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5715             # $conf->{cloudinit}, so we could just not do this?
5716             # But we do it above, so for now let's be consistent.
5717             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5718         }
5719     }
5720
5721     # override offline migrated volumes, conf is out of date still
5722     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5723         for my $key (sort keys $offline_volumes->%*) {
5724             my $parsed = parse_drive($key, $conf->{$key});
5725             $parsed->{file} = $offline_volumes->{$key};
5726             $conf->{$key} = print_drive($parsed);
5727         }
5728     }
5729
5730     my $defaults = load_defaults();
5731
5732     # set environment variable useful inside network script
5733     # for remote migration the config is available on the target node!
5734     if (!$migrate_opts->{remote_node}) {
5735         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5736     }
5737
5738     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5739
5740     my $forcemachine = $params->{forcemachine};
5741     my $forcecpu = $params->{forcecpu};
5742     if ($resume) {
5743         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5744         $forcemachine = $conf->{runningmachine};
5745         $forcecpu = $conf->{runningcpu};
5746         print "Resuming suspended VM\n";
5747     }
5748
5749     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5750         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5751
5752     my $migration_ip;
5753     my $get_migration_ip = sub {
5754         my ($nodename) = @_;
5755
5756         return $migration_ip if defined($migration_ip);
5757
5758         my $cidr = $migrate_opts->{network};
5759
5760         if (!defined($cidr)) {
5761             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5762             $cidr = $dc_conf->{migration}->{network};
5763         }
5764
5765         if (defined($cidr)) {
5766             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5767
5768             die "could not get IP: no address configured on local " .
5769                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5770
5771             die "could not get IP: multiple addresses configured on local " .
5772                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5773
5774             $migration_ip = @$ips[0];
5775         }
5776
5777         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5778             if !defined($migration_ip);
5779
5780         return $migration_ip;
5781     };
5782
5783     if ($statefile) {
5784         if ($statefile eq 'tcp') {
5785             my $migrate = $res->{migrate} = { proto => 'tcp' };
5786             $migrate->{addr} = "localhost";
5787             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5788             my $nodename = nodename();
5789
5790             if (!defined($migration_type)) {
5791                 if (defined($datacenterconf->{migration}->{type})) {
5792                     $migration_type = $datacenterconf->{migration}->{type};
5793                 } else {
5794                     $migration_type = 'secure';
5795                 }
5796             }
5797
5798             if ($migration_type eq 'insecure') {
5799                 $migrate->{addr} = $get_migration_ip->($nodename);
5800                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5801             }
5802
5803             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5804             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5805             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5806             push @$cmd, '-incoming', $migrate->{uri};
5807             push @$cmd, '-S';
5808
5809         } elsif ($statefile eq 'unix') {
5810             # should be default for secure migrations as a ssh TCP forward
5811             # tunnel is not deterministic reliable ready and fails regurarly
5812             # to set up in time, so use UNIX socket forwards
5813             my $migrate = $res->{migrate} = { proto => 'unix' };
5814             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5815             unlink $migrate->{addr};
5816
5817             $migrate->{uri} = "unix:$migrate->{addr}";
5818             push @$cmd, '-incoming', $migrate->{uri};
5819             push @$cmd, '-S';
5820
5821         } elsif (-e $statefile) {
5822             push @$cmd, '-loadstate', $statefile;
5823         } else {
5824             my $statepath = PVE::Storage::path($storecfg, $statefile);
5825             push @$vollist, $statefile;
5826             push @$cmd, '-loadstate', $statepath;
5827         }
5828     } elsif ($params->{paused}) {
5829         push @$cmd, '-S';
5830     }
5831
5832     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5833
5834     my $pci_devices = {}; # host pci devices
5835     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5836         my $dev = $conf->{"hostpci$i"} or next;
5837         $pci_devices->{$i} = parse_hostpci($dev);
5838     }
5839
5840     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5841     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5842
5843     # map to a flat list of pci ids
5844     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5845
5846     # reserve all PCI IDs before actually doing anything with them
5847     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5848
5849     eval {
5850         my $uuid;
5851         for my $id (sort keys %$pci_devices) {
5852             my $d = $pci_devices->{$id};
5853             for my $dev ($d->{pciid}->@*) {
5854                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5855
5856                 # nvidia grid needs the qemu parameter '-uuid' set
5857                 # use smbios uuid or mdev uuid as fallback for that
5858                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5859                     if (defined($conf->{smbios1})) {
5860                         my $smbios_conf = parse_smbios1($conf->{smbios1});
5861                         $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5862                     }
5863                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id) if !defined($uuid);
5864                 }
5865             }
5866         }
5867         push @$cmd, '-uuid', $uuid if defined($uuid);
5868     };
5869     if (my $err = $@) {
5870         eval { cleanup_pci_devices($vmid, $conf) };
5871         warn $@ if $@;
5872         die $err;
5873     }
5874
5875     PVE::Storage::activate_volumes($storecfg, $vollist);
5876
5877     eval {
5878         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5879     };
5880     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5881     # timeout should be more than enough here...
5882     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5883
5884     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5885
5886     my %run_params = (
5887         timeout => $statefile ? undef : $start_timeout,
5888         umask => 0077,
5889         noerr => 1,
5890     );
5891
5892     # when migrating, prefix QEMU output so other side can pick up any
5893     # errors that might occur and show the user
5894     if ($migratedfrom) {
5895         $run_params{quiet} = 1;
5896         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5897     }
5898
5899     my %systemd_properties = (
5900         Slice => 'qemu.slice',
5901         KillMode => 'process',
5902         SendSIGKILL => 0,
5903         TimeoutStopUSec => ULONG_MAX, # infinity
5904     );
5905
5906     if (PVE::CGroup::cgroup_mode() == 2) {
5907         $systemd_properties{CPUWeight} = $cpuunits;
5908     } else {
5909         $systemd_properties{CPUShares} = $cpuunits;
5910     }
5911
5912     if (my $cpulimit = $conf->{cpulimit}) {
5913         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5914     }
5915     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5916
5917     my $run_qemu = sub {
5918         PVE::Tools::run_fork sub {
5919             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5920
5921             my $tpmpid;
5922             if (my $tpm = $conf->{tpmstate0}) {
5923                 # start the TPM emulator so QEMU can connect on start
5924                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5925             }
5926
5927             my $exitcode = run_command($cmd, %run_params);
5928             if ($exitcode) {
5929                 if ($tpmpid) {
5930                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5931                     kill 'TERM', $tpmpid;
5932                 }
5933                 die "QEMU exited with code $exitcode\n";
5934             }
5935         };
5936     };
5937
5938     if ($conf->{hugepages}) {
5939
5940         my $code = sub {
5941             my $hotplug_features =
5942                 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5943             my $hugepages_topology =
5944                 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5945
5946             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5947
5948             PVE::QemuServer::Memory::hugepages_mount();
5949             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5950
5951             eval { $run_qemu->() };
5952             if (my $err = $@) {
5953                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5954                     if !$conf->{keephugepages};
5955                 die $err;
5956             }
5957
5958             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5959                 if !$conf->{keephugepages};
5960         };
5961         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5962
5963     } else {
5964         eval { $run_qemu->() };
5965     }
5966
5967     if (my $err = $@) {
5968         # deactivate volumes if start fails
5969         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5970         warn $@ if $@;
5971         eval { cleanup_pci_devices($vmid, $conf) };
5972         warn $@ if $@;
5973
5974         die "start failed: $err";
5975     }
5976
5977     # re-reserve all PCI IDs now that we can know the actual VM PID
5978     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5979     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5980     warn $@ if $@;
5981
5982     if (defined($res->{migrate})) {
5983         print "migration listens on $res->{migrate}->{uri}\n";
5984     } elsif ($statefile) {
5985         eval { mon_cmd($vmid, "cont"); };
5986         warn $@ if $@;
5987     }
5988
5989     #start nbd server for storage migration
5990     if (my $nbd = $migrate_opts->{nbd}) {
5991         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5992
5993         my $migrate_storage_uri;
5994         # nbd_protocol_version > 0 for unix socket support
5995         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5996             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5997             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5998             $migrate_storage_uri = "nbd:unix:$socket_path";
5999             $res->{migrate}->{unix_sockets} = [$socket_path];
6000         } else {
6001             my $nodename = nodename();
6002             my $localip = $get_migration_ip->($nodename);
6003             my $pfamily = PVE::Tools::get_host_address_family($nodename);
6004             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
6005
6006             mon_cmd($vmid, "nbd-server-start", addr => {
6007                 type => 'inet',
6008                 data => {
6009                     host => "${localip}",
6010                     port => "${storage_migrate_port}",
6011                 },
6012             });
6013             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
6014             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
6015         }
6016
6017         my $block_info = mon_cmd($vmid, "query-block");
6018         $block_info = { map { $_->{device} => $_ } $block_info->@* };
6019
6020         foreach my $opt (sort keys %$nbd) {
6021             my $drivestr = $nbd->{$opt}->{drivestr};
6022             my $volid = $nbd->{$opt}->{volid};
6023
6024             my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
6025
6026             mon_cmd(
6027                 $vmid,
6028                 "block-export-add",
6029                 id => "drive-$opt",
6030                 'node-name' => $block_node,
6031                 writable => JSON::true,
6032                 type => "nbd",
6033                 name => "drive-$opt", # NBD export name
6034             );
6035
6036             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6037             print "storage migration listens on $nbd_uri volume:$drivestr\n";
6038             print "re-using replicated volume: $opt - $volid\n"
6039                 if $nbd->{$opt}->{replicated};
6040
6041             $res->{drives}->{$opt} = $nbd->{$opt};
6042             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6043         }
6044     }
6045
6046     if ($migratedfrom) {
6047         eval {
6048             set_migration_caps($vmid);
6049         };
6050         warn $@ if $@;
6051
6052         if ($spice_port) {
6053             print "spice listens on port $spice_port\n";
6054             $res->{spice_port} = $spice_port;
6055             if ($migrate_opts->{spice_ticket}) {
6056                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6057                     $migrate_opts->{spice_ticket});
6058                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6059             }
6060         }
6061
6062     } else {
6063         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6064             if !$statefile && $conf->{balloon};
6065
6066         foreach my $opt (keys %$conf) {
6067             next if $opt !~  m/^net\d+$/;
6068             my $nicconf = parse_net($conf->{$opt});
6069             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6070         }
6071         add_nets_bridge_fdb($conf, $vmid);
6072     }
6073
6074    if (!defined($conf->{balloon}) || $conf->{balloon}) {
6075         eval {
6076             mon_cmd(
6077                 $vmid,
6078                 'qom-set',
6079                 path => "machine/peripheral/balloon0",
6080                 property => "guest-stats-polling-interval",
6081                 value => 2
6082             );
6083         };
6084         log_warn("could not set polling interval for ballooning - $@") if $@;
6085     }
6086
6087     if ($resume) {
6088         print "Resumed VM, removing state\n";
6089         if (my $vmstate = $conf->{vmstate}) {
6090             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6091             PVE::Storage::vdisk_free($storecfg, $vmstate);
6092         }
6093         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6094         PVE::QemuConfig->write_config($vmid, $conf);
6095     }
6096
6097     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6098
6099     return $res;
6100 }
6101
6102 sub vm_commandline {
6103     my ($storecfg, $vmid, $snapname) = @_;
6104
6105     my $conf = PVE::QemuConfig->load_config($vmid);
6106
6107     my ($forcemachine, $forcecpu);
6108     if ($snapname) {
6109         my $snapshot = $conf->{snapshots}->{$snapname};
6110         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6111
6112         # check for machine or CPU overrides in snapshot
6113         $forcemachine = $snapshot->{runningmachine};
6114         $forcecpu = $snapshot->{runningcpu};
6115
6116         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6117
6118         $conf = $snapshot;
6119     }
6120
6121     my $defaults = load_defaults();
6122
6123     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6124
6125     return PVE::Tools::cmd2string($cmd);
6126 }
6127
6128 sub vm_reset {
6129     my ($vmid, $skiplock) = @_;
6130
6131     PVE::QemuConfig->lock_config($vmid, sub {
6132
6133         my $conf = PVE::QemuConfig->load_config($vmid);
6134
6135         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6136
6137         mon_cmd($vmid, "system_reset");
6138     });
6139 }
6140
6141 sub get_vm_volumes {
6142     my ($conf) = @_;
6143
6144     my $vollist = [];
6145     foreach_volid($conf, sub {
6146         my ($volid, $attr) = @_;
6147
6148         return if $volid =~ m|^/|;
6149
6150         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6151         return if !$sid;
6152
6153         push @$vollist, $volid;
6154     });
6155
6156     return $vollist;
6157 }
6158
6159 sub cleanup_pci_devices {
6160     my ($vmid, $conf) = @_;
6161
6162     foreach my $key (keys %$conf) {
6163         next if $key !~ m/^hostpci(\d+)$/;
6164         my $hostpciindex = $1;
6165         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6166         my $d = parse_hostpci($conf->{$key});
6167         if ($d->{mdev}) {
6168             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6169             # don't want to break ABI just for this two liner
6170             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6171
6172             # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6173             # out when we do it first. so wait for 10 seconds and then try it
6174             my $pciid = $d->{pciid}->[0]->{id};
6175             my $info = PVE::SysFSTools::pci_device_info("$pciid");
6176             if ($info->{vendor} eq '10de') {
6177                 sleep 10;
6178             }
6179
6180             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6181         }
6182     }
6183     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6184 }
6185
6186 sub vm_stop_cleanup {
6187     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6188
6189     eval {
6190
6191         if (!$keepActive) {
6192             my $vollist = get_vm_volumes($conf);
6193             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6194
6195             if (my $tpmdrive = $conf->{tpmstate0}) {
6196                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6197                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6198                 if ($storeid) {
6199                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6200                 }
6201             }
6202         }
6203
6204         foreach my $ext (qw(mon qmp pid vnc qga)) {
6205             unlink "/var/run/qemu-server/${vmid}.$ext";
6206         }
6207
6208         if ($conf->{ivshmem}) {
6209             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6210             # just delete it for now, VMs which have this already open do not
6211             # are affected, but new VMs will get a separated one. If this
6212             # becomes an issue we either add some sort of ref-counting or just
6213             # add a "don't delete on stop" flag to the ivshmem format.
6214             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6215         }
6216
6217         cleanup_pci_devices($vmid, $conf);
6218
6219         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6220     };
6221     warn $@ if $@; # avoid errors - just warn
6222 }
6223
6224 # call only in locked context
6225 sub _do_vm_stop {
6226     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6227
6228     my $pid = check_running($vmid, $nocheck);
6229     return if !$pid;
6230
6231     my $conf;
6232     if (!$nocheck) {
6233         $conf = PVE::QemuConfig->load_config($vmid);
6234         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6235         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6236             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6237             $timeout = $opts->{down} if $opts->{down};
6238         }
6239         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6240     }
6241
6242     eval {
6243         if ($shutdown) {
6244             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6245                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6246             } else {
6247                 mon_cmd($vmid, "system_powerdown");
6248             }
6249         } else {
6250             mon_cmd($vmid, "quit");
6251         }
6252     };
6253     my $err = $@;
6254
6255     if (!$err) {
6256         $timeout = 60 if !defined($timeout);
6257
6258         my $count = 0;
6259         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6260             $count++;
6261             sleep 1;
6262         }
6263
6264         if ($count >= $timeout) {
6265             if ($force) {
6266                 warn "VM still running - terminating now with SIGTERM\n";
6267                 kill 15, $pid;
6268             } else {
6269                 die "VM quit/powerdown failed - got timeout\n";
6270             }
6271         } else {
6272             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6273             return;
6274         }
6275     } else {
6276        if (!check_running($vmid, $nocheck)) {
6277             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6278             return;
6279        }
6280        if ($force) {
6281             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6282             kill 15, $pid;
6283         } else {
6284             die "VM quit/powerdown failed\n";
6285         }
6286     }
6287
6288     # wait again
6289     $timeout = 10;
6290
6291     my $count = 0;
6292     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6293         $count++;
6294         sleep 1;
6295     }
6296
6297     if ($count >= $timeout) {
6298         warn "VM still running - terminating now with SIGKILL\n";
6299         kill 9, $pid;
6300         sleep 1;
6301     }
6302
6303     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6304 }
6305
6306 # Note: use $nocheck to skip tests if VM configuration file exists.
6307 # We need that when migration VMs to other nodes (files already moved)
6308 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6309 sub vm_stop {
6310     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6311
6312     $force = 1 if !defined($force) && !$shutdown;
6313
6314     if ($migratedfrom){
6315         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6316         kill 15, $pid if $pid;
6317         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6318         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6319         return;
6320     }
6321
6322     PVE::QemuConfig->lock_config($vmid, sub {
6323         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6324    });
6325 }
6326
6327 sub vm_reboot {
6328     my ($vmid, $timeout) = @_;
6329
6330     PVE::QemuConfig->lock_config($vmid, sub {
6331         eval {
6332
6333             # only reboot if running, as qmeventd starts it again on a stop event
6334             return if !check_running($vmid);
6335
6336             create_reboot_request($vmid);
6337
6338             my $storecfg = PVE::Storage::config();
6339             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6340
6341         };
6342         if (my $err = $@) {
6343             # avoid that the next normal shutdown will be confused for a reboot
6344             clear_reboot_request($vmid);
6345             die $err;
6346         }
6347    });
6348 }
6349
6350 # note: if using the statestorage parameter, the caller has to check privileges
6351 sub vm_suspend {
6352     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6353
6354     my $conf;
6355     my $path;
6356     my $storecfg;
6357     my $vmstate;
6358
6359     PVE::QemuConfig->lock_config($vmid, sub {
6360
6361         $conf = PVE::QemuConfig->load_config($vmid);
6362
6363         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6364         PVE::QemuConfig->check_lock($conf)
6365             if !($skiplock || $is_backing_up);
6366
6367         die "cannot suspend to disk during backup\n"
6368             if $is_backing_up && $includestate;
6369
6370         if ($includestate) {
6371             $conf->{lock} = 'suspending';
6372             my $date = strftime("%Y-%m-%d", localtime(time()));
6373             $storecfg = PVE::Storage::config();
6374             if (!$statestorage) {
6375                 $statestorage = find_vmstate_storage($conf, $storecfg);
6376                 # check permissions for the storage
6377                 my $rpcenv = PVE::RPCEnvironment::get();
6378                 if ($rpcenv->{type} ne 'cli') {
6379                     my $authuser = $rpcenv->get_user();
6380                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6381                 }
6382             }
6383
6384
6385             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6386                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6387             $path = PVE::Storage::path($storecfg, $vmstate);
6388             PVE::QemuConfig->write_config($vmid, $conf);
6389         } else {
6390             mon_cmd($vmid, "stop");
6391         }
6392     });
6393
6394     if ($includestate) {
6395         # save vm state
6396         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6397
6398         eval {
6399             set_migration_caps($vmid, 1);
6400             mon_cmd($vmid, "savevm-start", statefile => $path);
6401             for(;;) {
6402                 my $state = mon_cmd($vmid, "query-savevm");
6403                 if (!$state->{status}) {
6404                     die "savevm not active\n";
6405                 } elsif ($state->{status} eq 'active') {
6406                     sleep(1);
6407                     next;
6408                 } elsif ($state->{status} eq 'completed') {
6409                     print "State saved, quitting\n";
6410                     last;
6411                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6412                     die "query-savevm failed with error '$state->{error}'\n"
6413                 } else {
6414                     die "query-savevm returned status '$state->{status}'\n";
6415                 }
6416             }
6417         };
6418         my $err = $@;
6419
6420         PVE::QemuConfig->lock_config($vmid, sub {
6421             $conf = PVE::QemuConfig->load_config($vmid);
6422             if ($err) {
6423                 # cleanup, but leave suspending lock, to indicate something went wrong
6424                 eval {
6425                     mon_cmd($vmid, "savevm-end");
6426                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6427                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6428                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6429                     PVE::QemuConfig->write_config($vmid, $conf);
6430                 };
6431                 warn $@ if $@;
6432                 die $err;
6433             }
6434
6435             die "lock changed unexpectedly\n"
6436                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6437
6438             mon_cmd($vmid, "quit");
6439             $conf->{lock} = 'suspended';
6440             PVE::QemuConfig->write_config($vmid, $conf);
6441         });
6442     }
6443 }
6444
6445 # $nocheck is set when called as part of a migration - in this context the
6446 # location of the config file (source or target node) is not deterministic,
6447 # since migration cannot wait for pmxcfs to process the rename
6448 sub vm_resume {
6449     my ($vmid, $skiplock, $nocheck) = @_;
6450
6451     PVE::QemuConfig->lock_config($vmid, sub {
6452         my $res = mon_cmd($vmid, 'query-status');
6453         my $resume_cmd = 'cont';
6454         my $reset = 0;
6455         my $conf;
6456         if ($nocheck) {
6457             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6458             if ($@) {
6459                 my $vmlist = PVE::Cluster::get_vmlist();
6460                 if (exists($vmlist->{ids}->{$vmid})) {
6461                     my $node = $vmlist->{ids}->{$vmid}->{node};
6462                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6463                 }
6464                 if (!$conf) {
6465                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6466                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6467                 }
6468             }
6469         } else {
6470             $conf = PVE::QemuConfig->load_config($vmid);
6471         }
6472
6473         if ($res->{status}) {
6474             return if $res->{status} eq 'running'; # job done, go home
6475             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6476             $reset = 1 if $res->{status} eq 'shutdown';
6477         }
6478
6479         if (!$nocheck) {
6480             PVE::QemuConfig->check_lock($conf)
6481                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6482         }
6483
6484         if ($reset) {
6485             # required if a VM shuts down during a backup and we get a resume
6486             # request before the backup finishes for example
6487             mon_cmd($vmid, "system_reset");
6488         }
6489
6490         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6491
6492         mon_cmd($vmid, $resume_cmd);
6493     });
6494 }
6495
6496 sub vm_sendkey {
6497     my ($vmid, $skiplock, $key) = @_;
6498
6499     PVE::QemuConfig->lock_config($vmid, sub {
6500
6501         my $conf = PVE::QemuConfig->load_config($vmid);
6502
6503         # there is no qmp command, so we use the human monitor command
6504         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6505         die $res if $res ne '';
6506     });
6507 }
6508
6509 # vzdump restore implementaion
6510
6511 sub tar_archive_read_firstfile {
6512     my $archive = shift;
6513
6514     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6515
6516     # try to detect archive type first
6517     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6518         die "unable to open file '$archive'\n";
6519     my $firstfile = <$fh>;
6520     kill 15, $pid;
6521     close $fh;
6522
6523     die "ERROR: archive contaions no data\n" if !$firstfile;
6524     chomp $firstfile;
6525
6526     return $firstfile;
6527 }
6528
6529 sub tar_restore_cleanup {
6530     my ($storecfg, $statfile) = @_;
6531
6532     print STDERR "starting cleanup\n";
6533
6534     if (my $fd = IO::File->new($statfile, "r")) {
6535         while (defined(my $line = <$fd>)) {
6536             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6537                 my $volid = $2;
6538                 eval {
6539                     if ($volid =~ m|^/|) {
6540                         unlink $volid || die 'unlink failed\n';
6541                     } else {
6542                         PVE::Storage::vdisk_free($storecfg, $volid);
6543                     }
6544                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6545                 };
6546                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6547             } else {
6548                 print STDERR "unable to parse line in statfile - $line";
6549             }
6550         }
6551         $fd->close();
6552     }
6553 }
6554
6555 sub restore_file_archive {
6556     my ($archive, $vmid, $user, $opts) = @_;
6557
6558     return restore_vma_archive($archive, $vmid, $user, $opts)
6559         if $archive eq '-';
6560
6561     my $info = PVE::Storage::archive_info($archive);
6562     my $format = $opts->{format} // $info->{format};
6563     my $comp = $info->{compression};
6564
6565     # try to detect archive format
6566     if ($format eq 'tar') {
6567         return restore_tar_archive($archive, $vmid, $user, $opts);
6568     } else {
6569         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6570     }
6571 }
6572
6573 # hepler to remove disks that will not be used after restore
6574 my $restore_cleanup_oldconf = sub {
6575     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6576
6577     my $kept_disks = {};
6578
6579     PVE::QemuConfig->foreach_volume($oldconf, sub {
6580         my ($ds, $drive) = @_;
6581
6582         return if drive_is_cdrom($drive, 1);
6583
6584         my $volid = $drive->{file};
6585         return if !$volid || $volid =~ m|^/|;
6586
6587         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6588         return if !$path || !$owner || ($owner != $vmid);
6589
6590         # Note: only delete disk we want to restore
6591         # other volumes will become unused
6592         if ($virtdev_hash->{$ds}) {
6593             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6594             if (my $err = $@) {
6595                 warn $err;
6596             }
6597         } else {
6598             $kept_disks->{$volid} = 1;
6599         }
6600     });
6601
6602     # after the restore we have no snapshots anymore
6603     for my $snapname (keys $oldconf->{snapshots}->%*) {
6604         my $snap = $oldconf->{snapshots}->{$snapname};
6605         if ($snap->{vmstate}) {
6606             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6607             if (my $err = $@) {
6608                 warn $err;
6609             }
6610         }
6611
6612         for my $volid (keys $kept_disks->%*) {
6613             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6614             warn $@ if $@;
6615         }
6616     }
6617 };
6618
6619 # Helper to parse vzdump backup device hints
6620 #
6621 # $rpcenv: Environment, used to ckeck storage permissions
6622 # $user: User ID, to check storage permissions
6623 # $storecfg: Storage configuration
6624 # $fh: the file handle for reading the configuration
6625 # $devinfo: should contain device sizes for all backu-up'ed devices
6626 # $options: backup options (pool, default storage)
6627 #
6628 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6629 my $parse_backup_hints = sub {
6630     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6631
6632     my $check_storage = sub { # assert if an image can be allocate
6633         my ($storeid, $scfg) = @_;
6634         die "Content type 'images' is not available on storage '$storeid'\n"
6635             if !$scfg->{content}->{images};
6636         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6637             if $user ne 'root@pam';
6638     };
6639
6640     my $virtdev_hash = {};
6641     while (defined(my $line = <$fh>)) {
6642         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6643             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6644             die "archive does not contain data for drive '$virtdev'\n"
6645                 if !$devinfo->{$devname};
6646
6647             if (defined($options->{storage})) {
6648                 $storeid = $options->{storage} || 'local';
6649             } elsif (!$storeid) {
6650                 $storeid = 'local';
6651             }
6652             $format = 'raw' if !$format;
6653             $devinfo->{$devname}->{devname} = $devname;
6654             $devinfo->{$devname}->{virtdev} = $virtdev;
6655             $devinfo->{$devname}->{format} = $format;
6656             $devinfo->{$devname}->{storeid} = $storeid;
6657
6658             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6659             $check_storage->($storeid, $scfg); # permission and content type check
6660
6661             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6662         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6663             my $virtdev = $1;
6664             my $drive = parse_drive($virtdev, $2);
6665
6666             if (drive_is_cloudinit($drive)) {
6667                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6668                 $storeid = $options->{storage} if defined ($options->{storage});
6669                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6670                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6671
6672                 $check_storage->($storeid, $scfg); # permission and content type check
6673
6674                 $virtdev_hash->{$virtdev} = {
6675                     format => $format,
6676                     storeid => $storeid,
6677                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6678                     is_cloudinit => 1,
6679                 };
6680             }
6681         }
6682     }
6683
6684     return $virtdev_hash;
6685 };
6686
6687 # Helper to allocate and activate all volumes required for a restore
6688 #
6689 # $storecfg: Storage configuration
6690 # $virtdev_hash: as returned by parse_backup_hints()
6691 #
6692 # Returns: { $virtdev => $volid }
6693 my $restore_allocate_devices = sub {
6694     my ($storecfg, $virtdev_hash, $vmid) = @_;
6695
6696     my $map = {};
6697     foreach my $virtdev (sort keys %$virtdev_hash) {
6698         my $d = $virtdev_hash->{$virtdev};
6699         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6700         my $storeid = $d->{storeid};
6701         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6702
6703         # test if requested format is supported
6704         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6705         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6706         $d->{format} = $defFormat if !$supported;
6707
6708         my $name;
6709         if ($d->{is_cloudinit}) {
6710             $name = "vm-$vmid-cloudinit";
6711             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6712             if ($scfg->{path}) {
6713                 $name .= ".$d->{format}";
6714             }
6715         }
6716
6717         my $volid = PVE::Storage::vdisk_alloc(
6718             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6719
6720         print STDERR "new volume ID is '$volid'\n";
6721         $d->{volid} = $volid;
6722
6723         PVE::Storage::activate_volumes($storecfg, [$volid]);
6724
6725         $map->{$virtdev} = $volid;
6726     }
6727
6728     return $map;
6729 };
6730
6731 sub restore_update_config_line {
6732     my ($cookie, $map, $line, $unique) = @_;
6733
6734     return '' if $line =~ m/^\#qmdump\#/;
6735     return '' if $line =~ m/^\#vzdump\#/;
6736     return '' if $line =~ m/^lock:/;
6737     return '' if $line =~ m/^unused\d+:/;
6738     return '' if $line =~ m/^parent:/;
6739
6740     my $res = '';
6741
6742     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6743     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6744         # try to convert old 1.X settings
6745         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6746         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6747             my ($model, $macaddr) = split(/\=/, $devconfig);
6748             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6749             my $net = {
6750                 model => $model,
6751                 bridge => "vmbr$ind",
6752                 macaddr => $macaddr,
6753             };
6754             my $netstr = print_net($net);
6755
6756             $res .= "net$cookie->{netcount}: $netstr\n";
6757             $cookie->{netcount}++;
6758         }
6759     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6760         my ($id, $netstr) = ($1, $2);
6761         my $net = parse_net($netstr);
6762         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6763         $netstr = print_net($net);
6764         $res .= "$id: $netstr\n";
6765     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6766         my $virtdev = $1;
6767         my $value = $3;
6768         my $di = parse_drive($virtdev, $value);
6769         if (defined($di->{backup}) && !$di->{backup}) {
6770             $res .= "#$line";
6771         } elsif ($map->{$virtdev}) {
6772             delete $di->{format}; # format can change on restore
6773             $di->{file} = $map->{$virtdev};
6774             $value = print_drive($di);
6775             $res .= "$virtdev: $value\n";
6776         } else {
6777             $res .= $line;
6778         }
6779     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6780         my $vmgenid = $1;
6781         if ($vmgenid ne '0') {
6782             # always generate a new vmgenid if there was a valid one setup
6783             $vmgenid = generate_uuid();
6784         }
6785         $res .= "vmgenid: $vmgenid\n";
6786     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6787         my ($uuid, $uuid_str);
6788         UUID::generate($uuid);
6789         UUID::unparse($uuid, $uuid_str);
6790         my $smbios1 = parse_smbios1($2);
6791         $smbios1->{uuid} = $uuid_str;
6792         $res .= $1.print_smbios1($smbios1)."\n";
6793     } else {
6794         $res .= $line;
6795     }
6796
6797     return $res;
6798 }
6799
6800 my $restore_deactivate_volumes = sub {
6801     my ($storecfg, $virtdev_hash) = @_;
6802
6803     my $vollist = [];
6804     for my $dev (values $virtdev_hash->%*) {
6805         push $vollist->@*, $dev->{volid} if $dev->{volid};
6806     }
6807
6808     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6809     print STDERR $@ if $@;
6810 };
6811
6812 my $restore_destroy_volumes = sub {
6813     my ($storecfg, $virtdev_hash) = @_;
6814
6815     for my $dev (values $virtdev_hash->%*) {
6816         my $volid = $dev->{volid} or next;
6817         eval {
6818             PVE::Storage::vdisk_free($storecfg, $volid);
6819             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6820         };
6821         print STDERR "unable to cleanup '$volid' - $@" if $@;
6822     }
6823 };
6824
6825 my $restore_merge_config = sub {
6826     my ($filename, $backup_conf_raw, $override_conf) = @_;
6827
6828     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6829     for my $key (keys $override_conf->%*) {
6830         $backup_conf->{$key} = $override_conf->{$key};
6831     }
6832
6833     return $backup_conf;
6834 };
6835
6836 sub scan_volids {
6837     my ($cfg, $vmid) = @_;
6838
6839     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6840
6841     my $volid_hash = {};
6842     foreach my $storeid (keys %$info) {
6843         foreach my $item (@{$info->{$storeid}}) {
6844             next if !($item->{volid} && $item->{size});
6845             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6846             $volid_hash->{$item->{volid}} = $item;
6847         }
6848     }
6849
6850     return $volid_hash;
6851 }
6852
6853 sub update_disk_config {
6854     my ($vmid, $conf, $volid_hash) = @_;
6855
6856     my $changes;
6857     my $prefix = "VM $vmid";
6858
6859     # used and unused disks
6860     my $referenced = {};
6861
6862     # Note: it is allowed to define multiple storages with same path (alias), so
6863     # we need to check both 'volid' and real 'path' (two different volid can point
6864     # to the same path).
6865
6866     my $referencedpath = {};
6867
6868     # update size info
6869     PVE::QemuConfig->foreach_volume($conf, sub {
6870         my ($opt, $drive) = @_;
6871
6872         my $volid = $drive->{file};
6873         return if !$volid;
6874         my $volume = $volid_hash->{$volid};
6875
6876         # mark volid as "in-use" for next step
6877         $referenced->{$volid} = 1;
6878         if ($volume && (my $path = $volume->{path})) {
6879             $referencedpath->{$path} = 1;
6880         }
6881
6882         return if drive_is_cdrom($drive);
6883         return if !$volume;
6884
6885         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6886         if (defined($updated)) {
6887             $changes = 1;
6888             $conf->{$opt} = print_drive($updated);
6889             print "$prefix ($opt): $msg\n";
6890         }
6891     });
6892
6893     # remove 'unusedX' entry if volume is used
6894     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6895         my ($opt, $drive) = @_;
6896
6897         my $volid = $drive->{file};
6898         return if !$volid;
6899
6900         my $path;
6901         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6902         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6903             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6904             $changes = 1;
6905             delete $conf->{$opt};
6906         }
6907
6908         $referenced->{$volid} = 1;
6909         $referencedpath->{$path} = 1 if $path;
6910     });
6911
6912     foreach my $volid (sort keys %$volid_hash) {
6913         next if $volid =~ m/vm-$vmid-state-/;
6914         next if $referenced->{$volid};
6915         my $path = $volid_hash->{$volid}->{path};
6916         next if !$path; # just to be sure
6917         next if $referencedpath->{$path};
6918         $changes = 1;
6919         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6920         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6921         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6922     }
6923
6924     return $changes;
6925 }
6926
6927 sub rescan {
6928     my ($vmid, $nolock, $dryrun) = @_;
6929
6930     my $cfg = PVE::Storage::config();
6931
6932     print "rescan volumes...\n";
6933     my $volid_hash = scan_volids($cfg, $vmid);
6934
6935     my $updatefn =  sub {
6936         my ($vmid) = @_;
6937
6938         my $conf = PVE::QemuConfig->load_config($vmid);
6939
6940         PVE::QemuConfig->check_lock($conf);
6941
6942         my $vm_volids = {};
6943         foreach my $volid (keys %$volid_hash) {
6944             my $info = $volid_hash->{$volid};
6945             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6946         }
6947
6948         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6949
6950         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6951     };
6952
6953     if (defined($vmid)) {
6954         if ($nolock) {
6955             &$updatefn($vmid);
6956         } else {
6957             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6958         }
6959     } else {
6960         my $vmlist = config_list();
6961         foreach my $vmid (keys %$vmlist) {
6962             if ($nolock) {
6963                 &$updatefn($vmid);
6964             } else {
6965                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6966             }
6967         }
6968     }
6969 }
6970
6971 sub restore_proxmox_backup_archive {
6972     my ($archive, $vmid, $user, $options) = @_;
6973
6974     my $storecfg = PVE::Storage::config();
6975
6976     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6977     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6978
6979     my $fingerprint = $scfg->{fingerprint};
6980     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6981
6982     my $repo = PVE::PBSClient::get_repository($scfg);
6983     my $namespace = $scfg->{namespace};
6984
6985     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6986     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6987     local $ENV{PBS_PASSWORD} = $password;
6988     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6989
6990     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6991         PVE::Storage::parse_volname($storecfg, $archive);
6992
6993     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6994
6995     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6996
6997     my $tmpdir = "/var/tmp/vzdumptmp$$";
6998     rmtree $tmpdir;
6999     mkpath $tmpdir;
7000
7001     my $conffile = PVE::QemuConfig->config_file($vmid);
7002      # disable interrupts (always do cleanups)
7003     local $SIG{INT} =
7004         local $SIG{TERM} =
7005         local $SIG{QUIT} =
7006         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7007
7008     # Note: $oldconf is undef if VM does not exists
7009     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7010     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7011     my $new_conf_raw = '';
7012
7013     my $rpcenv = PVE::RPCEnvironment::get();
7014     my $devinfo = {}; # info about drives included in backup
7015     my $virtdev_hash = {}; # info about allocated drives
7016
7017     eval {
7018         # enable interrupts
7019         local $SIG{INT} =
7020             local $SIG{TERM} =
7021             local $SIG{QUIT} =
7022             local $SIG{HUP} =
7023             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7024
7025         my $cfgfn = "$tmpdir/qemu-server.conf";
7026         my $firewall_config_fn = "$tmpdir/fw.conf";
7027         my $index_fn = "$tmpdir/index.json";
7028
7029         my $cmd = "restore";
7030
7031         my $param = [$pbs_backup_name, "index.json", $index_fn];
7032         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7033         my $index = PVE::Tools::file_get_contents($index_fn);
7034         $index = decode_json($index);
7035
7036         foreach my $info (@{$index->{files}}) {
7037             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7038                 my $devname = $1;
7039                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7040                     $devinfo->{$devname}->{size} = $1;
7041                 } else {
7042                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7043                 }
7044             }
7045         }
7046
7047         my $is_qemu_server_backup = scalar(
7048             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7049         );
7050         if (!$is_qemu_server_backup) {
7051             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7052         }
7053         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7054
7055         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7056         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7057
7058         if ($has_firewall_config) {
7059             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7060             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7061
7062             my $pve_firewall_dir = '/etc/pve/firewall';
7063             mkdir $pve_firewall_dir; # make sure the dir exists
7064             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7065         }
7066
7067         my $fh = IO::File->new($cfgfn, "r") ||
7068             die "unable to read qemu-server.conf - $!\n";
7069
7070         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7071
7072         # fixme: rate limit?
7073
7074         # create empty/temp config
7075         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7076
7077         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7078
7079         # allocate volumes
7080         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7081
7082         foreach my $virtdev (sort keys %$virtdev_hash) {
7083             my $d = $virtdev_hash->{$virtdev};
7084             next if $d->{is_cloudinit}; # no need to restore cloudinit
7085
7086             # this fails if storage is unavailable
7087             my $volid = $d->{volid};
7088             my $path = PVE::Storage::path($storecfg, $volid);
7089
7090             # for live-restore we only want to preload the efidisk and TPM state
7091             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7092
7093             my @ns_arg;
7094             if (defined(my $ns = $scfg->{namespace})) {
7095                 @ns_arg = ('--ns', $ns);
7096             }
7097
7098             my $pbs_restore_cmd = [
7099                 '/usr/bin/pbs-restore',
7100                 '--repository', $repo,
7101                 @ns_arg,
7102                 $pbs_backup_name,
7103                 "$d->{devname}.img.fidx",
7104                 $path,
7105                 '--verbose',
7106                 ];
7107
7108             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7109             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7110
7111             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7112                 push @$pbs_restore_cmd, '--skip-zero';
7113             }
7114
7115             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7116             print "restore proxmox backup image: $dbg_cmdstring\n";
7117             run_command($pbs_restore_cmd);
7118         }
7119
7120         $fh->seek(0, 0) || die "seek failed - $!\n";
7121
7122         my $cookie = { netcount => 0 };
7123         while (defined(my $line = <$fh>)) {
7124             $new_conf_raw .= restore_update_config_line(
7125                 $cookie,
7126                 $map,
7127                 $line,
7128                 $options->{unique},
7129             );
7130         }
7131
7132         $fh->close();
7133     };
7134     my $err = $@;
7135
7136     if ($err || !$options->{live}) {
7137         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7138     }
7139
7140     rmtree $tmpdir;
7141
7142     if ($err) {
7143         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7144         die $err;
7145     }
7146
7147     if ($options->{live}) {
7148         # keep lock during live-restore
7149         $new_conf_raw .= "\nlock: create";
7150     }
7151
7152     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7153     PVE::QemuConfig->write_config($vmid, $new_conf);
7154
7155     eval { rescan($vmid, 1); };
7156     warn $@ if $@;
7157
7158     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7159
7160     if ($options->{live}) {
7161         # enable interrupts
7162         local $SIG{INT} =
7163             local $SIG{TERM} =
7164             local $SIG{QUIT} =
7165             local $SIG{HUP} =
7166             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7167
7168         my $conf = PVE::QemuConfig->load_config($vmid);
7169         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7170
7171         # these special drives are already restored before start
7172         delete $devinfo->{'drive-efidisk0'};
7173         delete $devinfo->{'drive-tpmstate0-backup'};
7174
7175         my $pbs_opts = {
7176             repo => $repo,
7177             keyfile => $keyfile,
7178             snapshot => $pbs_backup_name,
7179             namespace => $namespace,
7180         };
7181         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7182
7183         PVE::QemuConfig->remove_lock($vmid, "create");
7184     }
7185 }
7186
7187 sub pbs_live_restore {
7188     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7189
7190     print "starting VM for live-restore\n";
7191     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7192
7193     my $pbs_backing = {};
7194     for my $ds (keys %$restored_disks) {
7195         $ds =~ m/^drive-(.*)$/;
7196         my $confname = $1;
7197         $pbs_backing->{$confname} = {
7198             repository => $opts->{repo},
7199             snapshot => $opts->{snapshot},
7200             archive => "$ds.img.fidx",
7201         };
7202         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7203         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7204
7205         my $drive = parse_drive($confname, $conf->{$confname});
7206         print "restoring '$ds' to '$drive->{file}'\n";
7207     }
7208
7209     my $drives_streamed = 0;
7210     eval {
7211         # make sure HA doesn't interrupt our restore by stopping the VM
7212         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7213             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7214         }
7215
7216         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7217         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7218         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7219
7220         my $qmeventd_fd = register_qmeventd_handle($vmid);
7221
7222         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7223         # this will effectively collapse the backing image chain consisting of
7224         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7225         # removes itself once all backing images vanish with 'auto-remove=on')
7226         my $jobs = {};
7227         for my $ds (sort keys %$restored_disks) {
7228             my $job_id = "restore-$ds";
7229             mon_cmd($vmid, 'block-stream',
7230                 'job-id' => $job_id,
7231                 device => "$ds",
7232             );
7233             $jobs->{$job_id} = {};
7234         }
7235
7236         mon_cmd($vmid, 'cont');
7237         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7238
7239         print "restore-drive jobs finished successfully, removing all tracking block devices"
7240             ." to disconnect from Proxmox Backup Server\n";
7241
7242         for my $ds (sort keys %$restored_disks) {
7243             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7244         }
7245
7246         close($qmeventd_fd);
7247     };
7248
7249     my $err = $@;
7250
7251     if ($err) {
7252         warn "An error occurred during live-restore: $err\n";
7253         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7254         die "live-restore failed\n";
7255     }
7256 }
7257
7258 sub restore_vma_archive {
7259     my ($archive, $vmid, $user, $opts, $comp) = @_;
7260
7261     my $readfrom = $archive;
7262
7263     my $cfg = PVE::Storage::config();
7264     my $commands = [];
7265     my $bwlimit = $opts->{bwlimit};
7266
7267     my $dbg_cmdstring = '';
7268     my $add_pipe = sub {
7269         my ($cmd) = @_;
7270         push @$commands, $cmd;
7271         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7272         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7273         $readfrom = '-';
7274     };
7275
7276     my $input = undef;
7277     if ($archive eq '-') {
7278         $input = '<&STDIN';
7279     } else {
7280         # If we use a backup from a PVE defined storage we also consider that
7281         # storage's rate limit:
7282         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7283         if (defined($volid)) {
7284             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7285             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7286             if ($readlimit) {
7287                 print STDERR "applying read rate limit: $readlimit\n";
7288                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7289                 $add_pipe->($cstream);
7290             }
7291         }
7292     }
7293
7294     if ($comp) {
7295         my $info = PVE::Storage::decompressor_info('vma', $comp);
7296         my $cmd = $info->{decompressor};
7297         push @$cmd, $readfrom;
7298         $add_pipe->($cmd);
7299     }
7300
7301     my $tmpdir = "/var/tmp/vzdumptmp$$";
7302     rmtree $tmpdir;
7303
7304     # disable interrupts (always do cleanups)
7305     local $SIG{INT} =
7306         local $SIG{TERM} =
7307         local $SIG{QUIT} =
7308         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7309
7310     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7311     POSIX::mkfifo($mapfifo, 0600);
7312     my $fifofh;
7313     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7314
7315     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7316
7317     my $oldtimeout;
7318     my $timeout = 5;
7319
7320     my $devinfo = {}; # info about drives included in backup
7321     my $virtdev_hash = {}; # info about allocated drives
7322
7323     my $rpcenv = PVE::RPCEnvironment::get();
7324
7325     my $conffile = PVE::QemuConfig->config_file($vmid);
7326
7327     # Note: $oldconf is undef if VM does not exist
7328     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7329     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7330     my $new_conf_raw = '';
7331
7332     my %storage_limits;
7333
7334     my $print_devmap = sub {
7335         my $cfgfn = "$tmpdir/qemu-server.conf";
7336
7337         # we can read the config - that is already extracted
7338         my $fh = IO::File->new($cfgfn, "r") ||
7339             die "unable to read qemu-server.conf - $!\n";
7340
7341         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7342         if (-f $fwcfgfn) {
7343             my $pve_firewall_dir = '/etc/pve/firewall';
7344             mkdir $pve_firewall_dir; # make sure the dir exists
7345             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7346         }
7347
7348         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7349
7350         foreach my $info (values %{$virtdev_hash}) {
7351             my $storeid = $info->{storeid};
7352             next if defined($storage_limits{$storeid});
7353
7354             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7355             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7356             $storage_limits{$storeid} = $limit * 1024;
7357         }
7358
7359         foreach my $devname (keys %$devinfo) {
7360             die "found no device mapping information for device '$devname'\n"
7361                 if !$devinfo->{$devname}->{virtdev};
7362         }
7363
7364         # create empty/temp config
7365         if ($oldconf) {
7366             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7367             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7368         }
7369
7370         # allocate volumes
7371         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7372
7373         # print restore information to $fifofh
7374         foreach my $virtdev (sort keys %$virtdev_hash) {
7375             my $d = $virtdev_hash->{$virtdev};
7376             next if $d->{is_cloudinit}; # no need to restore cloudinit
7377
7378             my $storeid = $d->{storeid};
7379             my $volid = $d->{volid};
7380
7381             my $map_opts = '';
7382             if (my $limit = $storage_limits{$storeid}) {
7383                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7384             }
7385
7386             my $write_zeros = 1;
7387             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7388                 $write_zeros = 0;
7389             }
7390
7391             my $path = PVE::Storage::path($cfg, $volid);
7392
7393             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7394
7395             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7396         }
7397
7398         $fh->seek(0, 0) || die "seek failed - $!\n";
7399
7400         my $cookie = { netcount => 0 };
7401         while (defined(my $line = <$fh>)) {
7402             $new_conf_raw .= restore_update_config_line(
7403                 $cookie,
7404                 $map,
7405                 $line,
7406                 $opts->{unique},
7407             );
7408         }
7409
7410         $fh->close();
7411     };
7412
7413     eval {
7414         # enable interrupts
7415         local $SIG{INT} =
7416             local $SIG{TERM} =
7417             local $SIG{QUIT} =
7418             local $SIG{HUP} =
7419             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7420         local $SIG{ALRM} = sub { die "got timeout\n"; };
7421
7422         $oldtimeout = alarm($timeout);
7423
7424         my $parser = sub {
7425             my $line = shift;
7426
7427             print "$line\n";
7428
7429             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7430                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7431                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7432             } elsif ($line =~ m/^CTIME: /) {
7433                 # we correctly received the vma config, so we can disable
7434                 # the timeout now for disk allocation (set to 10 minutes, so
7435                 # that we always timeout if something goes wrong)
7436                 alarm(600);
7437                 &$print_devmap();
7438                 print $fifofh "done\n";
7439                 my $tmp = $oldtimeout || 0;
7440                 $oldtimeout = undef;
7441                 alarm($tmp);
7442                 close($fifofh);
7443                 $fifofh = undef;
7444             }
7445         };
7446
7447         print "restore vma archive: $dbg_cmdstring\n";
7448         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7449     };
7450     my $err = $@;
7451
7452     alarm($oldtimeout) if $oldtimeout;
7453
7454     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7455
7456     close($fifofh) if $fifofh;
7457     unlink $mapfifo;
7458     rmtree $tmpdir;
7459
7460     if ($err) {
7461         $restore_destroy_volumes->($cfg, $virtdev_hash);
7462         die $err;
7463     }
7464
7465     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7466     PVE::QemuConfig->write_config($vmid, $new_conf);
7467
7468     eval { rescan($vmid, 1); };
7469     warn $@ if $@;
7470
7471     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7472 }
7473
7474 sub restore_tar_archive {
7475     my ($archive, $vmid, $user, $opts) = @_;
7476
7477     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7478         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7479         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7480     }
7481
7482     if ($archive ne '-') {
7483         my $firstfile = tar_archive_read_firstfile($archive);
7484         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7485             if $firstfile ne 'qemu-server.conf';
7486     }
7487
7488     my $storecfg = PVE::Storage::config();
7489
7490     # avoid zombie disks when restoring over an existing VM -> cleanup first
7491     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7492     # skiplock=1 because qmrestore has set the 'create' lock itself already
7493     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7494     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7495
7496     my $tocmd = "/usr/lib/qemu-server/qmextract";
7497
7498     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7499     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7500     $tocmd .= ' --prealloc' if $opts->{prealloc};
7501     $tocmd .= ' --info' if $opts->{info};
7502
7503     # tar option "xf" does not autodetect compression when read from STDIN,
7504     # so we pipe to zcat
7505     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7506         PVE::Tools::shellquote("--to-command=$tocmd");
7507
7508     my $tmpdir = "/var/tmp/vzdumptmp$$";
7509     mkpath $tmpdir;
7510
7511     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7512     local $ENV{VZDUMP_VMID} = $vmid;
7513     local $ENV{VZDUMP_USER} = $user;
7514
7515     my $conffile = PVE::QemuConfig->config_file($vmid);
7516     my $new_conf_raw = '';
7517
7518     # disable interrupts (always do cleanups)
7519     local $SIG{INT} =
7520         local $SIG{TERM} =
7521         local $SIG{QUIT} =
7522         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7523
7524     eval {
7525         # enable interrupts
7526         local $SIG{INT} =
7527             local $SIG{TERM} =
7528             local $SIG{QUIT} =
7529             local $SIG{HUP} =
7530             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7531
7532         if ($archive eq '-') {
7533             print "extracting archive from STDIN\n";
7534             run_command($cmd, input => "<&STDIN");
7535         } else {
7536             print "extracting archive '$archive'\n";
7537             run_command($cmd);
7538         }
7539
7540         return if $opts->{info};
7541
7542         # read new mapping
7543         my $map = {};
7544         my $statfile = "$tmpdir/qmrestore.stat";
7545         if (my $fd = IO::File->new($statfile, "r")) {
7546             while (defined (my $line = <$fd>)) {
7547                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7548                     $map->{$1} = $2 if $1;
7549                 } else {
7550                     print STDERR "unable to parse line in statfile - $line\n";
7551                 }
7552             }
7553             $fd->close();
7554         }
7555
7556         my $confsrc = "$tmpdir/qemu-server.conf";
7557
7558         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7559
7560         my $cookie = { netcount => 0 };
7561         while (defined (my $line = <$srcfd>)) {
7562             $new_conf_raw .= restore_update_config_line(
7563                 $cookie,
7564                 $map,
7565                 $line,
7566                 $opts->{unique},
7567             );
7568         }
7569
7570         $srcfd->close();
7571     };
7572     if (my $err = $@) {
7573         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7574         die $err;
7575     }
7576
7577     rmtree $tmpdir;
7578
7579     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7580
7581     PVE::Cluster::cfs_update(); # make sure we read new file
7582
7583     eval { rescan($vmid, 1); };
7584     warn $@ if $@;
7585 };
7586
7587 sub foreach_storage_used_by_vm {
7588     my ($conf, $func) = @_;
7589
7590     my $sidhash = {};
7591
7592     PVE::QemuConfig->foreach_volume($conf, sub {
7593         my ($ds, $drive) = @_;
7594         return if drive_is_cdrom($drive);
7595
7596         my $volid = $drive->{file};
7597
7598         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7599         $sidhash->{$sid} = $sid if $sid;
7600     });
7601
7602     foreach my $sid (sort keys %$sidhash) {
7603         &$func($sid);
7604     }
7605 }
7606
7607 my $qemu_snap_storage = {
7608     rbd => 1,
7609 };
7610 sub do_snapshots_with_qemu {
7611     my ($storecfg, $volid, $deviceid) = @_;
7612
7613     return if $deviceid =~ m/tpmstate0/;
7614
7615     my $storage_name = PVE::Storage::parse_volume_id($volid);
7616     my $scfg = $storecfg->{ids}->{$storage_name};
7617     die "could not find storage '$storage_name'\n" if !defined($scfg);
7618
7619     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7620         return 1;
7621     }
7622
7623     if ($volid =~ m/\.(qcow2|qed)$/){
7624         return 1;
7625     }
7626
7627     return;
7628 }
7629
7630 sub qga_check_running {
7631     my ($vmid, $nowarn) = @_;
7632
7633     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7634     if ($@) {
7635         warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7636         return 0;
7637     }
7638     return 1;
7639 }
7640
7641 sub template_create {
7642     my ($vmid, $conf, $disk) = @_;
7643
7644     my $storecfg = PVE::Storage::config();
7645
7646     PVE::QemuConfig->foreach_volume($conf, sub {
7647         my ($ds, $drive) = @_;
7648
7649         return if drive_is_cdrom($drive);
7650         return if $disk && $ds ne $disk;
7651
7652         my $volid = $drive->{file};
7653         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7654
7655         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7656         $drive->{file} = $voliddst;
7657         $conf->{$ds} = print_drive($drive);
7658         PVE::QemuConfig->write_config($vmid, $conf);
7659     });
7660 }
7661
7662 sub convert_iscsi_path {
7663     my ($path) = @_;
7664
7665     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7666         my $portal = $1;
7667         my $target = $2;
7668         my $lun = $3;
7669
7670         my $initiator_name = get_initiator_name();
7671
7672         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7673                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7674     }
7675
7676     die "cannot convert iscsi path '$path', unkown format\n";
7677 }
7678
7679 sub qemu_img_convert {
7680     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7681
7682     my $storecfg = PVE::Storage::config();
7683     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7684     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7685
7686     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7687
7688     my $cachemode;
7689     my $src_path;
7690     my $src_is_iscsi = 0;
7691     my $src_format;
7692
7693     if ($src_storeid) {
7694         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7695         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7696         $src_format = qemu_img_format($src_scfg, $src_volname);
7697         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7698         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7699         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7700     } elsif (-f $src_volid || -b $src_volid) {
7701         $src_path = $src_volid;
7702         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7703             $src_format = $1;
7704         }
7705     }
7706
7707     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7708
7709     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7710     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7711     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7712     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7713
7714     my $cmd = [];
7715     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7716     push @$cmd, '-l', "snapshot.name=$snapname"
7717         if $snapname && $src_format && $src_format eq "qcow2";
7718     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7719     push @$cmd, '-T', $cachemode if defined($cachemode);
7720     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7721
7722     if ($src_is_iscsi) {
7723         push @$cmd, '--image-opts';
7724         $src_path = convert_iscsi_path($src_path);
7725     } elsif ($src_format) {
7726         push @$cmd, '-f', $src_format;
7727     }
7728
7729     if ($dst_is_iscsi) {
7730         push @$cmd, '--target-image-opts';
7731         $dst_path = convert_iscsi_path($dst_path);
7732     } else {
7733         push @$cmd, '-O', $dst_format;
7734     }
7735
7736     push @$cmd, $src_path;
7737
7738     if (!$dst_is_iscsi && $is_zero_initialized) {
7739         push @$cmd, "zeroinit:$dst_path";
7740     } else {
7741         push @$cmd, $dst_path;
7742     }
7743
7744     my $parser = sub {
7745         my $line = shift;
7746         if($line =~ m/\((\S+)\/100\%\)/){
7747             my $percent = $1;
7748             my $transferred = int($size * $percent / 100);
7749             my $total_h = render_bytes($size, 1);
7750             my $transferred_h = render_bytes($transferred, 1);
7751
7752             print "transferred $transferred_h of $total_h ($percent%)\n";
7753         }
7754
7755     };
7756
7757     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7758     my $err = $@;
7759     die "copy failed: $err" if $err;
7760 }
7761
7762 sub qemu_img_format {
7763     my ($scfg, $volname) = @_;
7764
7765     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7766         return $1;
7767     } else {
7768         return "raw";
7769     }
7770 }
7771
7772 sub qemu_drive_mirror {
7773     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7774
7775     $jobs = {} if !$jobs;
7776
7777     my $qemu_target;
7778     my $format;
7779     $jobs->{"drive-$drive"} = {};
7780
7781     if ($dst_volid =~ /^nbd:/) {
7782         $qemu_target = $dst_volid;
7783         $format = "nbd";
7784     } else {
7785         my $storecfg = PVE::Storage::config();
7786         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7787
7788         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7789
7790         $format = qemu_img_format($dst_scfg, $dst_volname);
7791
7792         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7793
7794         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7795     }
7796
7797     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7798     $opts->{format} = $format if $format;
7799
7800     if (defined($src_bitmap)) {
7801         $opts->{sync} = 'incremental';
7802         $opts->{bitmap} = $src_bitmap;
7803         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7804     }
7805
7806     if (defined($bwlimit)) {
7807         $opts->{speed} = $bwlimit * 1024;
7808         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7809     } else {
7810         print "drive mirror is starting for drive-$drive\n";
7811     }
7812
7813     # if a job already runs for this device we get an error, catch it for cleanup
7814     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7815     if (my $err = $@) {
7816         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7817         warn "$@\n" if $@;
7818         die "mirroring error: $err\n";
7819     }
7820
7821     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7822 }
7823
7824 # $completion can be either
7825 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7826 # 'cancel': wait until all jobs are ready, block-job-cancel them
7827 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7828 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7829 sub qemu_drive_mirror_monitor {
7830     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7831
7832     $completion //= 'complete';
7833     $op //= "mirror";
7834
7835     eval {
7836         my $err_complete = 0;
7837
7838         my $starttime = time ();
7839         while (1) {
7840             die "block job ('$op') timed out\n" if $err_complete > 300;
7841
7842             my $stats = mon_cmd($vmid, "query-block-jobs");
7843             my $ctime = time();
7844
7845             my $running_jobs = {};
7846             for my $stat (@$stats) {
7847                 next if $stat->{type} ne $op;
7848                 $running_jobs->{$stat->{device}} = $stat;
7849             }
7850
7851             my $readycounter = 0;
7852
7853             for my $job_id (sort keys %$jobs) {
7854                 my $job = $running_jobs->{$job_id};
7855
7856                 my $vanished = !defined($job);
7857                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7858                 if($complete || ($vanished && $completion eq 'auto')) {
7859                     print "$job_id: $op-job finished\n";
7860                     delete $jobs->{$job_id};
7861                     next;
7862                 }
7863
7864                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7865
7866                 my $busy = $job->{busy};
7867                 my $ready = $job->{ready};
7868                 if (my $total = $job->{len}) {
7869                     my $transferred = $job->{offset} || 0;
7870                     my $remaining = $total - $transferred;
7871                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7872
7873                     my $duration = $ctime - $starttime;
7874                     my $total_h = render_bytes($total, 1);
7875                     my $transferred_h = render_bytes($transferred, 1);
7876
7877                     my $status = sprintf(
7878                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7879                         render_duration($duration),
7880                     );
7881
7882                     if ($ready) {
7883                         if ($busy) {
7884                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7885                         } else {
7886                             $status .= ", ready";
7887                         }
7888                     }
7889                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7890                     $jobs->{$job_id}->{ready} = $ready;
7891                 }
7892
7893                 $readycounter++ if $job->{ready};
7894             }
7895
7896             last if scalar(keys %$jobs) == 0;
7897
7898             if ($readycounter == scalar(keys %$jobs)) {
7899                 print "all '$op' jobs are ready\n";
7900
7901                 # do the complete later (or has already been done)
7902                 last if $completion eq 'skip' || $completion eq 'auto';
7903
7904                 if ($vmiddst && $vmiddst != $vmid) {
7905                     my $agent_running = $qga && qga_check_running($vmid);
7906                     if ($agent_running) {
7907                         print "freeze filesystem\n";
7908                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7909                         warn $@ if $@;
7910                     } else {
7911                         print "suspend vm\n";
7912                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7913                         warn $@ if $@;
7914                     }
7915
7916                     # if we clone a disk for a new target vm, we don't switch the disk
7917                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7918
7919                     if ($agent_running) {
7920                         print "unfreeze filesystem\n";
7921                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7922                         warn $@ if $@;
7923                     } else {
7924                         print "resume vm\n";
7925                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7926                         warn $@ if $@;
7927                     }
7928
7929                     last;
7930                 } else {
7931
7932                     for my $job_id (sort keys %$jobs) {
7933                         # try to switch the disk if source and destination are on the same guest
7934                         print "$job_id: Completing block job_id...\n";
7935
7936                         my $op;
7937                         if ($completion eq 'complete') {
7938                             $op = 'block-job-complete';
7939                         } elsif ($completion eq 'cancel') {
7940                             $op = 'block-job-cancel';
7941                         } else {
7942                             die "invalid completion value: $completion\n";
7943                         }
7944                         eval { mon_cmd($vmid, $op, device => $job_id) };
7945                         if ($@ =~ m/cannot be completed/) {
7946                             print "$job_id: block job cannot be completed, trying again.\n";
7947                             $err_complete++;
7948                         }else {
7949                             print "$job_id: Completed successfully.\n";
7950                             $jobs->{$job_id}->{complete} = 1;
7951                         }
7952                     }
7953                 }
7954             }
7955             sleep 1;
7956         }
7957     };
7958     my $err = $@;
7959
7960     if ($err) {
7961         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7962         die "block job ($op) error: $err";
7963     }
7964 }
7965
7966 sub qemu_blockjobs_cancel {
7967     my ($vmid, $jobs) = @_;
7968
7969     foreach my $job (keys %$jobs) {
7970         print "$job: Cancelling block job\n";
7971         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7972         $jobs->{$job}->{cancel} = 1;
7973     }
7974
7975     while (1) {
7976         my $stats = mon_cmd($vmid, "query-block-jobs");
7977
7978         my $running_jobs = {};
7979         foreach my $stat (@$stats) {
7980             $running_jobs->{$stat->{device}} = $stat;
7981         }
7982
7983         foreach my $job (keys %$jobs) {
7984
7985             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7986                 print "$job: Done.\n";
7987                 delete $jobs->{$job};
7988             }
7989         }
7990
7991         last if scalar(keys %$jobs) == 0;
7992
7993         sleep 1;
7994     }
7995 }
7996
7997 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7998 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7999 my sub clone_disk_check_io_uring {
8000     my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8001
8002     return if !$use_drive_mirror;
8003
8004     # Don't complain when not changing storage.
8005     # Assume if it works for the source, it'll work for the target too.
8006     return if $src_storeid eq $dst_storeid;
8007
8008     my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8009     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8010
8011     my $cache_direct = drive_uses_cache_direct($src_drive);
8012
8013     my $src_uses_io_uring;
8014     if ($src_drive->{aio}) {
8015         $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8016     } else {
8017         $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8018     }
8019
8020     die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8021         if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8022 }
8023
8024 sub clone_disk {
8025     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8026
8027     my ($vmid, $running) = $source->@{qw(vmid running)};
8028     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8029
8030     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8031     my ($storage, $format) = $dest->@{qw(storage format)};
8032
8033     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8034
8035     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8036         die "cloning from/to EFI disk requires EFI disk\n"
8037             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8038         die "cloning from/to TPM state requires TPM state\n"
8039             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8040
8041         # This would lead to two device nodes in QEMU pointing to the same backing image!
8042         die "cannot change drive name when cloning disk from/to the same VM\n"
8043             if $use_drive_mirror && $vmid == $newvmid;
8044     }
8045
8046     die "cannot move TPM state while VM is running\n"
8047         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8048
8049     my $newvolid;
8050
8051     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8052     print "$src_drivename " if $src_drivename;
8053     print "($drive->{file})\n";
8054
8055     if (!$full) {
8056         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
8057         push @$newvollist, $newvolid;
8058     } else {
8059         my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8060         my $storeid = $storage || $src_storeid;
8061
8062         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8063
8064         my $name = undef;
8065         my $size = undef;
8066         if (drive_is_cloudinit($drive)) {
8067             $name = "vm-$newvmid-cloudinit";
8068             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8069             if ($scfg->{path}) {
8070                 $name .= ".$dst_format";
8071             }
8072             $snapname = undef;
8073             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8074         } elsif ($dst_drivename eq 'efidisk0') {
8075             $size = $efisize or die "internal error - need to specify EFI disk size\n";
8076         } elsif ($dst_drivename eq 'tpmstate0') {
8077             $dst_format = 'raw';
8078             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8079         } else {
8080             clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8081
8082             $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8083         }
8084         $newvolid = PVE::Storage::vdisk_alloc(
8085             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8086         );
8087         push @$newvollist, $newvolid;
8088
8089         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8090
8091         if (drive_is_cloudinit($drive)) {
8092             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8093             # if this is the case, we have to complete any block-jobs still there from
8094             # previous drive-mirrors
8095             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8096                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8097             }
8098             goto no_data_clone;
8099         }
8100
8101         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8102         if ($use_drive_mirror) {
8103             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8104                 $completion, $qga, $bwlimit);
8105         } else {
8106             if ($dst_drivename eq 'efidisk0') {
8107                 # the relevant data on the efidisk may be smaller than the source
8108                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8109                 # that is given by the OVMF_VARS.fd
8110                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8111                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8112
8113                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8114
8115                 # better for Ceph if block size is not too small, see bug #3324
8116                 my $bs = 1024*1024;
8117
8118                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8119
8120                 if ($src_format eq 'qcow2' && $snapname) {
8121                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8122                         if !min_version(kvm_user_version(), 6, 2);
8123                     push $cmd->@*, '-l', $snapname;
8124                 }
8125                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8126                 run_command($cmd);
8127             } else {
8128                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8129             }
8130         }
8131     }
8132
8133 no_data_clone:
8134     my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8135
8136     my $disk = dclone($drive);
8137     delete $disk->{format};
8138     $disk->{file} = $newvolid;
8139     $disk->{size} = $size if defined($size);
8140
8141     return $disk;
8142 }
8143
8144 sub get_running_qemu_version {
8145     my ($vmid) = @_;
8146     my $res = mon_cmd($vmid, "query-version");
8147     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8148 }
8149
8150 sub qemu_use_old_bios_files {
8151     my ($machine_type) = @_;
8152
8153     return if !$machine_type;
8154
8155     my $use_old_bios_files = undef;
8156
8157     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8158         $machine_type = $1;
8159         $use_old_bios_files = 1;
8160     } else {
8161         my $version = extract_version($machine_type, kvm_user_version());
8162         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8163         # load new efi bios files on migration. So this hack is required to allow
8164         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8165         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8166         $use_old_bios_files = !min_version($version, 2, 4);
8167     }
8168
8169     return ($use_old_bios_files, $machine_type);
8170 }
8171
8172 sub get_efivars_size {
8173     my ($conf, $efidisk) = @_;
8174
8175     my $arch = get_vm_arch($conf);
8176     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8177     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8178     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8179     return -s $ovmf_vars;
8180 }
8181
8182 sub update_efidisk_size {
8183     my ($conf) = @_;
8184
8185     return if !defined($conf->{efidisk0});
8186
8187     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8188     $disk->{size} = get_efivars_size($conf);
8189     $conf->{efidisk0} = print_drive($disk);
8190
8191     return;
8192 }
8193
8194 sub update_tpmstate_size {
8195     my ($conf) = @_;
8196
8197     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8198     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8199     $conf->{tpmstate0} = print_drive($disk);
8200 }
8201
8202 sub create_efidisk($$$$$$$) {
8203     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8204
8205     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8206
8207     my $vars_size_b = -s $ovmf_vars;
8208     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8209     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8210     PVE::Storage::activate_volumes($storecfg, [$volid]);
8211
8212     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8213     my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8214
8215     return ($volid, $size/1024);
8216 }
8217
8218 sub vm_iothreads_list {
8219     my ($vmid) = @_;
8220
8221     my $res = mon_cmd($vmid, 'query-iothreads');
8222
8223     my $iothreads = {};
8224     foreach my $iothread (@$res) {
8225         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8226     }
8227
8228     return $iothreads;
8229 }
8230
8231 sub scsihw_infos {
8232     my ($conf, $drive) = @_;
8233
8234     my $maxdev = 0;
8235
8236     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8237         $maxdev = 7;
8238     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8239         $maxdev = 1;
8240     } else {
8241         $maxdev = 256;
8242     }
8243
8244     my $controller = int($drive->{index} / $maxdev);
8245     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8246         ? "virtioscsi"
8247         : "scsihw";
8248
8249     return ($maxdev, $controller, $controller_prefix);
8250 }
8251
8252 sub resolve_dst_disk_format {
8253         my ($storecfg, $storeid, $src_volname, $format) = @_;
8254         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8255
8256         if (!$format) {
8257             # if no target format is specified, use the source disk format as hint
8258             if ($src_volname) {
8259                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8260                 $format = qemu_img_format($scfg, $src_volname);
8261             } else {
8262                 return $defFormat;
8263             }
8264         }
8265
8266         # test if requested format is supported - else use default
8267         my $supported = grep { $_ eq $format } @$validFormats;
8268         $format = $defFormat if !$supported;
8269         return $format;
8270 }
8271
8272 # NOTE: if this logic changes, please update docs & possibly gui logic
8273 sub find_vmstate_storage {
8274     my ($conf, $storecfg) = @_;
8275
8276     # first, return storage from conf if set
8277     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8278
8279     my ($target, $shared, $local);
8280
8281     foreach_storage_used_by_vm($conf, sub {
8282         my ($sid) = @_;
8283         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8284         my $dst = $scfg->{shared} ? \$shared : \$local;
8285         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8286     });
8287
8288     # second, use shared storage where VM has at least one disk
8289     # third, use local storage where VM has at least one disk
8290     # fall back to local storage
8291     $target = $shared // $local // 'local';
8292
8293     return $target;
8294 }
8295
8296 sub generate_uuid {
8297     my ($uuid, $uuid_str);
8298     UUID::generate($uuid);
8299     UUID::unparse($uuid, $uuid_str);
8300     return $uuid_str;
8301 }
8302
8303 sub generate_smbios1_uuid {
8304     return "uuid=".generate_uuid();
8305 }
8306
8307 sub nbd_stop {
8308     my ($vmid) = @_;
8309
8310     mon_cmd($vmid, 'nbd-server-stop');
8311 }
8312
8313 sub create_reboot_request {
8314     my ($vmid) = @_;
8315     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8316         or die "failed to create reboot trigger file: $!\n";
8317     close($fh);
8318 }
8319
8320 sub clear_reboot_request {
8321     my ($vmid) = @_;
8322     my $path = "/run/qemu-server/$vmid.reboot";
8323     my $res = 0;
8324
8325     $res = unlink($path);
8326     die "could not remove reboot request for $vmid: $!"
8327         if !$res && $! != POSIX::ENOENT;
8328
8329     return $res;
8330 }
8331
8332 sub bootorder_from_legacy {
8333     my ($conf, $bootcfg) = @_;
8334
8335     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8336     my $bootindex_hash = {};
8337     my $i = 1;
8338     foreach my $o (split(//, $boot)) {
8339         $bootindex_hash->{$o} = $i*100;
8340         $i++;
8341     }
8342
8343     my $bootorder = {};
8344
8345     PVE::QemuConfig->foreach_volume($conf, sub {
8346         my ($ds, $drive) = @_;
8347
8348         if (drive_is_cdrom ($drive, 1)) {
8349             if ($bootindex_hash->{d}) {
8350                 $bootorder->{$ds} = $bootindex_hash->{d};
8351                 $bootindex_hash->{d} += 1;
8352             }
8353         } elsif ($bootindex_hash->{c}) {
8354             $bootorder->{$ds} = $bootindex_hash->{c}
8355                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8356             $bootindex_hash->{c} += 1;
8357         }
8358     });
8359
8360     if ($bootindex_hash->{n}) {
8361         for (my $i = 0; $i < $MAX_NETS; $i++) {
8362             my $netname = "net$i";
8363             next if !$conf->{$netname};
8364             $bootorder->{$netname} = $bootindex_hash->{n};
8365             $bootindex_hash->{n} += 1;
8366         }
8367     }
8368
8369     return $bootorder;
8370 }
8371
8372 # Generate default device list for 'boot: order=' property. Matches legacy
8373 # default boot order, but with explicit device names. This is important, since
8374 # the fallback for when neither 'order' nor the old format is specified relies
8375 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8376 sub get_default_bootdevices {
8377     my ($conf) = @_;
8378
8379     my @ret = ();
8380
8381     # harddisk
8382     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8383     push @ret, $first if $first;
8384
8385     # cdrom
8386     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8387     push @ret, $first if $first;
8388
8389     # network
8390     for (my $i = 0; $i < $MAX_NETS; $i++) {
8391         my $netname = "net$i";
8392         next if !$conf->{$netname};
8393         push @ret, $netname;
8394         last;
8395     }
8396
8397     return \@ret;
8398 }
8399
8400 sub device_bootorder {
8401     my ($conf) = @_;
8402
8403     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8404
8405     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8406
8407     my $bootorder = {};
8408     if (!defined($boot) || $boot->{legacy}) {
8409         $bootorder = bootorder_from_legacy($conf, $boot);
8410     } elsif ($boot->{order}) {
8411         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8412         for my $dev (PVE::Tools::split_list($boot->{order})) {
8413             $bootorder->{$dev} = $i++;
8414         }
8415     }
8416
8417     return $bootorder;
8418 }
8419
8420 sub register_qmeventd_handle {
8421     my ($vmid) = @_;
8422
8423     my $fh;
8424     my $peer = "/var/run/qmeventd.sock";
8425     my $count = 0;
8426
8427     for (;;) {
8428         $count++;
8429         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8430         last if $fh;
8431         if ($! != EINTR && $! != EAGAIN) {
8432             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8433         }
8434         if ($count > 4) {
8435             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8436               . "after $count retries\n";
8437         }
8438         usleep(25000);
8439     }
8440
8441     # send handshake to mark VM as backing up
8442     print $fh to_json({vzdump => {vmid => "$vmid"}});
8443
8444     # return handle to be closed later when inhibit is no longer required
8445     return $fh;
8446 }
8447
8448 # bash completion helper
8449
8450 sub complete_backup_archives {
8451     my ($cmdname, $pname, $cvalue) = @_;
8452
8453     my $cfg = PVE::Storage::config();
8454
8455     my $storeid;
8456
8457     if ($cvalue =~ m/^([^:]+):/) {
8458         $storeid = $1;
8459     }
8460
8461     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8462
8463     my $res = [];
8464     foreach my $id (keys %$data) {
8465         foreach my $item (@{$data->{$id}}) {
8466             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8467             push @$res, $item->{volid} if defined($item->{volid});
8468         }
8469     }
8470
8471     return $res;
8472 }
8473
8474 my $complete_vmid_full = sub {
8475     my ($running) = @_;
8476
8477     my $idlist = vmstatus();
8478
8479     my $res = [];
8480
8481     foreach my $id (keys %$idlist) {
8482         my $d = $idlist->{$id};
8483         if (defined($running)) {
8484             next if $d->{template};
8485             next if $running && $d->{status} ne 'running';
8486             next if !$running && $d->{status} eq 'running';
8487         }
8488         push @$res, $id;
8489
8490     }
8491     return $res;
8492 };
8493
8494 sub complete_vmid {
8495     return &$complete_vmid_full();
8496 }
8497
8498 sub complete_vmid_stopped {
8499     return &$complete_vmid_full(0);
8500 }
8501
8502 sub complete_vmid_running {
8503     return &$complete_vmid_full(1);
8504 }
8505
8506 sub complete_storage {
8507
8508     my $cfg = PVE::Storage::config();
8509     my $ids = $cfg->{ids};
8510
8511     my $res = [];
8512     foreach my $sid (keys %$ids) {
8513         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8514         next if !$ids->{$sid}->{content}->{images};
8515         push @$res, $sid;
8516     }
8517
8518     return $res;
8519 }
8520
8521 sub complete_migration_storage {
8522     my ($cmd, $param, $current_value, $all_args) = @_;
8523
8524     my $targetnode = @$all_args[1];
8525
8526     my $cfg = PVE::Storage::config();
8527     my $ids = $cfg->{ids};
8528
8529     my $res = [];
8530     foreach my $sid (keys %$ids) {
8531         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8532         next if !$ids->{$sid}->{content}->{images};
8533         push @$res, $sid;
8534     }
8535
8536     return $res;
8537 }
8538
8539 sub vm_is_paused {
8540     my ($vmid) = @_;
8541     my $qmpstatus = eval {
8542         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8543         mon_cmd($vmid, "query-status");
8544     };
8545     warn "$@\n" if $@;
8546     return $qmpstatus && $qmpstatus->{status} eq "paused";
8547 }
8548
8549 sub check_volume_storage_type {
8550     my ($storecfg, $vol) = @_;
8551
8552     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8553     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8554     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8555
8556     die "storage '$storeid' does not support content-type '$vtype'\n"
8557         if !$scfg->{content}->{$vtype};
8558
8559     return 1;
8560 }
8561
8562 sub add_nets_bridge_fdb {
8563     my ($conf, $vmid) = @_;
8564
8565     for my $opt (keys %$conf) {
8566         next if $opt !~ m/^net(\d+)$/;
8567         my $iface = "tap${vmid}i$1";
8568         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8569         my $net = parse_net($conf->{$opt}, 1) or next;
8570
8571         my $mac = $net->{macaddr};
8572         if (!$mac) {
8573             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8574                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8575             next;
8576         }
8577
8578         my $bridge = $net->{bridge};
8579         if (!$bridge) {
8580             log_warn("Interface '$iface' not attached to any bridge.");
8581             next;
8582         }
8583         if ($have_sdn) {
8584             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8585         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8586             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8587         }
8588     }
8589 }
8590
8591 sub del_nets_bridge_fdb {
8592     my ($conf, $vmid) = @_;
8593
8594     for my $opt (keys %$conf) {
8595         next if $opt !~ m/^net(\d+)$/;
8596         my $iface = "tap${vmid}i$1";
8597
8598         my $net = parse_net($conf->{$opt}) or next;
8599         my $mac = $net->{macaddr} or next;
8600
8601         my $bridge = $net->{bridge};
8602         if ($have_sdn) {
8603             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8604         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8605             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8606         }
8607     }
8608 }
8609
8610 1;